importer: dedupe refinement — min(ns,nb) completeness pairing (stretch I)
- cross-province groups now pair ns<->nb by completeness (img > km > older) up to min per region instead of collapsing the whole group to one row; surplus siblings (multi-unit dealer ads) stay active — no false merges - verified on live DB: dup 80 -> 77, active 560 -> 563, total 640; the 3 freed rows are the BMW G 310 R dealer siblings (450/451/452); deck top order unchanged (Indian Scout 91 x3 then Vulcan 900 73); re-import idempotent (77 stable) - selfcheck +2: multi-unit pairing semantics, reversibility (flip to active + re-import re-derives the pairing) — 16/16 green
This commit is contained in:
@@ -162,10 +162,18 @@ def upsert(conn, row, now):
|
||||
|
||||
|
||||
def dedupe(conn, present):
|
||||
"""Collapse cross-province twins. `present` maps (region, src_id) -> db_id.
|
||||
Groups by (normalized_title, price_num); a group spanning both provinces
|
||||
keeps the most-complete row canonical (img > km > older) and marks the rest
|
||||
'dup'. The twin's URL is preserved on the canonical row's desc."""
|
||||
"""Collapse cross-province twins with min(ns,nb) completeness pairing.
|
||||
|
||||
`present` maps (region, src_id) -> db_id. Groups by
|
||||
(normalized_title, price_num). For a group spanning both provinces the
|
||||
two region lists are sorted best-first (img present > km present > older)
|
||||
and paired one-to-one up to min(ns_count, nb_count): each pair keeps the
|
||||
more-complete row canonical and marks the twin 'dup'. Surplus siblings
|
||||
(multi-unit dealer ads) stay active instead of being false-merged.
|
||||
The twin's URL is preserved on the canonical row's desc. (DATA-MODEL P2
|
||||
addendum: pairing replaces collapse-to-one — dup ≈ 77 vs 80 on today's
|
||||
snapshot.) Reversible: dubs carry dup_of; flipping status back to
|
||||
'active' and re-importing re-derives the pairing."""
|
||||
groups = {}
|
||||
for (region, src_id), db_id in present.items():
|
||||
row = conn.execute(
|
||||
@@ -174,31 +182,32 @@ def dedupe(conn, present):
|
||||
key = (norm_title(row['title']), row['price_num'])
|
||||
groups.setdefault(key, []).append((region, db_id, row))
|
||||
|
||||
def completeness(m):
|
||||
"""Best first: img present, then km present, then older row."""
|
||||
return (0 if m[2]['img_url'] else 1,
|
||||
0 if m[2]['km'] is not None else 1,
|
||||
m[2]['first_seen'] or '')
|
||||
|
||||
dup_count = 0
|
||||
for key, members in groups.items():
|
||||
if len({m[0] for m in members}) < 2:
|
||||
continue # not cross-province
|
||||
# most complete first: img present, then km present, then older row
|
||||
members.sort(key=lambda m: (
|
||||
0 if m[2]['img_url'] else 1,
|
||||
0 if m[2]['km'] is not None else 1,
|
||||
m[2]['first_seen'] or ''))
|
||||
canonical = members[0]
|
||||
twin_urls = []
|
||||
for region, db_id, row in members[1:]:
|
||||
ns = sorted((m for m in members if m[0] == 'ns'), key=completeness)
|
||||
nb = sorted((m for m in members if m[0] == 'nb'), key=completeness)
|
||||
for a, b in zip(ns, nb):
|
||||
canonical, twin = (a, b) if completeness(a) <= completeness(b) else (b, a)
|
||||
conn.execute(
|
||||
"UPDATE listings SET status='dup', dup_of=?, updated_at=? WHERE id=?",
|
||||
(canonical[1], now_iso(), db_id))
|
||||
(canonical[1], now_iso(), twin[1]))
|
||||
dup_count += 1
|
||||
if row['url']:
|
||||
twin_urls.append(row['url'])
|
||||
if twin_urls:
|
||||
# preserve twin URLs on the canonical row (desc suffix), after scoring
|
||||
cur = conn.execute('SELECT desc FROM listings WHERE id=?', (canonical[1],))
|
||||
base_desc = cur.fetchone()['desc'] or ''
|
||||
suffix = ' [also listed: ' + ' | '.join(twin_urls) + ']'
|
||||
conn.execute('UPDATE listings SET desc=? WHERE id=?',
|
||||
(base_desc + suffix, canonical[1]))
|
||||
if twin[2]['url']:
|
||||
# preserve the twin's URL on the canonical row (desc suffix)
|
||||
cur = conn.execute('SELECT desc FROM listings WHERE id=?',
|
||||
(canonical[1],))
|
||||
base_desc = cur.fetchone()['desc'] or ''
|
||||
suffix = ' [also listed: ' + twin[2]['url'] + ']'
|
||||
conn.execute('UPDATE listings SET desc=? WHERE id=?',
|
||||
(base_desc + suffix, canonical[1]))
|
||||
return dup_count
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user