importer: dedupe refinement — min(ns,nb) completeness pairing (stretch I)

- cross-province groups now pair ns<->nb by completeness (img > km > older)
  up to min per region instead of collapsing the whole group to one row;
  surplus siblings (multi-unit dealer ads) stay active — no false merges
- verified on live DB: dup 80 -> 77, active 560 -> 563, total 640; the 3
  freed rows are the BMW G 310 R dealer siblings (450/451/452); deck top
  order unchanged (Indian Scout 91 x3 then Vulcan 900 73); re-import
  idempotent (77 stable)
- selfcheck +2: multi-unit pairing semantics, reversibility (flip to
  active + re-import re-derives the pairing) — 16/16 green
This commit is contained in:
Torres
2026-11-10 21:25:02 -04:00
parent 8367e2b7c8
commit d7fb694626
2 changed files with 107 additions and 22 deletions

View File

@@ -162,10 +162,18 @@ def upsert(conn, row, now):
def dedupe(conn, present):
"""Collapse cross-province twins. `present` maps (region, src_id) -> db_id.
Groups by (normalized_title, price_num); a group spanning both provinces
keeps the most-complete row canonical (img > km > older) and marks the rest
'dup'. The twin's URL is preserved on the canonical row's desc."""
"""Collapse cross-province twins with min(ns,nb) completeness pairing.
`present` maps (region, src_id) -> db_id. Groups by
(normalized_title, price_num). For a group spanning both provinces the
two region lists are sorted best-first (img present > km present > older)
and paired one-to-one up to min(ns_count, nb_count): each pair keeps the
more-complete row canonical and marks the twin 'dup'. Surplus siblings
(multi-unit dealer ads) stay active instead of being false-merged.
The twin's URL is preserved on the canonical row's desc. (DATA-MODEL P2
addendum: pairing replaces collapse-to-one — dup ≈ 77 vs 80 on today's
snapshot.) Reversible: dubs carry dup_of; flipping status back to
'active' and re-importing re-derives the pairing."""
groups = {}
for (region, src_id), db_id in present.items():
row = conn.execute(
@@ -174,31 +182,32 @@ def dedupe(conn, present):
key = (norm_title(row['title']), row['price_num'])
groups.setdefault(key, []).append((region, db_id, row))
def completeness(m):
"""Best first: img present, then km present, then older row."""
return (0 if m[2]['img_url'] else 1,
0 if m[2]['km'] is not None else 1,
m[2]['first_seen'] or '')
dup_count = 0
for key, members in groups.items():
if len({m[0] for m in members}) < 2:
continue # not cross-province
# most complete first: img present, then km present, then older row
members.sort(key=lambda m: (
0 if m[2]['img_url'] else 1,
0 if m[2]['km'] is not None else 1,
m[2]['first_seen'] or ''))
canonical = members[0]
twin_urls = []
for region, db_id, row in members[1:]:
ns = sorted((m for m in members if m[0] == 'ns'), key=completeness)
nb = sorted((m for m in members if m[0] == 'nb'), key=completeness)
for a, b in zip(ns, nb):
canonical, twin = (a, b) if completeness(a) <= completeness(b) else (b, a)
conn.execute(
"UPDATE listings SET status='dup', dup_of=?, updated_at=? WHERE id=?",
(canonical[1], now_iso(), db_id))
(canonical[1], now_iso(), twin[1]))
dup_count += 1
if row['url']:
twin_urls.append(row['url'])
if twin_urls:
# preserve twin URLs on the canonical row (desc suffix), after scoring
cur = conn.execute('SELECT desc FROM listings WHERE id=?', (canonical[1],))
base_desc = cur.fetchone()['desc'] or ''
suffix = ' [also listed: ' + ' | '.join(twin_urls) + ']'
conn.execute('UPDATE listings SET desc=? WHERE id=?',
(base_desc + suffix, canonical[1]))
if twin[2]['url']:
# preserve the twin's URL on the canonical row (desc suffix)
cur = conn.execute('SELECT desc FROM listings WHERE id=?',
(canonical[1],))
base_desc = cur.fetchone()['desc'] or ''
suffix = ' [also listed: ' + twin[2]['url'] + ']'
conn.execute('UPDATE listings SET desc=? WHERE id=?',
(base_desc + suffix, canonical[1]))
return dup_count