diff --git a/app/importer.py b/app/importer.py index e9fab6c..3d5446d 100644 --- a/app/importer.py +++ b/app/importer.py @@ -162,10 +162,18 @@ def upsert(conn, row, now): def dedupe(conn, present): - """Collapse cross-province twins. `present` maps (region, src_id) -> db_id. - Groups by (normalized_title, price_num); a group spanning both provinces - keeps the most-complete row canonical (img > km > older) and marks the rest - 'dup'. The twin's URL is preserved on the canonical row's desc.""" + """Collapse cross-province twins with min(ns,nb) completeness pairing. + + `present` maps (region, src_id) -> db_id. Groups by + (normalized_title, price_num). For a group spanning both provinces the + two region lists are sorted best-first (img present > km present > older) + and paired one-to-one up to min(ns_count, nb_count): each pair keeps the + more-complete row canonical and marks the twin 'dup'. Surplus siblings + (multi-unit dealer ads) stay active instead of being false-merged. + The twin's URL is preserved on the canonical row's desc. (DATA-MODEL P2 + addendum: pairing replaces collapse-to-one — dup ≈ 77 vs 80 on today's + snapshot.) Reversible: dubs carry dup_of; flipping status back to + 'active' and re-importing re-derives the pairing.""" groups = {} for (region, src_id), db_id in present.items(): row = conn.execute( @@ -174,31 +182,32 @@ def dedupe(conn, present): key = (norm_title(row['title']), row['price_num']) groups.setdefault(key, []).append((region, db_id, row)) + def completeness(m): + """Best first: img present, then km present, then older row.""" + return (0 if m[2]['img_url'] else 1, + 0 if m[2]['km'] is not None else 1, + m[2]['first_seen'] or '') + dup_count = 0 for key, members in groups.items(): if len({m[0] for m in members}) < 2: continue # not cross-province - # most complete first: img present, then km present, then older row - members.sort(key=lambda m: ( - 0 if m[2]['img_url'] else 1, - 0 if m[2]['km'] is not None else 1, - m[2]['first_seen'] or '')) - canonical = members[0] - twin_urls = [] - for region, db_id, row in members[1:]: + ns = sorted((m for m in members if m[0] == 'ns'), key=completeness) + nb = sorted((m for m in members if m[0] == 'nb'), key=completeness) + for a, b in zip(ns, nb): + canonical, twin = (a, b) if completeness(a) <= completeness(b) else (b, a) conn.execute( "UPDATE listings SET status='dup', dup_of=?, updated_at=? WHERE id=?", - (canonical[1], now_iso(), db_id)) + (canonical[1], now_iso(), twin[1])) dup_count += 1 - if row['url']: - twin_urls.append(row['url']) - if twin_urls: - # preserve twin URLs on the canonical row (desc suffix), after scoring - cur = conn.execute('SELECT desc FROM listings WHERE id=?', (canonical[1],)) - base_desc = cur.fetchone()['desc'] or '' - suffix = ' [also listed: ' + ' | '.join(twin_urls) + ']' - conn.execute('UPDATE listings SET desc=? WHERE id=?', - (base_desc + suffix, canonical[1])) + if twin[2]['url']: + # preserve the twin's URL on the canonical row (desc suffix) + cur = conn.execute('SELECT desc FROM listings WHERE id=?', + (canonical[1],)) + base_desc = cur.fetchone()['desc'] or '' + suffix = ' [also listed: ' + twin[2]['url'] + ']' + conn.execute('UPDATE listings SET desc=? WHERE id=?', + (base_desc + suffix, canonical[1])) return dup_count diff --git a/app/selfcheck.py b/app/selfcheck.py index 2dfd4e4..0e1ac11 100644 --- a/app/selfcheck.py +++ b/app/selfcheck.py @@ -219,12 +219,88 @@ def test_gone_full_flow(): shutil.rmtree(d, ignore_errors=True) +def test_dedupe_multiunit_pairing(): + # Cross-province group with 2 ns + 1 nb: min(ns,nb) = 1 pair -> 1 dup; + # the surplus ns sibling stays active (no false merge of distinct bikes). + d, db_path = fresh_db() + try: + state_dir = os.path.join(d, 'state') + twin = '2025 BMW G 310 R' + write_states(state_dir, + make_state({'ns1': make_item(twin, price_num=5444, img='http://x/1.jpg'), + 'ns2': make_item(twin, price_num=5444)}), + make_state({'nb1': make_item(twin, price_num=5444)})) + res = imp.run_import(state_dir, db_path) + assert res['ok'] and res['total'] == 3 and res['dup'] == 1, res + conn = db.connect(db_path) + db.init_db(conn) + nb = conn.execute( + "SELECT status, dup_of FROM listings WHERE region='nb'").fetchone() + assert nb['status'] == 'dup', nb + canon = conn.execute( + 'SELECT region, status, img_url FROM listings WHERE id=?', + (nb['dup_of'],)).fetchone() + assert canon['region'] == 'ns' and canon['status'] == 'active' + assert canon['img_url'] is not None # best-complete ns was canonical + # the surplus ns sibling stays active + active_ns = conn.execute( + "SELECT COUNT(*) AS n FROM listings WHERE region='ns' AND status='active'" + ).fetchone()['n'] + assert active_ns == 2, active_ns + # the paired twin's URL is preserved on the canonical's desc + desc = conn.execute('SELECT desc FROM listings WHERE id=?', + (nb['dup_of'],)).fetchone()['desc'] or '' + conn.close() + assert 'also listed' in desc + finally: + shutil.rmtree(d, ignore_errors=True) + + +def test_dedupe_reversible(): + # A dup can be flipped active by hand; the next import re-derives it. + d, db_path = fresh_db() + try: + state_dir = os.path.join(d, 'state') + twin = make_item('2021 Indian Scout Bobber', price_num=6500, km='8000') + write_states(state_dir, + make_state({'ns1': twin}), + make_state({'nb1': dict(twin)})) + imp.run_import(state_dir, db_path) + conn = db.connect(db_path) + db.init_db(conn) + nb_id = conn.execute( + "SELECT id FROM listings WHERE region='nb'").fetchone()['id'] + conn.execute("UPDATE listings SET status='active', dup_of=NULL WHERE id=?", + (nb_id,)) + conn.commit() + assert conn.execute( + "SELECT COUNT(*) AS n FROM listings WHERE status='dup'" + ).fetchone()['n'] == 0 + assert conn.execute( + "SELECT COUNT(*) AS n FROM listings WHERE status='active'" + ).fetchone()['n'] == 2 + conn.close() + imp.run_import(state_dir, db_path) # re-import re-pairs it + conn = db.connect(db_path) + db.init_db(conn) + dup, active = (conn.execute( + "SELECT COUNT(*) AS n FROM listings WHERE status='dup'").fetchone()['n'], + conn.execute( + "SELECT COUNT(*) AS n FROM listings WHERE status='active'").fetchone()['n']) + conn.close() + assert (dup, active) == (1, 1), (dup, active) + finally: + shutil.rmtree(d, ignore_errors=True) + + TESTS = [ ('gate rejections', test_gate_rejections), ('gate abort leaves DB untouched', test_gate_abort_leaves_db), ('dedupe cross-province twins', test_dedupe_twins), ('dedupe false-positive (same province)', test_dedupe_false_positive_same_province), ('dedupe false-positive (price differs)', test_dedupe_false_positive_price_diff), + ('dedupe multi-unit pairing (min(ns,nb))', test_dedupe_multiunit_pairing), + ('dedupe reversibility (flip + re-import)', test_dedupe_reversible), ('scoring spot-checks', test_scoring), ('gone after 2 misses (unit)', test_gone_after_two_misses), ('gone after 2 misses (full flow)', test_gone_full_flow),