importer: dedupe refinement — min(ns,nb) completeness pairing (stretch I)

- cross-province groups now pair ns<->nb by completeness (img > km > older)
  up to min per region instead of collapsing the whole group to one row;
  surplus siblings (multi-unit dealer ads) stay active — no false merges
- verified on live DB: dup 80 -> 77, active 560 -> 563, total 640; the 3
  freed rows are the BMW G 310 R dealer siblings (450/451/452); deck top
  order unchanged (Indian Scout 91 x3 then Vulcan 900 73); re-import
  idempotent (77 stable)
- selfcheck +2: multi-unit pairing semantics, reversibility (flip to
  active + re-import re-derives the pairing) — 16/16 green
This commit is contained in:
Torres
2026-11-10 21:25:02 -04:00
parent 8367e2b7c8
commit d7fb694626
2 changed files with 107 additions and 22 deletions

View File

@@ -219,12 +219,88 @@ def test_gone_full_flow():
shutil.rmtree(d, ignore_errors=True)
def test_dedupe_multiunit_pairing():
# Cross-province group with 2 ns + 1 nb: min(ns,nb) = 1 pair -> 1 dup;
# the surplus ns sibling stays active (no false merge of distinct bikes).
d, db_path = fresh_db()
try:
state_dir = os.path.join(d, 'state')
twin = '2025 BMW G 310 R'
write_states(state_dir,
make_state({'ns1': make_item(twin, price_num=5444, img='http://x/1.jpg'),
'ns2': make_item(twin, price_num=5444)}),
make_state({'nb1': make_item(twin, price_num=5444)}))
res = imp.run_import(state_dir, db_path)
assert res['ok'] and res['total'] == 3 and res['dup'] == 1, res
conn = db.connect(db_path)
db.init_db(conn)
nb = conn.execute(
"SELECT status, dup_of FROM listings WHERE region='nb'").fetchone()
assert nb['status'] == 'dup', nb
canon = conn.execute(
'SELECT region, status, img_url FROM listings WHERE id=?',
(nb['dup_of'],)).fetchone()
assert canon['region'] == 'ns' and canon['status'] == 'active'
assert canon['img_url'] is not None # best-complete ns was canonical
# the surplus ns sibling stays active
active_ns = conn.execute(
"SELECT COUNT(*) AS n FROM listings WHERE region='ns' AND status='active'"
).fetchone()['n']
assert active_ns == 2, active_ns
# the paired twin's URL is preserved on the canonical's desc
desc = conn.execute('SELECT desc FROM listings WHERE id=?',
(nb['dup_of'],)).fetchone()['desc'] or ''
conn.close()
assert 'also listed' in desc
finally:
shutil.rmtree(d, ignore_errors=True)
def test_dedupe_reversible():
# A dup can be flipped active by hand; the next import re-derives it.
d, db_path = fresh_db()
try:
state_dir = os.path.join(d, 'state')
twin = make_item('2021 Indian Scout Bobber', price_num=6500, km='8000')
write_states(state_dir,
make_state({'ns1': twin}),
make_state({'nb1': dict(twin)}))
imp.run_import(state_dir, db_path)
conn = db.connect(db_path)
db.init_db(conn)
nb_id = conn.execute(
"SELECT id FROM listings WHERE region='nb'").fetchone()['id']
conn.execute("UPDATE listings SET status='active', dup_of=NULL WHERE id=?",
(nb_id,))
conn.commit()
assert conn.execute(
"SELECT COUNT(*) AS n FROM listings WHERE status='dup'"
).fetchone()['n'] == 0
assert conn.execute(
"SELECT COUNT(*) AS n FROM listings WHERE status='active'"
).fetchone()['n'] == 2
conn.close()
imp.run_import(state_dir, db_path) # re-import re-pairs it
conn = db.connect(db_path)
db.init_db(conn)
dup, active = (conn.execute(
"SELECT COUNT(*) AS n FROM listings WHERE status='dup'").fetchone()['n'],
conn.execute(
"SELECT COUNT(*) AS n FROM listings WHERE status='active'").fetchone()['n'])
conn.close()
assert (dup, active) == (1, 1), (dup, active)
finally:
shutil.rmtree(d, ignore_errors=True)
TESTS = [
('gate rejections', test_gate_rejections),
('gate abort leaves DB untouched', test_gate_abort_leaves_db),
('dedupe cross-province twins', test_dedupe_twins),
('dedupe false-positive (same province)', test_dedupe_false_positive_same_province),
('dedupe false-positive (price differs)', test_dedupe_false_positive_price_diff),
('dedupe multi-unit pairing (min(ns,nb))', test_dedupe_multiunit_pairing),
('dedupe reversibility (flip + re-import)', test_dedupe_reversible),
('scoring spot-checks', test_scoring),
('gone after 2 misses (unit)', test_gone_after_two_misses),
('gone after 2 misses (full flow)', test_gone_full_flow),