importer: dedupe refinement — min(ns,nb) completeness pairing (stretch I)
- cross-province groups now pair ns<->nb by completeness (img > km > older) up to min per region instead of collapsing the whole group to one row; surplus siblings (multi-unit dealer ads) stay active — no false merges - verified on live DB: dup 80 -> 77, active 560 -> 563, total 640; the 3 freed rows are the BMW G 310 R dealer siblings (450/451/452); deck top order unchanged (Indian Scout 91 x3 then Vulcan 900 73); re-import idempotent (77 stable) - selfcheck +2: multi-unit pairing semantics, reversibility (flip to active + re-import re-derives the pairing) — 16/16 green
This commit is contained in:
@@ -219,12 +219,88 @@ def test_gone_full_flow():
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
def test_dedupe_multiunit_pairing():
|
||||
# Cross-province group with 2 ns + 1 nb: min(ns,nb) = 1 pair -> 1 dup;
|
||||
# the surplus ns sibling stays active (no false merge of distinct bikes).
|
||||
d, db_path = fresh_db()
|
||||
try:
|
||||
state_dir = os.path.join(d, 'state')
|
||||
twin = '2025 BMW G 310 R'
|
||||
write_states(state_dir,
|
||||
make_state({'ns1': make_item(twin, price_num=5444, img='http://x/1.jpg'),
|
||||
'ns2': make_item(twin, price_num=5444)}),
|
||||
make_state({'nb1': make_item(twin, price_num=5444)}))
|
||||
res = imp.run_import(state_dir, db_path)
|
||||
assert res['ok'] and res['total'] == 3 and res['dup'] == 1, res
|
||||
conn = db.connect(db_path)
|
||||
db.init_db(conn)
|
||||
nb = conn.execute(
|
||||
"SELECT status, dup_of FROM listings WHERE region='nb'").fetchone()
|
||||
assert nb['status'] == 'dup', nb
|
||||
canon = conn.execute(
|
||||
'SELECT region, status, img_url FROM listings WHERE id=?',
|
||||
(nb['dup_of'],)).fetchone()
|
||||
assert canon['region'] == 'ns' and canon['status'] == 'active'
|
||||
assert canon['img_url'] is not None # best-complete ns was canonical
|
||||
# the surplus ns sibling stays active
|
||||
active_ns = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE region='ns' AND status='active'"
|
||||
).fetchone()['n']
|
||||
assert active_ns == 2, active_ns
|
||||
# the paired twin's URL is preserved on the canonical's desc
|
||||
desc = conn.execute('SELECT desc FROM listings WHERE id=?',
|
||||
(nb['dup_of'],)).fetchone()['desc'] or ''
|
||||
conn.close()
|
||||
assert 'also listed' in desc
|
||||
finally:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
def test_dedupe_reversible():
|
||||
# A dup can be flipped active by hand; the next import re-derives it.
|
||||
d, db_path = fresh_db()
|
||||
try:
|
||||
state_dir = os.path.join(d, 'state')
|
||||
twin = make_item('2021 Indian Scout Bobber', price_num=6500, km='8000')
|
||||
write_states(state_dir,
|
||||
make_state({'ns1': twin}),
|
||||
make_state({'nb1': dict(twin)}))
|
||||
imp.run_import(state_dir, db_path)
|
||||
conn = db.connect(db_path)
|
||||
db.init_db(conn)
|
||||
nb_id = conn.execute(
|
||||
"SELECT id FROM listings WHERE region='nb'").fetchone()['id']
|
||||
conn.execute("UPDATE listings SET status='active', dup_of=NULL WHERE id=?",
|
||||
(nb_id,))
|
||||
conn.commit()
|
||||
assert conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE status='dup'"
|
||||
).fetchone()['n'] == 0
|
||||
assert conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE status='active'"
|
||||
).fetchone()['n'] == 2
|
||||
conn.close()
|
||||
imp.run_import(state_dir, db_path) # re-import re-pairs it
|
||||
conn = db.connect(db_path)
|
||||
db.init_db(conn)
|
||||
dup, active = (conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE status='dup'").fetchone()['n'],
|
||||
conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE status='active'").fetchone()['n'])
|
||||
conn.close()
|
||||
assert (dup, active) == (1, 1), (dup, active)
|
||||
finally:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
TESTS = [
|
||||
('gate rejections', test_gate_rejections),
|
||||
('gate abort leaves DB untouched', test_gate_abort_leaves_db),
|
||||
('dedupe cross-province twins', test_dedupe_twins),
|
||||
('dedupe false-positive (same province)', test_dedupe_false_positive_same_province),
|
||||
('dedupe false-positive (price differs)', test_dedupe_false_positive_price_diff),
|
||||
('dedupe multi-unit pairing (min(ns,nb))', test_dedupe_multiunit_pairing),
|
||||
('dedupe reversibility (flip + re-import)', test_dedupe_reversible),
|
||||
('scoring spot-checks', test_scoring),
|
||||
('gone after 2 misses (unit)', test_gone_after_two_misses),
|
||||
('gone after 2 misses (full flow)', test_gone_full_flow),
|
||||
|
||||
Reference in New Issue
Block a user