importer: dedupe refinement — min(ns,nb) completeness pairing (stretch I)
- cross-province groups now pair ns<->nb by completeness (img > km > older) up to min per region instead of collapsing the whole group to one row; surplus siblings (multi-unit dealer ads) stay active — no false merges - verified on live DB: dup 80 -> 77, active 560 -> 563, total 640; the 3 freed rows are the BMW G 310 R dealer siblings (450/451/452); deck top order unchanged (Indian Scout 91 x3 then Vulcan 900 73); re-import idempotent (77 stable) - selfcheck +2: multi-unit pairing semantics, reversibility (flip to active + re-import re-derives the pairing) — 16/16 green
This commit is contained in:
@@ -162,10 +162,18 @@ def upsert(conn, row, now):
|
|||||||
|
|
||||||
|
|
||||||
def dedupe(conn, present):
|
def dedupe(conn, present):
|
||||||
"""Collapse cross-province twins. `present` maps (region, src_id) -> db_id.
|
"""Collapse cross-province twins with min(ns,nb) completeness pairing.
|
||||||
Groups by (normalized_title, price_num); a group spanning both provinces
|
|
||||||
keeps the most-complete row canonical (img > km > older) and marks the rest
|
`present` maps (region, src_id) -> db_id. Groups by
|
||||||
'dup'. The twin's URL is preserved on the canonical row's desc."""
|
(normalized_title, price_num). For a group spanning both provinces the
|
||||||
|
two region lists are sorted best-first (img present > km present > older)
|
||||||
|
and paired one-to-one up to min(ns_count, nb_count): each pair keeps the
|
||||||
|
more-complete row canonical and marks the twin 'dup'. Surplus siblings
|
||||||
|
(multi-unit dealer ads) stay active instead of being false-merged.
|
||||||
|
The twin's URL is preserved on the canonical row's desc. (DATA-MODEL P2
|
||||||
|
addendum: pairing replaces collapse-to-one — dup ≈ 77 vs 80 on today's
|
||||||
|
snapshot.) Reversible: dubs carry dup_of; flipping status back to
|
||||||
|
'active' and re-importing re-derives the pairing."""
|
||||||
groups = {}
|
groups = {}
|
||||||
for (region, src_id), db_id in present.items():
|
for (region, src_id), db_id in present.items():
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
@@ -174,31 +182,32 @@ def dedupe(conn, present):
|
|||||||
key = (norm_title(row['title']), row['price_num'])
|
key = (norm_title(row['title']), row['price_num'])
|
||||||
groups.setdefault(key, []).append((region, db_id, row))
|
groups.setdefault(key, []).append((region, db_id, row))
|
||||||
|
|
||||||
|
def completeness(m):
|
||||||
|
"""Best first: img present, then km present, then older row."""
|
||||||
|
return (0 if m[2]['img_url'] else 1,
|
||||||
|
0 if m[2]['km'] is not None else 1,
|
||||||
|
m[2]['first_seen'] or '')
|
||||||
|
|
||||||
dup_count = 0
|
dup_count = 0
|
||||||
for key, members in groups.items():
|
for key, members in groups.items():
|
||||||
if len({m[0] for m in members}) < 2:
|
if len({m[0] for m in members}) < 2:
|
||||||
continue # not cross-province
|
continue # not cross-province
|
||||||
# most complete first: img present, then km present, then older row
|
ns = sorted((m for m in members if m[0] == 'ns'), key=completeness)
|
||||||
members.sort(key=lambda m: (
|
nb = sorted((m for m in members if m[0] == 'nb'), key=completeness)
|
||||||
0 if m[2]['img_url'] else 1,
|
for a, b in zip(ns, nb):
|
||||||
0 if m[2]['km'] is not None else 1,
|
canonical, twin = (a, b) if completeness(a) <= completeness(b) else (b, a)
|
||||||
m[2]['first_seen'] or ''))
|
|
||||||
canonical = members[0]
|
|
||||||
twin_urls = []
|
|
||||||
for region, db_id, row in members[1:]:
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE listings SET status='dup', dup_of=?, updated_at=? WHERE id=?",
|
"UPDATE listings SET status='dup', dup_of=?, updated_at=? WHERE id=?",
|
||||||
(canonical[1], now_iso(), db_id))
|
(canonical[1], now_iso(), twin[1]))
|
||||||
dup_count += 1
|
dup_count += 1
|
||||||
if row['url']:
|
if twin[2]['url']:
|
||||||
twin_urls.append(row['url'])
|
# preserve the twin's URL on the canonical row (desc suffix)
|
||||||
if twin_urls:
|
cur = conn.execute('SELECT desc FROM listings WHERE id=?',
|
||||||
# preserve twin URLs on the canonical row (desc suffix), after scoring
|
(canonical[1],))
|
||||||
cur = conn.execute('SELECT desc FROM listings WHERE id=?', (canonical[1],))
|
base_desc = cur.fetchone()['desc'] or ''
|
||||||
base_desc = cur.fetchone()['desc'] or ''
|
suffix = ' [also listed: ' + twin[2]['url'] + ']'
|
||||||
suffix = ' [also listed: ' + ' | '.join(twin_urls) + ']'
|
conn.execute('UPDATE listings SET desc=? WHERE id=?',
|
||||||
conn.execute('UPDATE listings SET desc=? WHERE id=?',
|
(base_desc + suffix, canonical[1]))
|
||||||
(base_desc + suffix, canonical[1]))
|
|
||||||
return dup_count
|
return dup_count
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -219,12 +219,88 @@ def test_gone_full_flow():
|
|||||||
shutil.rmtree(d, ignore_errors=True)
|
shutil.rmtree(d, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedupe_multiunit_pairing():
|
||||||
|
# Cross-province group with 2 ns + 1 nb: min(ns,nb) = 1 pair -> 1 dup;
|
||||||
|
# the surplus ns sibling stays active (no false merge of distinct bikes).
|
||||||
|
d, db_path = fresh_db()
|
||||||
|
try:
|
||||||
|
state_dir = os.path.join(d, 'state')
|
||||||
|
twin = '2025 BMW G 310 R'
|
||||||
|
write_states(state_dir,
|
||||||
|
make_state({'ns1': make_item(twin, price_num=5444, img='http://x/1.jpg'),
|
||||||
|
'ns2': make_item(twin, price_num=5444)}),
|
||||||
|
make_state({'nb1': make_item(twin, price_num=5444)}))
|
||||||
|
res = imp.run_import(state_dir, db_path)
|
||||||
|
assert res['ok'] and res['total'] == 3 and res['dup'] == 1, res
|
||||||
|
conn = db.connect(db_path)
|
||||||
|
db.init_db(conn)
|
||||||
|
nb = conn.execute(
|
||||||
|
"SELECT status, dup_of FROM listings WHERE region='nb'").fetchone()
|
||||||
|
assert nb['status'] == 'dup', nb
|
||||||
|
canon = conn.execute(
|
||||||
|
'SELECT region, status, img_url FROM listings WHERE id=?',
|
||||||
|
(nb['dup_of'],)).fetchone()
|
||||||
|
assert canon['region'] == 'ns' and canon['status'] == 'active'
|
||||||
|
assert canon['img_url'] is not None # best-complete ns was canonical
|
||||||
|
# the surplus ns sibling stays active
|
||||||
|
active_ns = conn.execute(
|
||||||
|
"SELECT COUNT(*) AS n FROM listings WHERE region='ns' AND status='active'"
|
||||||
|
).fetchone()['n']
|
||||||
|
assert active_ns == 2, active_ns
|
||||||
|
# the paired twin's URL is preserved on the canonical's desc
|
||||||
|
desc = conn.execute('SELECT desc FROM listings WHERE id=?',
|
||||||
|
(nb['dup_of'],)).fetchone()['desc'] or ''
|
||||||
|
conn.close()
|
||||||
|
assert 'also listed' in desc
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(d, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedupe_reversible():
|
||||||
|
# A dup can be flipped active by hand; the next import re-derives it.
|
||||||
|
d, db_path = fresh_db()
|
||||||
|
try:
|
||||||
|
state_dir = os.path.join(d, 'state')
|
||||||
|
twin = make_item('2021 Indian Scout Bobber', price_num=6500, km='8000')
|
||||||
|
write_states(state_dir,
|
||||||
|
make_state({'ns1': twin}),
|
||||||
|
make_state({'nb1': dict(twin)}))
|
||||||
|
imp.run_import(state_dir, db_path)
|
||||||
|
conn = db.connect(db_path)
|
||||||
|
db.init_db(conn)
|
||||||
|
nb_id = conn.execute(
|
||||||
|
"SELECT id FROM listings WHERE region='nb'").fetchone()['id']
|
||||||
|
conn.execute("UPDATE listings SET status='active', dup_of=NULL WHERE id=?",
|
||||||
|
(nb_id,))
|
||||||
|
conn.commit()
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*) AS n FROM listings WHERE status='dup'"
|
||||||
|
).fetchone()['n'] == 0
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*) AS n FROM listings WHERE status='active'"
|
||||||
|
).fetchone()['n'] == 2
|
||||||
|
conn.close()
|
||||||
|
imp.run_import(state_dir, db_path) # re-import re-pairs it
|
||||||
|
conn = db.connect(db_path)
|
||||||
|
db.init_db(conn)
|
||||||
|
dup, active = (conn.execute(
|
||||||
|
"SELECT COUNT(*) AS n FROM listings WHERE status='dup'").fetchone()['n'],
|
||||||
|
conn.execute(
|
||||||
|
"SELECT COUNT(*) AS n FROM listings WHERE status='active'").fetchone()['n'])
|
||||||
|
conn.close()
|
||||||
|
assert (dup, active) == (1, 1), (dup, active)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(d, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
TESTS = [
|
TESTS = [
|
||||||
('gate rejections', test_gate_rejections),
|
('gate rejections', test_gate_rejections),
|
||||||
('gate abort leaves DB untouched', test_gate_abort_leaves_db),
|
('gate abort leaves DB untouched', test_gate_abort_leaves_db),
|
||||||
('dedupe cross-province twins', test_dedupe_twins),
|
('dedupe cross-province twins', test_dedupe_twins),
|
||||||
('dedupe false-positive (same province)', test_dedupe_false_positive_same_province),
|
('dedupe false-positive (same province)', test_dedupe_false_positive_same_province),
|
||||||
('dedupe false-positive (price differs)', test_dedupe_false_positive_price_diff),
|
('dedupe false-positive (price differs)', test_dedupe_false_positive_price_diff),
|
||||||
|
('dedupe multi-unit pairing (min(ns,nb))', test_dedupe_multiunit_pairing),
|
||||||
|
('dedupe reversibility (flip + re-import)', test_dedupe_reversible),
|
||||||
('scoring spot-checks', test_scoring),
|
('scoring spot-checks', test_scoring),
|
||||||
('gone after 2 misses (unit)', test_gone_after_two_misses),
|
('gone after 2 misses (unit)', test_gone_after_two_misses),
|
||||||
('gone after 2 misses (full flow)', test_gone_full_flow),
|
('gone after 2 misses (full flow)', test_gone_full_flow),
|
||||||
|
|||||||
Reference in New Issue
Block a user