importer: dedupe refinement — min(ns,nb) completeness pairing (stretch I)
- cross-province groups now pair ns<->nb by completeness (img > km > older) up to min per region instead of collapsing the whole group to one row; surplus siblings (multi-unit dealer ads) stay active — no false merges - verified on live DB: dup 80 -> 77, active 560 -> 563, total 640; the 3 freed rows are the BMW G 310 R dealer siblings (450/451/452); deck top order unchanged (Indian Scout 91 x3 then Vulcan 900 73); re-import idempotent (77 stable) - selfcheck +2: multi-unit pairing semantics, reversibility (flip to active + re-import re-derives the pairing) — 16/16 green
This commit is contained in:
Torres
@@ -162,10 +162,18 @@ def upsert(conn, row, now):
|
||||
|
||||
|
||||
def dedupe(conn, present):
|
||||
"""Collapse cross-province twins. `present` maps (region, src_id) -> db_id.
|
||||
Groups by (normalized_title, price_num); a group spanning both provinces
|
||||
keeps the most-complete row canonical (img > km > older) and marks the rest
|
||||
'dup'. The twin's URL is preserved on the canonical row's desc."""
|
||||
"""Collapse cross-province twins with min(ns,nb) completeness pairing.
|
||||
|
||||
`present` maps (region, src_id) -> db_id. Groups by
|
||||
(normalized_title, price_num). For a group spanning both provinces the
|
||||
two region lists are sorted best-first (img present > km present > older)
|
||||
and paired one-to-one up to min(ns_count, nb_count): each pair keeps the
|
||||
more-complete row canonical and marks the twin 'dup'. Surplus siblings
|
||||
(multi-unit dealer ads) stay active instead of being false-merged.
|
||||
The twin's URL is preserved on the canonical row's desc. (DATA-MODEL P2
|
||||
addendum: pairing replaces collapse-to-one — dup ≈ 77 vs 80 on today's
|
||||
snapshot.) Reversible: dubs carry dup_of; flipping status back to
|
||||
'active' and re-importing re-derives the pairing."""
|
||||
groups = {}
|
||||
for (region, src_id), db_id in present.items():
|
||||
row = conn.execute(
|
||||
@@ -174,31 +182,32 @@ def dedupe(conn, present):
|
||||
key = (norm_title(row['title']), row['price_num'])
|
||||
groups.setdefault(key, []).append((region, db_id, row))
|
||||
|
||||
def completeness(m):
|
||||
"""Best first: img present, then km present, then older row."""
|
||||
return (0 if m[2]['img_url'] else 1,
|
||||
0 if m[2]['km'] is not None else 1,
|
||||
m[2]['first_seen'] or '')
|
||||
|
||||
dup_count = 0
|
||||
for key, members in groups.items():
|
||||
if len({m[0] for m in members}) < 2:
|
||||
continue # not cross-province
|
||||
# most complete first: img present, then km present, then older row
|
||||
members.sort(key=lambda m: (
|
||||
0 if m[2]['img_url'] else 1,
|
||||
0 if m[2]['km'] is not None else 1,
|
||||
m[2]['first_seen'] or ''))
|
||||
canonical = members[0]
|
||||
twin_urls = []
|
||||
for region, db_id, row in members[1:]:
|
||||
ns = sorted((m for m in members if m[0] == 'ns'), key=completeness)
|
||||
nb = sorted((m for m in members if m[0] == 'nb'), key=completeness)
|
||||
for a, b in zip(ns, nb):
|
||||
canonical, twin = (a, b) if completeness(a) <= completeness(b) else (b, a)
|
||||
conn.execute(
|
||||
"UPDATE listings SET status='dup', dup_of=?, updated_at=? WHERE id=?",
|
||||
(canonical[1], now_iso(), db_id))
|
||||
(canonical[1], now_iso(), twin[1]))
|
||||
dup_count += 1
|
||||
if row['url']:
|
||||
twin_urls.append(row['url'])
|
||||
if twin_urls:
|
||||
# preserve twin URLs on the canonical row (desc suffix), after scoring
|
||||
cur = conn.execute('SELECT desc FROM listings WHERE id=?', (canonical[1],))
|
||||
base_desc = cur.fetchone()['desc'] or ''
|
||||
suffix = ' [also listed: ' + ' | '.join(twin_urls) + ']'
|
||||
conn.execute('UPDATE listings SET desc=? WHERE id=?',
|
||||
(base_desc + suffix, canonical[1]))
|
||||
if twin[2]['url']:
|
||||
# preserve the twin's URL on the canonical row (desc suffix)
|
||||
cur = conn.execute('SELECT desc FROM listings WHERE id=?',
|
||||
(canonical[1],))
|
||||
base_desc = cur.fetchone()['desc'] or ''
|
||||
suffix = ' [also listed: ' + twin[2]['url'] + ']'
|
||||
conn.execute('UPDATE listings SET desc=? WHERE id=?',
|
||||
(base_desc + suffix, canonical[1]))
|
||||
return dup_count
|
||||
|
||||
|
||||
|
||||
@@ -219,12 +219,88 @@ def test_gone_full_flow():
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
def test_dedupe_multiunit_pairing():
|
||||
# Cross-province group with 2 ns + 1 nb: min(ns,nb) = 1 pair -> 1 dup;
|
||||
# the surplus ns sibling stays active (no false merge of distinct bikes).
|
||||
d, db_path = fresh_db()
|
||||
try:
|
||||
state_dir = os.path.join(d, 'state')
|
||||
twin = '2025 BMW G 310 R'
|
||||
write_states(state_dir,
|
||||
make_state({'ns1': make_item(twin, price_num=5444, img='http://x/1.jpg'),
|
||||
'ns2': make_item(twin, price_num=5444)}),
|
||||
make_state({'nb1': make_item(twin, price_num=5444)}))
|
||||
res = imp.run_import(state_dir, db_path)
|
||||
assert res['ok'] and res['total'] == 3 and res['dup'] == 1, res
|
||||
conn = db.connect(db_path)
|
||||
db.init_db(conn)
|
||||
nb = conn.execute(
|
||||
"SELECT status, dup_of FROM listings WHERE region='nb'").fetchone()
|
||||
assert nb['status'] == 'dup', nb
|
||||
canon = conn.execute(
|
||||
'SELECT region, status, img_url FROM listings WHERE id=?',
|
||||
(nb['dup_of'],)).fetchone()
|
||||
assert canon['region'] == 'ns' and canon['status'] == 'active'
|
||||
assert canon['img_url'] is not None # best-complete ns was canonical
|
||||
# the surplus ns sibling stays active
|
||||
active_ns = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE region='ns' AND status='active'"
|
||||
).fetchone()['n']
|
||||
assert active_ns == 2, active_ns
|
||||
# the paired twin's URL is preserved on the canonical's desc
|
||||
desc = conn.execute('SELECT desc FROM listings WHERE id=?',
|
||||
(nb['dup_of'],)).fetchone()['desc'] or ''
|
||||
conn.close()
|
||||
assert 'also listed' in desc
|
||||
finally:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
def test_dedupe_reversible():
|
||||
# A dup can be flipped active by hand; the next import re-derives it.
|
||||
d, db_path = fresh_db()
|
||||
try:
|
||||
state_dir = os.path.join(d, 'state')
|
||||
twin = make_item('2021 Indian Scout Bobber', price_num=6500, km='8000')
|
||||
write_states(state_dir,
|
||||
make_state({'ns1': twin}),
|
||||
make_state({'nb1': dict(twin)}))
|
||||
imp.run_import(state_dir, db_path)
|
||||
conn = db.connect(db_path)
|
||||
db.init_db(conn)
|
||||
nb_id = conn.execute(
|
||||
"SELECT id FROM listings WHERE region='nb'").fetchone()['id']
|
||||
conn.execute("UPDATE listings SET status='active', dup_of=NULL WHERE id=?",
|
||||
(nb_id,))
|
||||
conn.commit()
|
||||
assert conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE status='dup'"
|
||||
).fetchone()['n'] == 0
|
||||
assert conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE status='active'"
|
||||
).fetchone()['n'] == 2
|
||||
conn.close()
|
||||
imp.run_import(state_dir, db_path) # re-import re-pairs it
|
||||
conn = db.connect(db_path)
|
||||
db.init_db(conn)
|
||||
dup, active = (conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE status='dup'").fetchone()['n'],
|
||||
conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM listings WHERE status='active'").fetchone()['n'])
|
||||
conn.close()
|
||||
assert (dup, active) == (1, 1), (dup, active)
|
||||
finally:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
TESTS = [
|
||||
('gate rejections', test_gate_rejections),
|
||||
('gate abort leaves DB untouched', test_gate_abort_leaves_db),
|
||||
('dedupe cross-province twins', test_dedupe_twins),
|
||||
('dedupe false-positive (same province)', test_dedupe_false_positive_same_province),
|
||||
('dedupe false-positive (price differs)', test_dedupe_false_positive_price_diff),
|
||||
('dedupe multi-unit pairing (min(ns,nb))', test_dedupe_multiunit_pairing),
|
||||
('dedupe reversibility (flip + re-import)', test_dedupe_reversible),
|
||||
('scoring spot-checks', test_scoring),
|
||||
('gone after 2 misses (unit)', test_gone_after_two_misses),
|
||||
('gone after 2 misses (full flow)', test_gone_full_flow),
|
||||
|
||||
Reference in New Issue
Block a user