app: P2 migrations (year/seller_type/extras) + importer backfill, guards idempotent

- db.init_db gains guarded ALTER TABLE additions for listings.year /
  seller_type / extras (schema.sql v1 untouched; PRAGMA-checked idempotent)
- rubric: extras_from_text() returns matched EXTRAS labels (table order)
- importer: normalize computes year (existing helper), seller_type ONLY
  from a real state-item signal (dealer/private, else NULL), extras JSON;
  upsert rewrites the new columns on both paths
- verified on live DB: import idempotent, counts stay 640/560/80, year
  backfilled on 599 rows, extras on 45, seller_type 0 (no state signal)
This commit is contained in:
Torres
2026-11-10 21:13:40 -04:00
parent 9bc6948861
commit 02a6b6df7e
3 changed files with 40 additions and 5 deletions

View File

@@ -101,6 +101,12 @@ def normalize(region, src_id, it, now):
km = None # 0 km is a placeholder, not delivery mileage
year = rubric.year_from_title(title)
score, family = rubric.score_item(title, it.get('desc') or '', price_num, km, year)
extras = rubric.extras_from_text(title, it.get('desc') or '')
# seller_type only from a REAL signal in the state item (the hunt image
# backfill / enrichment passes) — never inferred from titles.
seller = it.get('seller_type') or it.get('seller')
if seller not in ('dealer', 'private'):
seller = None
return {
'region': region,
'src_id': src_id,
@@ -114,6 +120,9 @@ def normalize(region, src_id, it, now):
'desc': it.get('desc') or None,
'family': family,
'score': score,
'year': year,
'seller_type': seller,
'extras': json.dumps(extras) if extras else None,
'first_seen': it.get('first_seen') or now,
'last_seen': now,
}
@@ -130,20 +139,24 @@ def upsert(conn, row, now):
conn.execute(
'''UPDATE listings SET title=?, price_display=?, price_num=?, km=?,
location=?, url=?, img_url=?, desc=?, family=?, score=?,
year=?, seller_type=?, extras=?,
status='active', dup_of=NULL, last_seen=?, updated_at=?
WHERE id=?''',
(row['title'], row['price_display'], row['price_num'], row['km'],
row['location'], row['url'], row['img_url'], row['desc'],
row['family'], row['score'], now, now, existing['id']))
row['family'], row['score'], row['year'], row['seller_type'],
row['extras'], now, now, existing['id']))
return existing['id'], False
cur = conn.execute(
'''INSERT INTO listings (region, src_id, title, price_display, price_num,
km, location, url, img_url, desc, family, score, status, dup_of,
km, location, url, img_url, desc, family, score, year, seller_type,
extras, status, dup_of,
first_seen, last_seen, created_at, updated_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
(row['region'], row['src_id'], row['title'], row['price_display'],
row['price_num'], row['km'], row['location'], row['url'],
row['img_url'], row['desc'], row['family'], row['score'], 'active',
row['img_url'], row['desc'], row['family'], row['score'], row['year'],
row['seller_type'], row['extras'], 'active',
None, row['first_seen'], row['last_seen'], now, now))
return cur.lastrowid, True