app: P2 migrations (year/seller_type/extras) + importer backfill, guards idempotent
- db.init_db gains guarded ALTER TABLE additions for listings.year / seller_type / extras (schema.sql v1 untouched; PRAGMA-checked idempotent) - rubric: extras_from_text() returns matched EXTRAS labels (table order) - importer: normalize computes year (existing helper), seller_type ONLY from a real state-item signal (dealer/private, else NULL), extras JSON; upsert rewrites the new columns on both paths - verified on live DB: import idempotent, counts stay 640/560/80, year backfilled on 599 rows, extras on 45, seller_type 0 (no state signal)
This commit is contained in:
14
app/db.py
14
app/db.py
@@ -23,7 +23,19 @@ def connect(db_path):
|
|||||||
|
|
||||||
|
|
||||||
def init_db(conn):
|
def init_db(conn):
|
||||||
"""Apply schema.sql (idempotent)."""
|
"""Apply schema.sql (idempotent) + guarded P2 column migrations.
|
||||||
|
|
||||||
|
P2 additions (docs/DATA-MODEL.md) are added with idempotent
|
||||||
|
``ALTER TABLE ... ADD COLUMN`` guarded by a PRAGMA check, so the locked
|
||||||
|
v1 schema.sql stays untouched and existing DBs migrate in place.
|
||||||
|
"""
|
||||||
with open(SCHEMA_PATH, encoding='utf-8') as fh:
|
with open(SCHEMA_PATH, encoding='utf-8') as fh:
|
||||||
conn.executescript(fh.read())
|
conn.executescript(fh.read())
|
||||||
|
cols = {r['name'] for r in conn.execute("PRAGMA table_info('listings')")}
|
||||||
|
if 'year' not in cols:
|
||||||
|
conn.execute('ALTER TABLE listings ADD COLUMN year INTEGER')
|
||||||
|
if 'seller_type' not in cols:
|
||||||
|
conn.execute('ALTER TABLE listings ADD COLUMN seller_type TEXT')
|
||||||
|
if 'extras' not in cols:
|
||||||
|
conn.execute('ALTER TABLE listings ADD COLUMN extras TEXT')
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|||||||
@@ -101,6 +101,12 @@ def normalize(region, src_id, it, now):
|
|||||||
km = None # 0 km is a placeholder, not delivery mileage
|
km = None # 0 km is a placeholder, not delivery mileage
|
||||||
year = rubric.year_from_title(title)
|
year = rubric.year_from_title(title)
|
||||||
score, family = rubric.score_item(title, it.get('desc') or '', price_num, km, year)
|
score, family = rubric.score_item(title, it.get('desc') or '', price_num, km, year)
|
||||||
|
extras = rubric.extras_from_text(title, it.get('desc') or '')
|
||||||
|
# seller_type only from a REAL signal in the state item (the hunt image
|
||||||
|
# backfill / enrichment passes) — never inferred from titles.
|
||||||
|
seller = it.get('seller_type') or it.get('seller')
|
||||||
|
if seller not in ('dealer', 'private'):
|
||||||
|
seller = None
|
||||||
return {
|
return {
|
||||||
'region': region,
|
'region': region,
|
||||||
'src_id': src_id,
|
'src_id': src_id,
|
||||||
@@ -114,6 +120,9 @@ def normalize(region, src_id, it, now):
|
|||||||
'desc': it.get('desc') or None,
|
'desc': it.get('desc') or None,
|
||||||
'family': family,
|
'family': family,
|
||||||
'score': score,
|
'score': score,
|
||||||
|
'year': year,
|
||||||
|
'seller_type': seller,
|
||||||
|
'extras': json.dumps(extras) if extras else None,
|
||||||
'first_seen': it.get('first_seen') or now,
|
'first_seen': it.get('first_seen') or now,
|
||||||
'last_seen': now,
|
'last_seen': now,
|
||||||
}
|
}
|
||||||
@@ -130,20 +139,24 @@ def upsert(conn, row, now):
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
'''UPDATE listings SET title=?, price_display=?, price_num=?, km=?,
|
'''UPDATE listings SET title=?, price_display=?, price_num=?, km=?,
|
||||||
location=?, url=?, img_url=?, desc=?, family=?, score=?,
|
location=?, url=?, img_url=?, desc=?, family=?, score=?,
|
||||||
|
year=?, seller_type=?, extras=?,
|
||||||
status='active', dup_of=NULL, last_seen=?, updated_at=?
|
status='active', dup_of=NULL, last_seen=?, updated_at=?
|
||||||
WHERE id=?''',
|
WHERE id=?''',
|
||||||
(row['title'], row['price_display'], row['price_num'], row['km'],
|
(row['title'], row['price_display'], row['price_num'], row['km'],
|
||||||
row['location'], row['url'], row['img_url'], row['desc'],
|
row['location'], row['url'], row['img_url'], row['desc'],
|
||||||
row['family'], row['score'], now, now, existing['id']))
|
row['family'], row['score'], row['year'], row['seller_type'],
|
||||||
|
row['extras'], now, now, existing['id']))
|
||||||
return existing['id'], False
|
return existing['id'], False
|
||||||
cur = conn.execute(
|
cur = conn.execute(
|
||||||
'''INSERT INTO listings (region, src_id, title, price_display, price_num,
|
'''INSERT INTO listings (region, src_id, title, price_display, price_num,
|
||||||
km, location, url, img_url, desc, family, score, status, dup_of,
|
km, location, url, img_url, desc, family, score, year, seller_type,
|
||||||
|
extras, status, dup_of,
|
||||||
first_seen, last_seen, created_at, updated_at)
|
first_seen, last_seen, created_at, updated_at)
|
||||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
|
||||||
(row['region'], row['src_id'], row['title'], row['price_display'],
|
(row['region'], row['src_id'], row['title'], row['price_display'],
|
||||||
row['price_num'], row['km'], row['location'], row['url'],
|
row['price_num'], row['km'], row['location'], row['url'],
|
||||||
row['img_url'], row['desc'], row['family'], row['score'], 'active',
|
row['img_url'], row['desc'], row['family'], row['score'], row['year'],
|
||||||
|
row['seller_type'], row['extras'], 'active',
|
||||||
None, row['first_seen'], row['last_seen'], now, now))
|
None, row['first_seen'], row['last_seen'], now, now))
|
||||||
return cur.lastrowid, True
|
return cur.lastrowid, True
|
||||||
|
|
||||||
|
|||||||
@@ -146,6 +146,16 @@ def score_item(title, desc, price_num, km, year=None):
|
|||||||
return s, label
|
return s, label
|
||||||
|
|
||||||
|
|
||||||
|
def extras_from_text(title, desc):
|
||||||
|
"""Return the labels of every EXTRAS pattern that matches title+desc.
|
||||||
|
|
||||||
|
One entry per label (labels are unique in the EXTRAS table); order is
|
||||||
|
the table order. Stored on listings.extras at import (P2).
|
||||||
|
"""
|
||||||
|
text = (title + ' ' + desc).lower()
|
||||||
|
return [label for pat, _pts, label in EXTRAS if re.search(pat, text)]
|
||||||
|
|
||||||
|
|
||||||
def year_from_title(title):
|
def year_from_title(title):
|
||||||
"""Extract a model year from the title, or None."""
|
"""Extract a model year from the title, or None."""
|
||||||
m = re.search(r'\b(19\d\d|20[0-2]\d)\b', title or '')
|
m = re.search(r'\b(19\d\d|20[0-2]\d)\b', title or '')
|
||||||
|
|||||||
Reference in New Issue
Block a user