b1: importer — carry hunt imgs gallery into listings.imgs (JSON, page order); img_url falls back to imgs[0]
This commit is contained in:
@@ -88,6 +88,24 @@ def iter_items(st):
|
|||||||
yield src_id, it
|
yield src_id, it
|
||||||
|
|
||||||
|
|
||||||
|
def imgs_from_item(it):
|
||||||
|
"""Gallery images from a state item: deduplicated, non-empty URL strings
|
||||||
|
in page order, or None. The hunt guarantees imgs[0] equals `img` (the
|
||||||
|
established primary)."""
|
||||||
|
raw = it.get('imgs')
|
||||||
|
if not isinstance(raw, (list, tuple)):
|
||||||
|
return None
|
||||||
|
out, seen = [], set()
|
||||||
|
for u in raw:
|
||||||
|
if not isinstance(u, str):
|
||||||
|
continue
|
||||||
|
u = u.strip()
|
||||||
|
if u and u not in seen:
|
||||||
|
seen.add(u)
|
||||||
|
out.append(u)
|
||||||
|
return out or None
|
||||||
|
|
||||||
|
|
||||||
def normalize(region, src_id, it, now):
|
def normalize(region, src_id, it, now):
|
||||||
"""Normalize one state item into a row dict (strings->ints robustly,
|
"""Normalize one state item into a row dict (strings->ints robustly,
|
||||||
missing fields -> NULL)."""
|
missing fields -> NULL)."""
|
||||||
@@ -102,6 +120,10 @@ def normalize(region, src_id, it, now):
|
|||||||
year = rubric.year_from_title(title)
|
year = rubric.year_from_title(title)
|
||||||
score, family = rubric.score_item(title, it.get('desc') or '', price_num, km, year)
|
score, family = rubric.score_item(title, it.get('desc') or '', price_num, km, year)
|
||||||
extras = rubric.extras_from_text(title, it.get('desc') or '')
|
extras = rubric.extras_from_text(title, it.get('desc') or '')
|
||||||
|
# B1: gallery (page order, entry 0 = the primary). img_url keeps the
|
||||||
|
# established single `img` signal and falls back to imgs[0].
|
||||||
|
imgs = imgs_from_item(it)
|
||||||
|
img = it.get('img') or (imgs[0] if imgs else None)
|
||||||
# seller_type only from a REAL signal in the state item (the hunt image
|
# seller_type only from a REAL signal in the state item (the hunt image
|
||||||
# backfill / enrichment passes) — never inferred from titles.
|
# backfill / enrichment passes) — never inferred from titles.
|
||||||
seller = it.get('seller_type') or it.get('seller')
|
seller = it.get('seller_type') or it.get('seller')
|
||||||
@@ -116,7 +138,8 @@ def normalize(region, src_id, it, now):
|
|||||||
'km': km,
|
'km': km,
|
||||||
'location': (it.get('loc') or it.get('city') or None),
|
'location': (it.get('loc') or it.get('city') or None),
|
||||||
'url': it.get('url') or '',
|
'url': it.get('url') or '',
|
||||||
'img_url': it.get('img') or None,
|
'img_url': img,
|
||||||
|
'imgs': json.dumps(imgs) if imgs else None,
|
||||||
'desc': it.get('desc') or None,
|
'desc': it.get('desc') or None,
|
||||||
'family': family,
|
'family': family,
|
||||||
'score': score,
|
'score': score,
|
||||||
@@ -138,25 +161,25 @@ def upsert(conn, row, now):
|
|||||||
if existing:
|
if existing:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
'''UPDATE listings SET title=?, price_display=?, price_num=?, km=?,
|
'''UPDATE listings SET title=?, price_display=?, price_num=?, km=?,
|
||||||
location=?, url=?, img_url=?, desc=?, family=?, score=?,
|
location=?, url=?, img_url=?, imgs=?, desc=?, family=?, score=?,
|
||||||
year=?, seller_type=?, extras=?,
|
year=?, seller_type=?, extras=?,
|
||||||
status='active', dup_of=NULL, last_seen=?, updated_at=?
|
status='active', dup_of=NULL, last_seen=?, updated_at=?
|
||||||
WHERE id=?''',
|
WHERE id=?''',
|
||||||
(row['title'], row['price_display'], row['price_num'], row['km'],
|
(row['title'], row['price_display'], row['price_num'], row['km'],
|
||||||
row['location'], row['url'], row['img_url'], row['desc'],
|
row['location'], row['url'], row['img_url'], row['imgs'],
|
||||||
row['family'], row['score'], row['year'], row['seller_type'],
|
row['desc'], row['family'], row['score'], row['year'],
|
||||||
row['extras'], now, now, existing['id']))
|
row['seller_type'], row['extras'], now, now, existing['id']))
|
||||||
return existing['id'], False
|
return existing['id'], False
|
||||||
cur = conn.execute(
|
cur = conn.execute(
|
||||||
'''INSERT INTO listings (region, src_id, title, price_display, price_num,
|
'''INSERT INTO listings (region, src_id, title, price_display, price_num,
|
||||||
km, location, url, img_url, desc, family, score, year, seller_type,
|
km, location, url, img_url, imgs, desc, family, score, year,
|
||||||
extras, status, dup_of,
|
seller_type, extras, status, dup_of,
|
||||||
first_seen, last_seen, created_at, updated_at)
|
first_seen, last_seen, created_at, updated_at)
|
||||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
|
||||||
(row['region'], row['src_id'], row['title'], row['price_display'],
|
(row['region'], row['src_id'], row['title'], row['price_display'],
|
||||||
row['price_num'], row['km'], row['location'], row['url'],
|
row['price_num'], row['km'], row['location'], row['url'],
|
||||||
row['img_url'], row['desc'], row['family'], row['score'], row['year'],
|
row['img_url'], row['imgs'], row['desc'], row['family'], row['score'],
|
||||||
row['seller_type'], row['extras'], 'active',
|
row['year'], row['seller_type'], row['extras'], 'active',
|
||||||
None, row['first_seen'], row['last_seen'], now, now))
|
None, row['first_seen'], row['last_seen'], now, now))
|
||||||
return cur.lastrowid, True
|
return cur.lastrowid, True
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user