diff --git a/app/importer.py b/app/importer.py index 3d5446d..37d8939 100644 --- a/app/importer.py +++ b/app/importer.py @@ -88,6 +88,24 @@ def iter_items(st): yield src_id, it +def imgs_from_item(it): + """Gallery images from a state item: deduplicated, non-empty URL strings + in page order, or None. The hunt guarantees imgs[0] equals `img` (the + established primary).""" + raw = it.get('imgs') + if not isinstance(raw, (list, tuple)): + return None + out, seen = [], set() + for u in raw: + if not isinstance(u, str): + continue + u = u.strip() + if u and u not in seen: + seen.add(u) + out.append(u) + return out or None + + def normalize(region, src_id, it, now): """Normalize one state item into a row dict (strings->ints robustly, missing fields -> NULL).""" @@ -102,6 +120,10 @@ def normalize(region, src_id, it, now): year = rubric.year_from_title(title) score, family = rubric.score_item(title, it.get('desc') or '', price_num, km, year) extras = rubric.extras_from_text(title, it.get('desc') or '') + # B1: gallery (page order, entry 0 = the primary). img_url keeps the + # established single `img` signal and falls back to imgs[0]. + imgs = imgs_from_item(it) + img = it.get('img') or (imgs[0] if imgs else None) # seller_type only from a REAL signal in the state item (the hunt image # backfill / enrichment passes) — never inferred from titles. seller = it.get('seller_type') or it.get('seller') @@ -116,7 +138,8 @@ def normalize(region, src_id, it, now): 'km': km, 'location': (it.get('loc') or it.get('city') or None), 'url': it.get('url') or '', - 'img_url': it.get('img') or None, + 'img_url': img, + 'imgs': json.dumps(imgs) if imgs else None, 'desc': it.get('desc') or None, 'family': family, 'score': score, @@ -138,25 +161,25 @@ def upsert(conn, row, now): if existing: conn.execute( '''UPDATE listings SET title=?, price_display=?, price_num=?, km=?, - location=?, url=?, img_url=?, desc=?, family=?, score=?, + location=?, url=?, img_url=?, imgs=?, desc=?, family=?, score=?, year=?, seller_type=?, extras=?, status='active', dup_of=NULL, last_seen=?, updated_at=? WHERE id=?''', (row['title'], row['price_display'], row['price_num'], row['km'], - row['location'], row['url'], row['img_url'], row['desc'], - row['family'], row['score'], row['year'], row['seller_type'], - row['extras'], now, now, existing['id'])) + row['location'], row['url'], row['img_url'], row['imgs'], + row['desc'], row['family'], row['score'], row['year'], + row['seller_type'], row['extras'], now, now, existing['id'])) return existing['id'], False cur = conn.execute( '''INSERT INTO listings (region, src_id, title, price_display, price_num, - km, location, url, img_url, desc, family, score, year, seller_type, - extras, status, dup_of, + km, location, url, img_url, imgs, desc, family, score, year, + seller_type, extras, status, dup_of, first_seen, last_seen, created_at, updated_at) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', (row['region'], row['src_id'], row['title'], row['price_display'], row['price_num'], row['km'], row['location'], row['url'], - row['img_url'], row['desc'], row['family'], row['score'], row['year'], - row['seller_type'], row['extras'], 'active', + row['img_url'], row['imgs'], row['desc'], row['family'], row['score'], + row['year'], row['seller_type'], row['extras'], 'active', None, row['first_seen'], row['last_seen'], now, now)) return cur.lastrowid, True