b1: imgfetch — gallery mirroring: imgs[n] -> {id}_{n}.webp (primary n=0 keeps legacy {id}.webp, dedup by dst), per-file CLI progress, resumable/skip-existing as P2

app/imgfetch.py

@@ -1,18 +1,24 @@
 """Bike-Tinder local image mirror.
 
-Downloads each active listing's card image once into data/img/{id}.webp.
+Downloads each active listing's photos once:
+  - gallery listings (B1 §11.1): imgs[n] -> data/img/{id}_{n}.webp for the
+    extra photos; the primary (n=0) lands at the legacy data/img/{id}.webp
+    so the v1 single-image chain is untouched;
+  - legacy listings: img_url -> data/img/{id}.webp (as before).
 
   .venv/bin/python -m app.imgfetch --db data/bike-tinder.sqlite3
   .venv/bin/python -m app.imgfetch --limit 15
 
 Polite by design: ~1 req/s (sleep between fetches), skip-existing, crash-safe
 (partial bytes land in .part and are atomically renamed), per-item failures
-never kill the run. Honest end summary: `fetched N · skipped M · failed K`.
+never kill the run. Per-file CLI progress; honest end summary:
+`fetched N · skipped M · failed K`.
 
 UA + habits match the hunt scanner (reference/motorcycle-hunt-scan.py) so the
 CDN sees the same client as the crawl. Stdlib only — no requests dependency.
 """
 import argparse
+import json
 import os
 import sys
 import time
@@ -44,8 +50,10 @@ def fetch_one(img_url, dst):
         })
     try:
         with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
-            if getattr(resp, 'status', 200) != 200:
-                return 'HTTP %s' % resp.status
+            status = (getattr(resp, 'status', None)
+                      or getattr(resp, 'code', None) or 200)
+            if status != 200:
+                return 'HTTP %s' % status
             buf = resp.read()
         if not is_webp(buf):
             return 'not webp (%d bytes, %r)' % (len(buf), buf[:8])
@@ -65,33 +73,76 @@ def fetch_one(img_url, dst):
                 pass
 
 
+def parse_imgs(raw):
+    """listings.imgs TEXT -> list of non-empty URL strings ([] when unparsable)."""
+    if not raw:
+        return []
+    try:
+        v = json.loads(raw)
+    except ValueError:
+        return []
+    if not isinstance(v, list):
+        return []
+    return [u.strip() for u in v if isinstance(u, str) and u.strip()]
+
+
+def build_work(rows, img_dir):
+    """Flatten listings rows into (listing_id, n_or_None, url, dst) downloads.
+
+    Gallery rows fetch every imgs[n]: n=0 -> the legacy {id}.webp (the primary
+    — one canonical file, no duplicate fetch), n>=1 -> {id}_{n}.webp. Legacy
+    rows (no gallery) keep the img_url -> {id}.webp chain. Work is deduped by
+    destination path.
+    """
+    work = []
+    for r in rows:
+        g = parse_imgs(r['imgs'])
+        if g:
+            for n, url in enumerate(g):
+                dst = os.path.join(img_dir, ('%d.webp' % r['id'] if n == 0
+                                  else '%d_%d.webp' % (r['id'], n)))
+                work.append((r['id'], n, url, dst))
+        elif r['img_url']:
+            work.append((r['id'], None, r['img_url'],
+                         os.path.join(img_dir, '%d.webp' % r['id'])))
+    seen, deduped = set(), []
+    for item in work:
+        if item[3] not in seen:
+            seen.add(item[3])
+            deduped.append(item)
+    return deduped
+
+
 def run_fetch(db_path, img_dir, limit=None, sleep=SLEEP, quiet=False):
     conn = db.connect(db_path)
     db.init_db(conn)
     rows = conn.execute(
-        "SELECT id, img_url FROM listings"
-        " WHERE status='active' AND img_url IS NOT NULL"
-        " ORDER BY id").fetchall()
+        "SELECT id, img_url, imgs FROM listings"
+        " WHERE status='active' ORDER BY id").fetchall()
     conn.close()
 
     os.makedirs(img_dir, exist_ok=True)
+    work = build_work(rows, img_dir)
+    total = len(work)
     fetched = skipped = failed = 0
-    for row in rows:
+    for idx, (lid, n, url, dst) in enumerate(work, 1):
         if limit is not None and fetched >= limit:
             break
-        dst = os.path.join(img_dir, '%d.webp' % row['id'])
         if os.path.exists(dst):
             skipped += 1
             continue
-        result = fetch_one(row['img_url'], dst)
+        result = fetch_one(url, dst)
         if result == 'fetched':
             fetched += 1
+            if not quiet:
+                print('  [%d/%d] id=%d n=%s ok' % (idx, total, lid, n))
         else:
             failed += 1
             if not quiet:
-                print('  fail  id=%d %s' % (row['id'], result))
+                print('  fail  id=%d n=%s %s' % (lid, n, result))
         time.sleep(sleep)
-    return {'fetched': fetched, 'skipped': skipped, 'failed': failed}
+    return {'fetched': fetched, 'skipped': skipped, 'failed': failed,
+            'total': total}
 
 
 def main(argv=None):
@@ -105,8 +156,8 @@ def main(argv=None):
     args = ap.parse_args(argv)
 
     res = run_fetch(args.db, args.img_dir, limit=args.limit, sleep=args.sleep)
-    print('fetched %d · skipped %d · failed %d' % (
-        res['fetched'], res['skipped'], res['failed']))
+    print('fetched %d · skipped %d · failed %d · total %d' % (
+        res['fetched'], res['skipped'], res['failed'], res['total']))
     return 0