From 049b4fd28008021a20e2a083d45a3a6fbe9a1423 Mon Sep 17 00:00:00 2001 From: Torres Date: Wed, 11 Nov 2026 00:04:56 -0400 Subject: [PATCH] =?UTF-8?q?b1:=20imgfetch=20=E2=80=94=20gallery=20mirrorin?= =?UTF-8?q?g:=20imgs[n]=20->=20{id}=5F{n}.webp=20(primary=20n=3D0=20keeps?= =?UTF-8?q?=20legacy=20{id}.webp,=20dedup=20by=20dst),=20per-file=20CLI=20?= =?UTF-8?q?progress,=20resumable/skip-existing=20as=20P2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/imgfetch.py | 79 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/app/imgfetch.py b/app/imgfetch.py index 44f9908..141f720 100644 --- a/app/imgfetch.py +++ b/app/imgfetch.py @@ -1,18 +1,24 @@ """Bike-Tinder local image mirror. -Downloads each active listing's card image once into data/img/{id}.webp. +Downloads each active listing's photos once: + - gallery listings (B1 §11.1): imgs[n] -> data/img/{id}_{n}.webp for the + extra photos; the primary (n=0) lands at the legacy data/img/{id}.webp + so the v1 single-image chain is untouched; + - legacy listings: img_url -> data/img/{id}.webp (as before). .venv/bin/python -m app.imgfetch --db data/bike-tinder.sqlite3 .venv/bin/python -m app.imgfetch --limit 15 Polite by design: ~1 req/s (sleep between fetches), skip-existing, crash-safe (partial bytes land in .part and are atomically renamed), per-item failures -never kill the run. Honest end summary: `fetched N · skipped M · failed K`. +never kill the run. Per-file CLI progress; honest end summary: +`fetched N · skipped M · failed K`. UA + habits match the hunt scanner (reference/motorcycle-hunt-scan.py) so the CDN sees the same client as the crawl. Stdlib only — no requests dependency. """ import argparse +import json import os import sys import time @@ -44,8 +50,10 @@ def fetch_one(img_url, dst): }) try: with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: - if getattr(resp, 'status', 200) != 200: - return 'HTTP %s' % resp.status + status = (getattr(resp, 'status', None) + or getattr(resp, 'code', None) or 200) + if status != 200: + return 'HTTP %s' % status buf = resp.read() if not is_webp(buf): return 'not webp (%d bytes, %r)' % (len(buf), buf[:8]) @@ -65,33 +73,76 @@ def fetch_one(img_url, dst): pass +def parse_imgs(raw): + """listings.imgs TEXT -> list of non-empty URL strings ([] when unparsable).""" + if not raw: + return [] + try: + v = json.loads(raw) + except ValueError: + return [] + if not isinstance(v, list): + return [] + return [u.strip() for u in v if isinstance(u, str) and u.strip()] + + +def build_work(rows, img_dir): + """Flatten listings rows into (listing_id, n_or_None, url, dst) downloads. + + Gallery rows fetch every imgs[n]: n=0 -> the legacy {id}.webp (the primary + — one canonical file, no duplicate fetch), n>=1 -> {id}_{n}.webp. Legacy + rows (no gallery) keep the img_url -> {id}.webp chain. Work is deduped by + destination path. + """ + work = [] + for r in rows: + g = parse_imgs(r['imgs']) + if g: + for n, url in enumerate(g): + dst = os.path.join(img_dir, ('%d.webp' % r['id'] if n == 0 + else '%d_%d.webp' % (r['id'], n))) + work.append((r['id'], n, url, dst)) + elif r['img_url']: + work.append((r['id'], None, r['img_url'], + os.path.join(img_dir, '%d.webp' % r['id']))) + seen, deduped = set(), [] + for item in work: + if item[3] not in seen: + seen.add(item[3]) + deduped.append(item) + return deduped + + def run_fetch(db_path, img_dir, limit=None, sleep=SLEEP, quiet=False): conn = db.connect(db_path) db.init_db(conn) rows = conn.execute( - "SELECT id, img_url FROM listings" - " WHERE status='active' AND img_url IS NOT NULL" - " ORDER BY id").fetchall() + "SELECT id, img_url, imgs FROM listings" + " WHERE status='active' ORDER BY id").fetchall() conn.close() os.makedirs(img_dir, exist_ok=True) + work = build_work(rows, img_dir) + total = len(work) fetched = skipped = failed = 0 - for row in rows: + for idx, (lid, n, url, dst) in enumerate(work, 1): if limit is not None and fetched >= limit: break - dst = os.path.join(img_dir, '%d.webp' % row['id']) if os.path.exists(dst): skipped += 1 continue - result = fetch_one(row['img_url'], dst) + result = fetch_one(url, dst) if result == 'fetched': fetched += 1 + if not quiet: + print(' [%d/%d] id=%d n=%s ok' % (idx, total, lid, n)) else: failed += 1 if not quiet: - print(' fail id=%d %s' % (row['id'], result)) + print(' fail id=%d n=%s %s' % (lid, n, result)) time.sleep(sleep) - return {'fetched': fetched, 'skipped': skipped, 'failed': failed} + return {'fetched': fetched, 'skipped': skipped, 'failed': failed, + 'total': total} def main(argv=None): @@ -105,8 +156,8 @@ def main(argv=None): args = ap.parse_args(argv) res = run_fetch(args.db, args.img_dir, limit=args.limit, sleep=args.sleep) - print('fetched %d · skipped %d · failed %d' % ( - res['fetched'], res['skipped'], res['failed'])) + print('fetched %d · skipped %d · failed %d · total %d' % ( + res['fetched'], res['skipped'], res['failed'], res['total'])) return 0