"""Bike-Tinder local image mirror. Downloads each active listing's card image once into data/img/{id}.webp. .venv/bin/python -m app.imgfetch --db data/bike-tinder.sqlite3 .venv/bin/python -m app.imgfetch --limit 15 Polite by design: ~1 req/s (sleep between fetches), skip-existing, crash-safe (partial bytes land in .part and are atomically renamed), per-item failures never kill the run. Honest end summary: `fetched N · skipped M · failed K`. UA + habits match the hunt scanner (reference/motorcycle-hunt-scan.py) so the CDN sees the same client as the crawl. Stdlib only — no requests dependency. """ import argparse import os import sys import time import urllib.error import urllib.request from . import db # Same UA string the hunt scanner uses. UA = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36') SLEEP = 1.0 TIMEOUT = 20 def is_webp(buf): return (len(buf) >= 12 and buf[:4] == b'RIFF' and buf[8:12] == b'WEBP') def fetch_one(img_url, dst): """Download img_url -> dst. Returns 'fetched' or an error string.""" part = dst + '.part' req = urllib.request.Request( img_url, headers={ 'User-Agent': UA, 'Accept-Language': 'en-CA,en;q=0.9', 'Accept': 'image/avif,image/webp,image/*,*/*;q=0.8', }) try: with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: if getattr(resp, 'status', 200) != 200: return 'HTTP %s' % resp.status buf = resp.read() if not is_webp(buf): return 'not webp (%d bytes, %r)' % (len(buf), buf[:8]) with open(part, 'wb') as fh: fh.write(buf) os.replace(part, dst) # atomic: never a half file under the final name return 'fetched' except urllib.error.HTTPError as e: return 'HTTP %s' % e.code except Exception as e: # noqa: BLE001 — one bad item never kills the run return '%s: %s' % (type(e).__name__, e) finally: if os.path.exists(part): try: os.remove(part) except OSError: pass def run_fetch(db_path, img_dir, limit=None, sleep=SLEEP, quiet=False): conn = db.connect(db_path) db.init_db(conn) rows = conn.execute( "SELECT id, img_url FROM listings" " WHERE status='active' AND img_url IS NOT NULL" " ORDER BY id").fetchall() conn.close() os.makedirs(img_dir, exist_ok=True) fetched = skipped = failed = 0 for row in rows: if limit is not None and fetched >= limit: break dst = os.path.join(img_dir, '%d.webp' % row['id']) if os.path.exists(dst): skipped += 1 continue result = fetch_one(row['img_url'], dst) if result == 'fetched': fetched += 1 else: failed += 1 if not quiet: print(' fail id=%d %s' % (row['id'], result)) time.sleep(sleep) return {'fetched': fetched, 'skipped': skipped, 'failed': failed} def main(argv=None): ap = argparse.ArgumentParser(description='Bike-Tinder image mirror') ap.add_argument('--db', default='data/bike-tinder.sqlite3') ap.add_argument('--img-dir', default='data/img') ap.add_argument('--limit', type=int, default=None, help='stop after this many downloads (small test runs)') ap.add_argument('--sleep', type=float, default=SLEEP, help='seconds between fetches (default %.1f)' % SLEEP) args = ap.parse_args(argv) res = run_fetch(args.db, args.img_dir, limit=args.limit, sleep=args.sleep) print('fetched %d · skipped %d · failed %d' % ( res['fetched'], res['skipped'], res['failed'])) return 0 if __name__ == '__main__': sys.exit(main())