Bike-Tinder/app/imgfetch.py

"""Bike-Tinder local image mirror.

Downloads each active listing's photos once:
  - gallery listings (B1 §11.1): imgs[n] -> data/img/{id}_{n}.webp for the
    extra photos; the primary (n=0) lands at the legacy data/img/{id}.webp
    so the v1 single-image chain is untouched;
  - legacy listings: img_url -> data/img/{id}.webp (as before).

  .venv/bin/python -m app.imgfetch --db data/bike-tinder.sqlite3
  .venv/bin/python -m app.imgfetch --limit 15

Polite by design: ~1 req/s (sleep between fetches), skip-existing, crash-safe
(partial bytes land in .part and are atomically renamed), per-item failures
never kill the run. Per-file CLI progress; honest end summary:
`fetched N · skipped M · failed K`.

UA + habits match the hunt scanner (reference/motorcycle-hunt-scan.py) so the
CDN sees the same client as the crawl. Stdlib only — no requests dependency.
"""
import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.request

from . import db

# Same UA string the hunt scanner uses.
UA = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
      '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')

SLEEP = 1.0
TIMEOUT = 20


def is_webp(buf):
    return (len(buf) >= 12 and buf[:4] == b'RIFF' and buf[8:12] == b'WEBP')


def fetch_one(img_url, dst):
    """Download img_url -> dst. Returns 'fetched' or an error string."""
    part = dst + '.part'
    req = urllib.request.Request(
        img_url, headers={
            'User-Agent': UA,
            'Accept-Language': 'en-CA,en;q=0.9',
            'Accept': 'image/avif,image/webp,image/*,*/*;q=0.8',
        })
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            status = (getattr(resp, 'status', None)
                      or getattr(resp, 'code', None) or 200)
            if status != 200:
                return 'HTTP %s' % status
            buf = resp.read()
        if not is_webp(buf):
            return 'not webp (%d bytes, %r)' % (len(buf), buf[:8])
        with open(part, 'wb') as fh:
            fh.write(buf)
        os.replace(part, dst)   # atomic: never a half file under the final name
        return 'fetched'
    except urllib.error.HTTPError as e:
        return 'HTTP %s' % e.code
    except Exception as e:      # noqa: BLE001 — one bad item never kills the run
        return '%s: %s' % (type(e).__name__, e)
    finally:
        if os.path.exists(part):
            try:
                os.remove(part)
            except OSError:
                pass


def parse_imgs(raw):
    """listings.imgs TEXT -> list of non-empty URL strings ([] when unparsable)."""
    if not raw:
        return []
    try:
        v = json.loads(raw)
    except ValueError:
        return []
    if not isinstance(v, list):
        return []
    return [u.strip() for u in v if isinstance(u, str) and u.strip()]


def build_work(rows, img_dir):
    """Flatten listings rows into (listing_id, n_or_None, url, dst) downloads.

    Gallery rows fetch every imgs[n]: n=0 -> the legacy {id}.webp (the primary
    — one canonical file, no duplicate fetch), n>=1 -> {id}_{n}.webp. Legacy
    rows (no gallery) keep the img_url -> {id}.webp chain. Work is deduped by
    destination path.
    """
    work = []
    for r in rows:
        g = parse_imgs(r['imgs'])
        if g:
            for n, url in enumerate(g):
                dst = os.path.join(img_dir, ('%d.webp' % r['id'] if n == 0
                                  else '%d_%d.webp' % (r['id'], n)))
                work.append((r['id'], n, url, dst))
        elif r['img_url']:
            work.append((r['id'], None, r['img_url'],
                         os.path.join(img_dir, '%d.webp' % r['id'])))
    seen, deduped = set(), []
    for item in work:
        if item[3] not in seen:
            seen.add(item[3])
            deduped.append(item)
    return deduped


def run_fetch(db_path, img_dir, limit=None, sleep=SLEEP, quiet=False):
    conn = db.connect(db_path)
    db.init_db(conn)
    rows = conn.execute(
        "SELECT id, img_url, imgs FROM listings"
        " WHERE status='active' ORDER BY id").fetchall()
    conn.close()

    os.makedirs(img_dir, exist_ok=True)
    work = build_work(rows, img_dir)
    total = len(work)
    fetched = skipped = failed = 0
    for idx, (lid, n, url, dst) in enumerate(work, 1):
        if limit is not None and fetched >= limit:
            break
        if os.path.exists(dst):
            skipped += 1
            continue
        result = fetch_one(url, dst)
        if result == 'fetched':
            fetched += 1
            if not quiet:
                print('  [%d/%d] id=%d n=%s ok' % (idx, total, lid, n))
        else:
            failed += 1
            if not quiet:
                print('  fail  id=%d n=%s %s' % (lid, n, result))
        time.sleep(sleep)
    return {'fetched': fetched, 'skipped': skipped, 'failed': failed,
            'total': total}


def main(argv=None):
    ap = argparse.ArgumentParser(description='Bike-Tinder image mirror')
    ap.add_argument('--db', default='data/bike-tinder.sqlite3')
    ap.add_argument('--img-dir', default='data/img')
    ap.add_argument('--limit', type=int, default=None,
                    help='stop after this many downloads (small test runs)')
    ap.add_argument('--sleep', type=float, default=SLEEP,
                    help='seconds between fetches (default %.1f)' % SLEEP)
    args = ap.parse_args(argv)

    res = run_fetch(args.db, args.img_dir, limit=args.limit, sleep=args.sleep)
    print('fetched %d · skipped %d · failed %d · total %d' % (
        res['fetched'], res['skipped'], res['failed'], res['total']))
    return 0


if __name__ == '__main__':
    sys.exit(main())