b1: imgfetch — gallery mirroring: imgs[n] -> {id}_{n}.webp (primary n=0 keeps legacy {id}.webp, dedup by dst), per-file CLI progress, resumable/skip-existing as P2

This commit is contained in:
Torres
2026-11-11 00:04:56 -04:00
parent 604b983a51
commit 049b4fd280

View File

@@ -1,18 +1,24 @@
"""Bike-Tinder local image mirror. """Bike-Tinder local image mirror.
Downloads each active listing's card image once into data/img/{id}.webp. Downloads each active listing's photos once:
- gallery listings (B1 §11.1): imgs[n] -> data/img/{id}_{n}.webp for the
extra photos; the primary (n=0) lands at the legacy data/img/{id}.webp
so the v1 single-image chain is untouched;
- legacy listings: img_url -> data/img/{id}.webp (as before).
.venv/bin/python -m app.imgfetch --db data/bike-tinder.sqlite3 .venv/bin/python -m app.imgfetch --db data/bike-tinder.sqlite3
.venv/bin/python -m app.imgfetch --limit 15 .venv/bin/python -m app.imgfetch --limit 15
Polite by design: ~1 req/s (sleep between fetches), skip-existing, crash-safe Polite by design: ~1 req/s (sleep between fetches), skip-existing, crash-safe
(partial bytes land in .part and are atomically renamed), per-item failures (partial bytes land in .part and are atomically renamed), per-item failures
never kill the run. Honest end summary: `fetched N · skipped M · failed K`. never kill the run. Per-file CLI progress; honest end summary:
`fetched N · skipped M · failed K`.
UA + habits match the hunt scanner (reference/motorcycle-hunt-scan.py) so the UA + habits match the hunt scanner (reference/motorcycle-hunt-scan.py) so the
CDN sees the same client as the crawl. Stdlib only — no requests dependency. CDN sees the same client as the crawl. Stdlib only — no requests dependency.
""" """
import argparse import argparse
import json
import os import os
import sys import sys
import time import time
@@ -44,8 +50,10 @@ def fetch_one(img_url, dst):
}) })
try: try:
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
if getattr(resp, 'status', 200) != 200: status = (getattr(resp, 'status', None)
return 'HTTP %s' % resp.status or getattr(resp, 'code', None) or 200)
if status != 200:
return 'HTTP %s' % status
buf = resp.read() buf = resp.read()
if not is_webp(buf): if not is_webp(buf):
return 'not webp (%d bytes, %r)' % (len(buf), buf[:8]) return 'not webp (%d bytes, %r)' % (len(buf), buf[:8])
@@ -65,33 +73,76 @@ def fetch_one(img_url, dst):
pass pass
def parse_imgs(raw):
"""listings.imgs TEXT -> list of non-empty URL strings ([] when unparsable)."""
if not raw:
return []
try:
v = json.loads(raw)
except ValueError:
return []
if not isinstance(v, list):
return []
return [u.strip() for u in v if isinstance(u, str) and u.strip()]
def build_work(rows, img_dir):
"""Flatten listings rows into (listing_id, n_or_None, url, dst) downloads.
Gallery rows fetch every imgs[n]: n=0 -> the legacy {id}.webp (the primary
— one canonical file, no duplicate fetch), n>=1 -> {id}_{n}.webp. Legacy
rows (no gallery) keep the img_url -> {id}.webp chain. Work is deduped by
destination path.
"""
work = []
for r in rows:
g = parse_imgs(r['imgs'])
if g:
for n, url in enumerate(g):
dst = os.path.join(img_dir, ('%d.webp' % r['id'] if n == 0
else '%d_%d.webp' % (r['id'], n)))
work.append((r['id'], n, url, dst))
elif r['img_url']:
work.append((r['id'], None, r['img_url'],
os.path.join(img_dir, '%d.webp' % r['id'])))
seen, deduped = set(), []
for item in work:
if item[3] not in seen:
seen.add(item[3])
deduped.append(item)
return deduped
def run_fetch(db_path, img_dir, limit=None, sleep=SLEEP, quiet=False): def run_fetch(db_path, img_dir, limit=None, sleep=SLEEP, quiet=False):
conn = db.connect(db_path) conn = db.connect(db_path)
db.init_db(conn) db.init_db(conn)
rows = conn.execute( rows = conn.execute(
"SELECT id, img_url FROM listings" "SELECT id, img_url, imgs FROM listings"
" WHERE status='active' AND img_url IS NOT NULL" " WHERE status='active' ORDER BY id").fetchall()
" ORDER BY id").fetchall()
conn.close() conn.close()
os.makedirs(img_dir, exist_ok=True) os.makedirs(img_dir, exist_ok=True)
work = build_work(rows, img_dir)
total = len(work)
fetched = skipped = failed = 0 fetched = skipped = failed = 0
for row in rows: for idx, (lid, n, url, dst) in enumerate(work, 1):
if limit is not None and fetched >= limit: if limit is not None and fetched >= limit:
break break
dst = os.path.join(img_dir, '%d.webp' % row['id'])
if os.path.exists(dst): if os.path.exists(dst):
skipped += 1 skipped += 1
continue continue
result = fetch_one(row['img_url'], dst) result = fetch_one(url, dst)
if result == 'fetched': if result == 'fetched':
fetched += 1 fetched += 1
if not quiet:
print(' [%d/%d] id=%d n=%s ok' % (idx, total, lid, n))
else: else:
failed += 1 failed += 1
if not quiet: if not quiet:
print(' fail id=%d %s' % (row['id'], result)) print(' fail id=%d n=%s %s' % (lid, n, result))
time.sleep(sleep) time.sleep(sleep)
return {'fetched': fetched, 'skipped': skipped, 'failed': failed} return {'fetched': fetched, 'skipped': skipped, 'failed': failed,
'total': total}
def main(argv=None): def main(argv=None):
@@ -105,8 +156,8 @@ def main(argv=None):
args = ap.parse_args(argv) args = ap.parse_args(argv)
res = run_fetch(args.db, args.img_dir, limit=args.limit, sleep=args.sleep) res = run_fetch(args.db, args.img_dir, limit=args.limit, sleep=args.sleep)
print('fetched %d · skipped %d · failed %d' % ( print('fetched %d · skipped %d · failed %d · total %d' % (
res['fetched'], res['skipped'], res['failed'])) res['fetched'], res['skipped'], res['failed'], res['total']))
return 0 return 0