57fae0df49
- app/static/index.html: single-file app — DOM/CSS/stack engine ported
verbatim from reference/mockup-001-fidelity.html (DEPTH/SC/TY/DIM, drag/
fly/undo, one .topcard, no-lock commits, stamps, dossier sheet,
pull-to-close, empty state); production deltas only (UI-SPEC §7): deck
buffer + cursor refill (40/page), optimistic swipe persistence with
2s/5s/10s/15s retry queue + toast, badge-once-per-session, real list
filters + search (server-side), real Matches screen, /img/{id} everywhere
with placeholder fallback, SW registration, safe areas, desktop centering
- manifest.json + sw.js (precache shell, network-first API, cache-first imgs)
- icons: 192/512/maskable/apple-touch-180, dark tile with the brand
accent-gradient mark (Pillow)
- app/imgfetch.py: local mirror, hunt's Chrome-120 UA, ~1s pacing,
skip-existing, .part writes atomic, per-item failure isolation,
fetched/skipped/failed summary; verified: --limit 15 -> 15 fetched, full
run 5 more, re-run 0 fetched 20 skipped (20/20 mirrored)
115 lines
3.8 KiB
Python
115 lines
3.8 KiB
Python
"""Bike-Tinder local image mirror.
|
|
|
|
Downloads each active listing's card image once into data/img/{id}.webp.
|
|
|
|
.venv/bin/python -m app.imgfetch --db data/bike-tinder.sqlite3
|
|
.venv/bin/python -m app.imgfetch --limit 15
|
|
|
|
Polite by design: ~1 req/s (sleep between fetches), skip-existing, crash-safe
|
|
(partial bytes land in .part and are atomically renamed), per-item failures
|
|
never kill the run. Honest end summary: `fetched N · skipped M · failed K`.
|
|
|
|
UA + habits match the hunt scanner (reference/motorcycle-hunt-scan.py) so the
|
|
CDN sees the same client as the crawl. Stdlib only — no requests dependency.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from . import db
|
|
|
|
# Same UA string the hunt scanner uses.
|
|
UA = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
|
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
|
|
|
|
SLEEP = 1.0
|
|
TIMEOUT = 20
|
|
|
|
|
|
def is_webp(buf):
|
|
return (len(buf) >= 12 and buf[:4] == b'RIFF' and buf[8:12] == b'WEBP')
|
|
|
|
|
|
def fetch_one(img_url, dst):
|
|
"""Download img_url -> dst. Returns 'fetched' or an error string."""
|
|
part = dst + '.part'
|
|
req = urllib.request.Request(
|
|
img_url, headers={
|
|
'User-Agent': UA,
|
|
'Accept-Language': 'en-CA,en;q=0.9',
|
|
'Accept': 'image/avif,image/webp,image/*,*/*;q=0.8',
|
|
})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
|
|
if getattr(resp, 'status', 200) != 200:
|
|
return 'HTTP %s' % resp.status
|
|
buf = resp.read()
|
|
if not is_webp(buf):
|
|
return 'not webp (%d bytes, %r)' % (len(buf), buf[:8])
|
|
with open(part, 'wb') as fh:
|
|
fh.write(buf)
|
|
os.replace(part, dst) # atomic: never a half file under the final name
|
|
return 'fetched'
|
|
except urllib.error.HTTPError as e:
|
|
return 'HTTP %s' % e.code
|
|
except Exception as e: # noqa: BLE001 — one bad item never kills the run
|
|
return '%s: %s' % (type(e).__name__, e)
|
|
finally:
|
|
if os.path.exists(part):
|
|
try:
|
|
os.remove(part)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def run_fetch(db_path, img_dir, limit=None, sleep=SLEEP, quiet=False):
|
|
conn = db.connect(db_path)
|
|
db.init_db(conn)
|
|
rows = conn.execute(
|
|
"SELECT id, img_url FROM listings"
|
|
" WHERE status='active' AND img_url IS NOT NULL"
|
|
" ORDER BY id").fetchall()
|
|
conn.close()
|
|
|
|
os.makedirs(img_dir, exist_ok=True)
|
|
fetched = skipped = failed = 0
|
|
for row in rows:
|
|
if limit is not None and fetched >= limit:
|
|
break
|
|
dst = os.path.join(img_dir, '%d.webp' % row['id'])
|
|
if os.path.exists(dst):
|
|
skipped += 1
|
|
continue
|
|
result = fetch_one(row['img_url'], dst)
|
|
if result == 'fetched':
|
|
fetched += 1
|
|
else:
|
|
failed += 1
|
|
if not quiet:
|
|
print(' fail id=%d %s' % (row['id'], result))
|
|
time.sleep(sleep)
|
|
return {'fetched': fetched, 'skipped': skipped, 'failed': failed}
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description='Bike-Tinder image mirror')
|
|
ap.add_argument('--db', default='data/bike-tinder.sqlite3')
|
|
ap.add_argument('--img-dir', default='data/img')
|
|
ap.add_argument('--limit', type=int, default=None,
|
|
help='stop after this many downloads (small test runs)')
|
|
ap.add_argument('--sleep', type=float, default=SLEEP,
|
|
help='seconds between fetches (default %.1f)' % SLEEP)
|
|
args = ap.parse_args(argv)
|
|
|
|
res = run_fetch(args.db, args.img_dir, limit=args.limit, sleep=args.sleep)
|
|
print('fetched %d · skipped %d · failed %d' % (
|
|
res['fetched'], res['skipped'], res['failed']))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|