From 18bf31f361458110ab6e40a2f2b1e6490348299f Mon Sep 17 00:00:00 2001 From: AlexanderRozhenko Date: Tue, 10 Nov 2026 17:48:32 -0400 Subject: [PATCH] docs: add SOURCES, DESIGN, DATA-MODEL; record v1 decisions (Kijiji source, rank-all deck, badge-only feed); vendor hunt reference modules --- README.md | 4 +- docs/DATA-MODEL.md | 95 +++++++ docs/DESIGN.md | 101 +++++++ docs/PLAN.md | 7 +- docs/RANKING.md | 9 +- docs/SOURCES.md | 36 +++ docs/UI-SPEC.md | 1 + reference/README.md | 8 + reference/motorcycle-hunt-rank.py | 231 +++++++++++++++ reference/motorcycle-hunt-scan.py | 448 ++++++++++++++++++++++++++++++ 10 files changed, 932 insertions(+), 8 deletions(-) create mode 100644 docs/DATA-MODEL.md create mode 100644 docs/DESIGN.md create mode 100644 docs/SOURCES.md create mode 100644 reference/README.md create mode 100644 reference/motorcycle-hunt-rank.py create mode 100644 reference/motorcycle-hunt-scan.py diff --git a/README.md b/README.md index 5065f4b..97d547f 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ A private, Tinder-style web app for browsing the motorcycle market: swipe real l [swipe history] → [ranking] ``` -- **Crawl** — one marketplace source (chosen for scrape-simplicity; see `docs/SOURCES.md`) is fetched nightly by a pure script. No LLM involved, zero token cost. +- **Crawl** — **Kijiji** (NS + NB; decided 2026-11-10 — see `docs/SOURCES.md`) is swept nightly by the existing hunt crawler; a script-only importer folds the results in. No LLM involved, zero token cost. - **Import** — the importer upserts the source's catalog into the app database: new listings appear, changes update, vanished listings retire after repeated misses (never a blocked page, never a guess). - **Deck** — a Tinder-like PWA: card stack, drag to swipe, big buttons. Like / Nope / Maybe. "It's a match" moment when you like something. - **List & edit** — every entry in one table; re-file any call later (a nope can become a like). -- **Ranking** — cold start uses the hunt's rubric score; after ~20–30 swipes a small, explainable model orders the deck by *your* taste (retrained nightly). "Because you liked X" chips, never a black box. +- **Ranking** — cold start uses the hunt's rubric score; a cheap, fully explainable swipe-boost lands first (v1.x), then a small learned model (Phase 3, retrained nightly) orders the deck by *your* taste. "Because you liked X" chips, never a black box. ## Docs diff --git a/docs/DATA-MODEL.md b/docs/DATA-MODEL.md new file mode 100644 index 0000000..0d24884 --- /dev/null +++ b/docs/DATA-MODEL.md @@ -0,0 +1,95 @@ +# DATA-MODEL — Bike-Tinder + +SQLite, WAL mode, `busy_timeout`. One file: `data/bike-tinder.sqlite3`. Schema as consulted (Torres, 2026-11-10), locked for v1. + +## Tables + +```sql +listings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + region TEXT NOT NULL CHECK (region IN ('ns','nb')), + src_id TEXT NOT NULL, -- source ad id (Kijiji id in v1) + title TEXT NOT NULL, + price_display TEXT, -- "$5,000"; "Please Contact" + price_num INTEGER, -- NULL when unpackable + km INTEGER, -- NULL when not stated (~2/3 of cards) + location TEXT, + url TEXT NOT NULL, -- source ad page + img_url TEXT, -- source CDN card image + desc TEXT, -- card description snippet + family TEXT, -- top rubric family label (taste signal) + score INTEGER NOT NULL DEFAULT 0, -- rubric score, recomputed each import + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','gone','dup')), + dup_of INTEGER, -- canonical row id when status='dup' + first_seen TEXT, last_seen TEXT, + created_at TEXT DEFAULT (datetime('now')), updated_at TEXT, + UNIQUE (region, src_id) +); +CREATE INDEX idx_listings_deck ON listings (status, score DESC); +CREATE INDEX idx_listings_dup ON listings (dup_of) WHERE status = 'dup'; + +swipes ( + listing_id INTEGER PRIMARY KEY REFERENCES listings(id) ON DELETE CASCADE, + action TEXT NOT NULL CHECK (action IN ('like','nope','maybe')), + created_at TEXT DEFAULT (datetime('now')), + updated_at TEXT +); + +import_runs ( + id, started_at, ok, partial, + src_fetched_ns, src_fetched_nb, + total, new, gone, dup, error +); -- one observability row per nightly run + +prefs ( + key TEXT PRIMARY KEY, value TEXT +); -- deck_size, depth, etc. — tiny +``` + +Notes: + +- One row per swipe; a new swipe **upserts** (a re-swiped card overwrites its action). The latest action is the truth; edits from the list view are upserts. Append-history is deliberately not kept in v1. +- `import_runs` is the dashboard's evidence trail — every run writes one row, success or failure. + +## State semantics + +- **new** — `active` with no `swipes` row → the deck. +- **seen** — has any swipe row. +- **gone** — was `active`, then absent from ≥2 consecutive *good* imports → `status='gone'` (kept forever, swipes preserved; reappearance flips it back to `active`). Single misses and failed/partial imports never mark gone. +- **dup** — the second province's twin of the same ad (dedupe below) → hidden from the deck, visible under its canonical row in the list. +- **nope** — never re-enters the deck unless cleared from the list (`DELETE` swipe → back to unswiped). +- **maybe** — soft skip: the deck query re-admits `maybe` rows after **7 days** (computed at query time — no cron needed). Editing a maybe to like/nope is just an upsert. +- **like** — pins to the Liked ("matches") view. + +## Deck query (one statement) + +```sql +SELECT * FROM listings +WHERE status = 'active' + AND id NOT IN (SELECT listing_id FROM swipes WHERE action = 'nope') + AND id NOT IN (SELECT listing_id FROM swipes WHERE action IN ('like','maybe') + AND (action = 'like' OR updated_at > datetime('now','-7 days'))) +ORDER BY score DESC, first_seen DESC +LIMIT :deck_size; +``` + +(Implementers: a `LEFT JOIN swipes` variant is equivalent and cheaper — the intent above is the contract. Unscored rows sort below scored ones; nothing is hidden.) + +## Dedupe (Kijiji cross-province) + +- Group imports by `(normalized_title, price_num)`; when a group spans both provinces, keep the more-complete row canonical (img present → km present → older row) and mark the twin(s) `status='dup', dup_of=canonical`. The twin's URL is preserved on the canonical row (extra note field / desc suffix). +- Today's snapshot: 58/320 NS rows are twins of NB rows (dealers cross-post one ad to both provinces). +- False merges are possible (same title + price, different bike) but rare; every merge is logged in `import_runs` and reversible in the DB. Honest v1 tradeoff. + +## Import gates (must pass or the run aborts) + +1. State file `ok == true` and `partial == false`. +2. `fetched_at` within 26 hours. +3. Non-empty `items`. +4. On failure: log, leave the DB untouched (deck stays last-known-good). + +## Maintenance + +- **Backup:** nightly `sqlite3 .backup` to `data/backup/` immediately after import (house rule; the DB is the app's only mutable state). +- **Prune:** mirrored image files for rows `gone` >30 days; rows themselves are never auto-deleted in v1. +- **VACUUM:** optional; the DB is KV-sized (a few thousand rows). diff --git a/docs/DESIGN.md b/docs/DESIGN.md new file mode 100644 index 0000000..86ac2a8 --- /dev/null +++ b/docs/DESIGN.md @@ -0,0 +1,101 @@ +# DESIGN — Bike-Tinder + +Architecture for v1 (single source, single user, tailnet-only PWA). Written 2026-11-10 by Alex; engineering input from the Torres consultation (same date). Items marked 【decided】 are locked for v1. + +## Components + +| piece | what | where | +|---|---|---| +| Crawler | existing `motorcycle-hunt-scan.py` (**unchanged**) — sweeps Kijiji NS+NB nightly at 08:00 | Alex's pipeline (`~/.hermes/scripts/`) | +| Importer | reads the crawler's state JSONs, gates them, upserts into the app DB, scores, dedupes, queues images | `app/importer.py` | +| API | FastAPI + uvicorn — deck, swipes, list, stats, images | `app/api.py` | +| PWA | no-build single-page frontend (deck, list, matches, trends) | `app/static/` | +| DB | SQLite (WAL mode) | `data/bike-tinder.sqlite3` (stack volume) | + +## Data flow + +``` +08:00 motorcycle-hunt crawl (existing, unchanged) → state JSONs: kijiji-ns.json, kijiji-nb.json +08:15 importer (script-only, zero LLM): + gates → upsert → dedupe → score → gone-marking → image queue + → SQLite (WAL) +Deck phone ⇄ API ⇄ SQLite + ↘ images: local mirror first, hotlink (302) fallback +``` + +## Import design 【decided】 + +- **Source of truth = the hunt's state files, not a re-crawl.** Rationale: zero duplicate load on Kijiji; inherits the crawler's retry / partial / last-known-good / absence logic instead of re-implementing it; the format is stable and self-describing (`ok`, `partial`, `fetched_at`, per-page counts). +- **Escape hatch:** the app vendors the Kijiji parser from `reference/motorcycle-hunt-scan.py` behind a `--direct` flag — if the hunt pipeline ever moves or dies, the importer can sweep Kijiji itself. +- **Gates (fail loud, keep last good):** each state file must have `ok == true`, `partial == false`, `fetched_at` within 26 h, and non-empty `items`. Any failure → abort that region, log to `import_runs`, leave the DB untouched. +- **Gone marking:** a listing absent from ≥2 consecutive *good* imports flips to `gone` (never deleted; reappearance flips it back). House discipline: a single miss — or any blocked page — never counts as "sold." +- **Dedupe:** cross-province twins (same normalized title + price) collapse to one canonical row (`dup` status), keeping the more complete row (img > km > older). Merges are logged and reversible. +- **Scoring:** rubric tables ported from `reference/motorcycle-hunt-rank.py` (family tiers + extras + veto) — the score is Glenn's accumulated taste, computed at import, stored on each row. See `RANKING.md`. +- **Idempotent:** safe to re-run any time; also exposed as `POST /api/import` for manual triggers. + +## Scheduling 【decided】 + +An in-container daily task (asyncio, 08:15 America/Halifax) inside the API process, with on-boot catch-up for missed runs. No host crontab is involved (the host user crontab is empty; the 08:00 crawl lives in Hermes cron — the app depends on neither). Every run writes an `import_runs` row; `GET /api/stats` surfaces the last run. + +## API surface (v1) + +| route | purpose | +|---|---| +| `GET /api/health` | liveness + last-import summary | +| `GET /api/stats` | counts: active / gone / dup / swiped; last import time | +| `GET /api/deck?limit=` | next cards (deck query below) | +| `POST /api/swipe` `{id, action}` | record or edit like / nope / maybe (upsert) | +| `DELETE /api/swipe/{id}` | clear a call (returns the card to the deck) | +| `GET /api/list?filter=&q=&offset=` | full list, filters + search | +| `POST /api/import` | manual import trigger | +| `GET /img/{id}` | cached image file, else 302 to source CDN | + +**Deck query:** `status='active'`, not noped, and either no swipe or a `maybe` older than 7 days — ordered `score DESC, first_seen DESC`. Unscored rows sort below scored ones but still appear — nothing is hidden. + +## Images 【decided: hybrid mirror + hotlink】 + +- The importer pulls each new listing's card image once into `data/img/` (~100 KB avg; ≈64 MB the first day, then ~20–30 MB/month). Cards reference the local mirror; a missing file falls back to a 302 hotlink of the source CDN (verified token-free and size-rewritable). +- Why: ad removal eventually kills CDN URLs (unverifiable until it happens — the mirror makes it a non-issue), phone-side speed, and "gone" cards stay viewable. +- Bootstrap: today's state carries `img` on only ~3% of rows — the first import does a one-time headless re-sweep (8 pages/province via the vendored parser — 16 fetches, not 640 detail pages) to harvest card images. A placeholder renders for any listing still without a photo; it never holds up the deck. Mirrored files for rows gone >30 days are pruned. + +## Frontend + +No-build SPA — vanilla JS + CSS, pointer-events drag, keyboard arrows on desktop (~300 lines). A small service worker caches the shell; the API is always live. Screens, gestures, and visual anatomy: see `UI-SPEC.md`. + +## Deployment (house constraints) + +- **Stack dir:** `/var/mnt/DATA/podman-stacks/bike-tinder/` — `docker-compose.yml` + `Dockerfile` (`python:3.14-slim` + fastapi/uvicorn/requests), volume `./data:/data`, port `127.0.0.1:9453:8000`, `restart: unless-stopped`. +- After adding files: `chcon -Rt container_file_t /var/mnt/DATA/podman-stacks/bike-tinder`. +- **Systemd user unit** `podman-stack-bike-tinder.service`, cloned from the machine's `podman-stack-searxng.service` pattern: `Type=oneshot`, `RemainAfterExit=yes`, **`Delegate=yes`** (keeps the stack out of the hermes-gateway cgroup so gateway restarts can't SIGKILL it); ExecStart/ExecStop = the venv `podman-compose up -d` / `down`. +- **Tailnet:** `tailscale serve --bg 9453` → reachable at `http://rotarran.tailf35a34.ts.net:9453`, tailnet-only. The tailnet name can't be curled from this host — first deploy verification happens from Glenn's phone. +- **Backups:** nightly `sqlite3 .backup` into `data/backup/` right after import (the DB is the only mutable state; house rule). Restore = stop stack, swap the file, start. + +## Security + +Single user; **no auth in v1** — localhost bind + tailnet-only reachability. An optional `AUTH_TOKEN` env var is noted as a one-line later hardening. No secrets are required in the stack. + +## Risks & mitigations (from consultation) + +| risk | mitigation | +|---|---| +| Kijiji markup drift | parser keys on `data-testid`; failures → gates abort, deck stays last-known-good; crawler's fail/partial tracking monitors | +| Anti-bot on deep sweeps | ship at the proven 8-page window; grow only after two clean weeks | +| Image persistence after ad removal | local mirror (built-in) | +| Cross-province dedupe false merges | logged, reversible, low blast radius; canonical keeps the more complete row | +| Hunt pipeline dependency | `fetched_at` gate fails loud; vendored `--direct` parser escape hatch | +| Dealer blast dominance in score ties | tie-break `first_seen DESC`; "hide this seller" planned as a v1.1 prefs row | +| Rubric blindness (no family match) | rows still appear, sorted below scored ones — nothing hidden | + +## Repo layout (target) + +``` +app/ importer.py · api.py · static/{index.html,app.js,app.css,manifest.json,sw.js,icons} +deploy/ docker-compose.yml · Dockerfile · podman-stack-bike-tinder.service +reference/ copies of the hunt modules the app vendors from +data/ runtime state (gitignored) +docs/ this design set +``` + +## Runbook (completed at Phase 4) + +Deploy / update / rollback steps, log locations, import troubleshooting (gates), backup-restore drill, phone verification checklist. diff --git a/docs/PLAN.md b/docs/PLAN.md index 5807a03..92ec010 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -10,8 +10,9 @@ A single-source, single-user, tailnet-only Tinder-style PWA on the Rotarran: nig - [x] Engineering consultation (Torres) → memo - [x] Design docs in this repo: SOURCES, DESIGN, DATA-MODEL, RANKING, UI-SPEC -- [ ] Glenn's read on source choice + docs -- **Exit criteria:** docs merged to `main`; source decision recorded. +- [x] Glenn's calls (2026-11-10): v1 source = **Kijiji** · deck = show all, taste-ranked · new-bike feed = in-app badge only +- [ ] Glenn's skim of the docs — build gate opens after +- **Exit criteria:** docs merged to `main` (done 2026-11-10); source decision recorded in `SOURCES.md`. ## Phase 1 — Data spine @@ -53,5 +54,5 @@ A single-source, single-user, tailnet-only Tinder-style PWA on the Rotarran: nig ## Open decisions -- v1 source — pending consultation evidence (see `docs/SOURCES.md`) +- v1 source — **resolved 2026-11-10: Kijiji** (see `docs/SOURCES.md`) - Mockup direction — Glenn picks at Phase 2 start diff --git a/docs/RANKING.md b/docs/RANKING.md index 42fce40..bfae225 100644 --- a/docs/RANKING.md +++ b/docs/RANKING.md @@ -1,6 +1,6 @@ # RANKING — Bike-Tinder -The deck is ordered by predicted taste. Two stages: a cold start (the hunt's existing rubric), then a small learned model that never becomes a black box. +The deck is ordered by predicted taste. Three rungs, in order: the hunt's existing rubric (v1), a cheap explainable boost from swipe clusters (v1.x), then a small learned model (Phase 3). Nothing ever becomes a black box. ## Signals - Like = strong positive (+1). Maybe = weak positive (+0.25, kept in its own bucket). Nope = negative (−1). List-view edits count identically to in-deck swipes; the latest call wins. @@ -13,7 +13,10 @@ Per listing: family tier (from the hunt rubric), displacement (cc), price, year, ## Stage 1 — Cold start Order by the hunt's existing rubric score (family tiers + MVI / whitewalls / windshield / spokes / ABS bonuses + price fit). This is already explainable (reason strings exist in the hunt system). Deck = rubric order until ≥ 20 swipes. -## Stage 2 — Learned order +## Stage 1.5 — Cheap swipe boost (v1.x, post-launch) +Before any model exists, the importer can nudge `score` with plain swarm signals — e.g. +N when recent likes cluster in a family, small time-decay otherwise — recomputed each import, zero ML, fully explainable. Ships as v1.x if Glenn wants it; v1 itself stays pure rubric. + +## Stage 2 — Learned order (Phase 3 roadmap) - Model: logistic regression on the features above; target = like vs nope (maybes contribute softly positive). Trained on all swipe history including edits. - Pure-Python or numpy implementation (tiny data — no heavy ML dependency). Params + training metadata live in the DB; versioned. - Retrain nightly right after import; on-demand retrain available from the app. @@ -27,7 +30,7 @@ Each card can show up to 2 chips: top positive contributions for that bike ("low ## Evaluation - Track: swipes/session, like-rate trend, maybe-rate, undo rate, and "liked-item rank" (how deep liked items sat — should trend shallow). -- Optional weekly digest via Conduit if Glenn wants it (default: silent). +- Notifications (decided 2026-11-10): in-app "new since last visit" badge only — no Conduit pushes. A weekly digest remains possible later if asked for. ## Open decisions - Maybe-handling: weak positive + separate bucket (proposed) vs pure third bucket — confirm with Glenn at Phase 2. diff --git a/docs/SOURCES.md b/docs/SOURCES.md new file mode 100644 index 0000000..df277a3 --- /dev/null +++ b/docs/SOURCES.md @@ -0,0 +1,36 @@ +# SOURCES — v1 source decision + +**Decision (2026-11-10): Kijiji — NS + NB windows (`kijiji-ns` / `kijiji-nb`). One site, one parser, two regional feeds.** + +Chosen after the Torres engineering consultation; the evidence below is from live probes + snapshot analysis run 2026-11-10 (memo + probe scripts kept with the project's working notes). + +## Candidates at a glance + +| candidate | live volume | churn | field fill | photos | parse | verdict | +|---|---|---|---|---|---|---| +| **Kijiji** (ns+nb) | ~640 cards in the current window; depth beyond it (page 15 still full in NS; ≥600 NS alone) | **~18 new + 18 gone / day (NS)** | price ~100%, km 30–36%, title/loc/desc ~100% | image on **every card** at parse time | SSR HTML; transient-partial quirk **already mitigated** (retries + last-known-good, re-verified stable) | **v1 source** | +| AutoTrader.ca (ns+nb) | ~117 cruiser listings; full motorcycle view padded with ATV/SxS (34/53 and 49/64 rows from ONE dealer) | slow | 100% (structured JSON) | inline | structured `__NEXT_DATA__` JSON — the easiest parse | source #2 later (field-rich enrichment) | +| Dealer Spike (procycle / tbb) | 160 units each | none (static stock) | price only; no km/desc/year | present | HTML w/ quirks | later source (new stock) | +| Facebook Marketplace | — | — | — | — | — | **never scraped — Glenn's manual column** | + +## Why Kijiji over "the easiest scrape" + +Glenn's words were "the easiest to scrape." Literally easiest = AutoTrader (structured JSON, `numberOfResults` handed to you). But the deck is the product: Kijiji carries ~4–5× the live volume, real daily churn to feed a nightly deck, a photo on every card, and a private + dealer mix — and its one known flakiness is already a solved problem in the production crawler that has run daily for weeks. Engineering ease that ships a thin, slow-turning deck is not actually easy. AutoTrader returns as source #2 through the same import path (its parser already exists in the hunt pipeline). + +## Evidence highlights (2026-11-10) + +- Snapshot: `kijiji-ns` 320 items, `kijiji-nb` 320 items; **58/320 NS rows are NB cross-post twins** → importer dedupes. +- Live fetch: page 1 fetched twice → byte-identical 40 cards (no flake that day); **40/40 cards carry image URLs**; pages 9 and 15 still full (depth available beyond the current window). +- Images: `media.kijiji.ca` URLs are **token-free and size-rewritable** (`?rule=kijijica-640-webp` → 200; `960-webp` rewrite → 200; URL without rule → 404 — the rule is the size selector, not a token). +- Anti-bot: none observed across 15 pages of probes + production traffic. + +## Scope & growth path + +- **v1 mirrors the proven 8-page/province window** (~640 cards). Depth exists past it; grow only after two clean weeks, throttled — anti-bot behavior on deep sweeps is untested long-term. +- **v2** adds AutoTrader (same importer path) — dense fields complement Kijiji's private-sale mix. +- **Later** dealer inventories (new stock); the import schema is source-agnostic from day one. + +## Revisit criteria + +- Kijiji block / anti-bot escalation persisting ≥2 weeks → flip the order, AutoTrader first. +- Kijiji markup drift breaking the parser → importer gates fail loud (deck stays last-known-good); the crawler's `fail_streak` monitoring already covers detection. diff --git a/docs/UI-SPEC.md b/docs/UI-SPEC.md index 49b1a03..7410da0 100644 --- a/docs/UI-SPEC.md +++ b/docs/UI-SPEC.md @@ -6,6 +6,7 @@ Fidelity goal (Glenn's words): "as close to Tinder as possible." Photo-first, da ### 1. Deck (default) - Full-height card stack: the top card is draggable; the next card peeks, scaled (~95%) behind it. +- Scope (decided 2026-11-10): every active bike appears in the deck eventually — ranking reorders, never hides. - Card anatomy: photo fills the card; bottom gradient overlay carries: year + make + model (title), price (large), sub-line: km · place · first seen. Small chips row when signals exist (e.g. "MVI", "whitewalls", "bags") — only derived from real listing data, never invented. - Stamps: during drag, "LIKE" (green) / "NOPE" (red) / "MAYBE" (amber) stamps fade in over the card proportional to horizontal drag; rotation up to ±12° follows the finger. - Commit: release past ~35% width (or a flick with velocity) → the card flies off; otherwise it springs back. diff --git a/reference/README.md b/reference/README.md new file mode 100644 index 0000000..5444778 --- /dev/null +++ b/reference/README.md @@ -0,0 +1,8 @@ +# reference/ + +Copies of the hunt-pipeline modules this app vendors from, taken 2026-11-10: + +- `motorcycle-hunt-rank.py` — rubric tables (family tiers, extras, veto) ported into the app's scorer. +- `motorcycle-hunt-scan.py` — the Kijiji parser vendored behind the importer's `--direct` escape hatch. + +Source of truth: `~/.hermes/scripts/` on the Rotarran (Alex's pipeline). If the originals change materially, refresh these copies deliberately — they are references, not live links. diff --git a/reference/motorcycle-hunt-rank.py b/reference/motorcycle-hunt-rank.py new file mode 100644 index 0000000..efe48e6 --- /dev/null +++ b/reference/motorcycle-hunt-rank.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +""" +motorcycle-hunt-rank — score the current hunt board for Glenn's small-cruiser +search (NS + NB). Reads the state JSONs written by motorcycle-hunt-scan.py and +prints a ranked candidate list. Reusable on cron wakes. + +Usage: + python3 motorcycle-hunt-rank.py [--top N] [--kind cruiser|style|all] [--src NAME] + +Rubric (see the motorcycle-hunt skill for the full rationale): + base weight per model family (S = Indian Scout + Vulcan 900 Custom top tier, + T1 = strong fits, T2/style = wildcards), plus extras (MVI +6, whitewalls +5, windshield + /bags +3, spokes +3, ABS +2), mileage and year bonuses, price-fit vs the + budgets (used soft ~$5k / stretch ~$6.5k; new ~$10k / stretch ~$11.5k). +""" +import argparse +import glob +import html +import json +import os +import re + +STATE = os.path.expanduser('~/.hermes/scripts/motorcycle-hunt.d') + +FAMILIES = [ + (r'\bscout\b', 92, 'S', 'Indian Scout — top priority (sport×cruise)'), + (r'vulcan\s*900\s*custom', 88, 'S', 'Vulcan 900 Custom — the dream spec'), + (r'vulcan\s*900', 62, 'T1', 'Vulcan 900'), + (r'vulcan\s*(classic|lt|nomad)', 58, 'T1', 'Vulcan Classic/LT'), + (r'vulcan\s*s\b', 50, 'T1', 'Vulcan S'), + (r'vulcan\s*500', 40, 'T2', 'Vulcan 500'), + (r'\bvulcan\b', 45, 'T1', 'Vulcan'), + (r'\bvolusia\b', 44, 'T1', 'Volusia (C50 predecessor)'), + (r'rebel\s*500', 50, 'T1', 'Rebel 500'), + (r'rebel\s*300', 44, 'T1', 'Rebel 300'), + (r'\brebel\b', 46, 'T1', 'Rebel'), + (r'boulevard\s*c50', 50, 'T1', 'Boulevard C50'), + (r'\bc50t?\b', 48, 'T1', 'Boulevard C50/C50T'), + (r'boulevard\s*(m50|s50)', 44, 'T1', 'Boulevard M50/S50'), + (r'boulevard\s*(c90|m90|m109)', 42, 'T2', 'Boulevard big twin'), + (r'm109r', 42, 'T2', 'Boulevard M109R'), + (r'\bboulevard\b', 40, 'T1', 'Boulevard'), + (r'\bintruder\b', 36, 'T2', 'Intruder'), + (r'shadow\s*(aero|phantom|spirit)', 48, 'T1', 'Shadow Aero/Phantom/Spirit'), + (r'shadow\s*750', 44, 'T1', 'Shadow 750'), + (r'\bshadow\b', 42, 'T1', 'Shadow'), + (r'\bsabre\b', 34, 'T2', 'Shadow Sabre'), + (r'v-?star\s*(650|950)', 46, 'T1', 'V-Star 650/950'), + (r'v-?star\s*1100', 40, 'T2', 'V-Star 1100'), + (r'v-?star', 42, 'T1', 'V-Star'), + (r'iron\s*883', 46, 'T1', 'Iron 883'), + (r'\bsportster\b', 40, 'T1', 'Sportster'), + (r'super\s*meteor', 48, 'T1', 'Super Meteor 650'), + (r'meteor\s*350', 42, 'T1', 'Meteor 350'), + (r'\bmeteor\b', 42, 'T1', 'Meteor'), + (r'hunter\s*350', 38, 'T2', 'Hunter 350'), + (r'classic\s*(350|650)', 36, 'T2', 'RE Classic'), + (r'royal\s*enfield', 34, 'T2', 'Royal Enfield'), + (r'\beliminator\b', 38, 'T2', 'Eliminator'), + (r'\bvirago\b', 34, 'T2', 'Virago'), + (r'\bmagna\b', 32, 'T2', 'Magna'), + (r'\bnightster\b', 32, 'T2', 'Nightster'), + (r'forty-?eight', 32, 'T2', 'Forty-Eight'), + (r'\bbonneville\b', 30, 'style', 'Bonneville (style-adjacent)'), + (r'\bw800\b', 28, 'style', 'W800'), + (r'z650rs', 28, 'style', 'Z650RS'), + (r'moto\s*guzzi|\bv7\b', 30, 'style', 'V7'), + (r'\bscrambler\b', 24, 'style', 'Scrambler'), + (r'\bcontinental\s*gt\b', 26, 'style', 'Continental GT'), +] + +# category vetoes — not a small cruiser (dirt/sport/touring/powersports/gear) +VETO = [ + (r'\b(atv|quad|side[\s-]?by[\s-]?side|utv|ranger|outlander|renegade|ryker|spyder|ski-doo|seadoo|sea-doo|snowmobile|snow\s*blower|lawn|generator|pump|watercraft|boat)\b', 'powersports'), + (r'\b(dirt\s*bike|motocross|enduro)\b', 'dirt'), + (r'\b(crf|kx\d|yz\d|rm-?z|klx|exc|xc-w|xc-f|fe\s*\d|fc\s*\d|mc\s*\d|sx\b|dr-?z|ttr|kdx)\b', 'dirt'), + (r'\b(ninja|zx-?\d|gsx-?[rs]|cbr|yzf|hayabusa|panigale|fireblade|gixxer|katana|r\d\s)\b', 'sport'), + (r'\b(v-?strom|versys|africa\s*twin|multistrada|t[eé]n[eé]r[eé]|transalp|super\s*adventure)\b', 'adv'), + (r'\b(gold\s*wing|voyager|electra\s*glide|road\s*king|ultra\s*classic|f6b)\b', 'tourer'), + (r'\b(helmet|half-?shell|jacket|glove|boots?\b|seat\b|exhaust|muffler|tire|tyre|rim\b|brake|lever|mirror|cover\b|battery|sprocket|for\s*parts|parts\b|project\s*bike|salvage|wreck)\b', 'gear'), +] + +EXTRAS = [ + (r'\b(mvi|safety|inspected|inspection|certified|e-?test)\b', 6, 'MVI/cert'), + (r'white\s*wall', 5, 'whitewalls'), + (r'\bspokes?\b|spoked', 3, 'spokes'), + (r'wind\s*screen|windshield', 3, 'windshield'), + (r'saddle\s*bags?|panniers?|\bbags\b', 3, 'bags'), + (r'\babs\b', 2, 'ABS'), +] + + +def load_sources(): + out = {} + for f in sorted(glob.glob(os.path.join(STATE, '*.json'))): + name = os.path.basename(f)[:-5] + if name.endswith('.tmp'): + continue + try: + with open(f, encoding='utf-8') as fh: + out[name] = json.load(fh) + except Exception: + pass + return out + + +def km_num(it): + s = str(it.get('km') or '') + m = re.search(r'([\d,]+)', s) + return int(m.group(1).replace(',', '')) if m else None + + +def classify(title, desc): + text = (title + ' ' + desc).lower() + ttext = title.lower() + for pat, why in VETO: + if re.search(pat, ttext): + return None, None, None, why + best = None + for pat, weight, tier, label in FAMILIES: + if re.search(pat, text): + if best is None or weight > best[1]: + best = (pat, weight, tier, label) + if not best: + return None, None, None, None + return best[2], best[1], best[3], None + + +def score_item(src, it): + title = it.get('title') or '' + desc = it.get('desc') or '' + kind, base, label, veto = classify(title, desc) + if not kind: + return None + flags = [] + s = base + text = (title + ' ' + desc).lower() + for pat, pts, fl in EXTRAS: + if re.search(pat, text): + s += pts + flags.append(fl) + price = it.get('price_num') + if price: + is_new = (it.get('cond') == 'NEW' or it.get('offer') == 'N') + if is_new: + if price <= 11500: + s += 3 + elif price <= 13500: + s += 1 + else: + s -= 1 + flags.append('new') + else: + if price <= 5000: + s += 5 + elif price <= 6500: + s += 3 + elif price <= 8000: + s += 1 + elif price <= 10000: + s -= 1 + else: + s -= 3 + else: + flags.append('no price') + km = km_num(it) + if km == 0: + km = None # 0 km on a listing is a placeholder, not delivery mileage + if km is not None: + if km < 5000: + s += 4 + elif km < 15000: + s += 2 + elif km > 90000: + s -= 5 + elif km > 60000: + s -= 3 + flags.append('{}km'.format(km)) + ym = re.search(r'\b(19\d\d|20[0-2]\d)\b', title) + year = int(ym.group(1)) if ym else None + if year: + if year >= 2020: + s += 2 + elif year >= 2015: + s += 1 + elif year <= 2005: + s -= 1 + return {'src': src, 'id': it.get('id') or '', 'kind': kind, 'label': label, + 'score': s, 'price': it.get('price') or '', 'km': km, 'year': year, + 'title': title, 'place': it.get('loc') or it.get('city') or '', + 'url': it.get('url') or '', 'flags': flags, 'desc': desc, + 'first_seen': it.get('first_seen', ''), 'prev': it.get('price_prev')} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--top', type=int, default=80) + ap.add_argument('--kind', default='all') + ap.add_argument('--src', default=None) + args = ap.parse_args() + + sources = load_sources() + rows, counts = [], {} + for src, st in sources.items(): + if args.src and src != args.src: + continue + for vid, it in (st.get('items') or {}).items(): + it = dict(it) + it.setdefault('id', vid) + r = score_item(src, it) + if r is None: + continue + counts[r['kind']] = counts.get(r['kind'], 0) + 1 + if args.kind != 'all' and r['kind'] != args.kind: + continue + rows.append(r) + rows.sort(key=lambda r: (-r['score'], r['price'] or '$999999')) + print('=== motorcycle-hunt rank | board:', {k: v for k, v in sorted(counts.items())}, + '| shown:', min(len(rows), args.top), '===') + for r in rows[:args.top]: + desc = re.sub(r'\s+', ' ', r['desc'])[:105] + print('[{}] {} | {} | {} | {} | {} | {} | km={} | {}'.format( + r['kind'], r['score'], r['label'], r['src'], r['id'], r['price'], + r['title'][:64], r['km'] if r['km'] is not None else '-', r['url'][:95])) + if desc: + print(' desc: {}'.format(desc)) + print('=== families unmatched (not shown): kinds =', counts, '===') + + +if __name__ == '__main__': + main() diff --git a/reference/motorcycle-hunt-scan.py b/reference/motorcycle-hunt-scan.py new file mode 100644 index 0000000..1328b94 --- /dev/null +++ b/reference/motorcycle-hunt-scan.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +""" +motorcycle-hunt — daily board scan for Glenn's small-cruiser hunt (NS + NB). + +Sources (fixed URLs = stable diff baseline; calibrated live 2026-11-09): + kijiji-ns Kijiji motorcycles, Nova Scotia (path pagination /page-N/) + kijiji-nb Kijiji motorcycles, New Brunswick (path pagination /page-N/) + autotrader-ns AutoTrader cruiser body, NS (?sort=age&desc=1&page=N, stops at <20) + autotrader-nb AutoTrader cruiser body, NB + procycle Pro Cycle (Dartmouth NS) inventory, Dealer Spike (pg=N) + tbb Toys For Big Boys (NB) inventory, Dealer Spike (pg=N) + +Prints one byte-stable line per listing: + src|id|price|title|place|km +sorted by source order then id. No timestamps in stdout — identical output +means nothing changed. A changed line set wakes the agent, which scores the +diff, updates the vault tracker, and alerts only on new-above-bar / +material price drop / genuine improvement. + +State: ~/.hermes/scripts/motorcycle-hunt.d/.json + items{id:{...,img?,first_seen,last_seen,price_prev?}}, absent{id:{first_absent,times_missing,last?}}, + pages{page:cards}, ok, partial, fail_streak, fetched_at. +Each item may carry `img`, a hotlinkable picture URL (kijiji ?rule=kijijica-640-webp, +autotrader /720x540.jpg, dealerspike v1/640x480; a model render for new dealer +units is fine too). img never appears in stdout, so it cannot affect the change +gate; a previous img is kept on re-sight (curated pictures survive rescans — +only genuinely new items take theirs from the card). + +Failure policy: per-page retries (Kijiji SSR flakes are common — retry then +flag). A failed source falls back to its last-known items so transient blocks +stay byte-stable (silent). Only at fail_streak>=2 does a `!fail|` marker +appear (single wake to inform, no daily noise). `!partial|` marks a day +where some pages failed or under-filled; expected cards are ~40 (Kijiji), +20 (AutoTrader), 20 (Dealer Spike). + +Flags: --seed/--full (full-depth sweep; also the daily coverage), --source NAME (refresh one +source), --pages N (override depth), --verbose (summary to stderr). +""" +import argparse +import datetime +import html +import json +import os +import re +import sys +import time +import urllib.request + +HOME = os.path.expanduser('~') +STATE_DIR = os.path.join(HOME, '.hermes', 'scripts', 'motorcycle-hunt.d') +UA = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36') +SLEEP = 2.0 + +KJ_NS_BASE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/c30l9002' +KJ_NS_PAGE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/page-{}/c30l9002' +KJ_NB_BASE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/c30l9005' +KJ_NB_PAGE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/page-{}/c30l9005' +AT_TMPL = ('https://www.autotrader.ca/motorcycles-atvs/lst/reg_{}/bt_cruiser' + '?sort=age&desc=1&page={}') +PC_TMPL = 'https://www.procycleonline.com/--inventory?category=motorcycle%20%2F%20scooter&pg={}' +TBB_TMPL = 'https://www.toysforbigboys.ca/--inventory?category=motorcycle+%2F+scooter&pg={}' + +SOURCES = [ + {'name': 'kijiji-ns', 'kind': 'kijiji', 'base': KJ_NS_BASE, 'page': KJ_NS_PAGE, 'pages': 4, 'min': 25}, + {'name': 'kijiji-nb', 'kind': 'kijiji', 'base': KJ_NB_BASE, 'page': KJ_NB_PAGE, 'pages': 4, 'min': 25}, + {'name': 'autotrader-ns', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'ns', 'pages': 4}, + {'name': 'autotrader-nb', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'nb', 'pages': 4}, + {'name': 'procycle', 'kind': 'dealerspike', 'tmpl': PC_TMPL, 'host': 'https://www.procycleonline.com', 'pages': 4}, + {'name': 'tbb', 'kind': 'dealerspike', 'tmpl': TBB_TMPL, 'host': 'https://www.toysforbigboys.ca', 'pages': 4}, +] +SEED_PAGES = {'kijiji-ns': 8, 'kijiji-nb': 8, 'autotrader-ns': 5, + 'autotrader-nb': 5, 'procycle': 8, 'tbb': 8} + + +def now_iso(): + return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def clean(s, limit=None): + if s is None: + return '' + s = html.unescape(str(s)) + s = re.sub(r'<[^>]+>', ' ', s) + s = re.sub(r'\s+', ' ', s).strip() + if limit and len(s) > limit: + s = s[:limit] + return s + + +def num_from_price(price): + if not price or '$' not in price: + return None + m = re.search(r'\d[\d,]*', price) + return int(m.group(0).replace(',', '')) if m else None + + +def norm_kj_img(u): + """Kijiji CDN image URL -> 640px webp variant (token-free, hotlinkable).""" + if not u: + return '' + u = html.unescape(u) + if 'media.kijiji.ca' in u: + return u.split('?')[0] + '?rule=kijijica-640-webp' + return '' + + +def norm_at_img(u): + """AutoScout24/AutoTrader picture -> 720x540 jpg variant.""" + if not u: + return '' + u = html.unescape(u) + if 'pictures.autoscout24' in u: + return re.sub(r'/\d+x\d+\.(?:jpg|jpeg|webp|png)$', '/720x540.jpg', u) + return '' + + +def norm_ds_img(u): + """Dealer Spike imglib -> 640x480 variant (case preserved on the path).""" + if not u: + return '' + u = html.unescape(u).replace('/', '/') + if u.startswith('//'): + u = 'https:' + u + m = re.search(r'cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/' + r'([^"\s|]+?\.(?:jpg|png))', u, re.I) + if m: + return 'https://cdn.dealerspike.com/imglib/v1/640x480/imglib/' + m.group(1) + return '' + + +def get(url, timeout=35): + req = urllib.request.Request(url, headers={ + 'User-Agent': UA, + 'Accept-Language': 'en-CA,en;q=0.9', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + }) + with urllib.request.urlopen(req, timeout=timeout) as r: + data = r.read() + if len(data) < 2000: + raise ValueError('short body: %d bytes' % len(data)) + return data.decode('utf-8', 'ignore') + + +# ---------- Kijiji ---------- + +def parse_kijiji(raw): + out = {} + for blk in raw.split('data-testid="listing-card"')[1:]: + blk = blk[:6000] + m = re.search(r'data-listingid="(\d+)"', blk) + if not m: + continue + vid = m.group(1) + link = re.search(r'data-testid="listing-link" href="([^"]+)"', blk) + t = re.search(r'data-testid="listing-link"[^>]*>(.*?)', blk, re.S) + pr = re.search(r'data-testid="(?:autos-listing-price|listing-price)"[^>]*>(.*?)(\d[\d,]*)\s*km<', pre) + lo = re.search(r'data-testid="listing-location"[^>]*>(.*?)]*>(.*?)

', blk, re.S) + imt = re.search(r']*data-testid="listing-card-image"[^>]*>', blk) + ims = re.search(r'src="([^"]+)"', imt.group(0)) if imt else None + price = clean(pr.group(1)) if pr else '' + out[vid] = { + 'title': clean(t.group(1), 90) if t else '', + 'price': price, + 'price_num': num_from_price(price), + 'loc': clean(lo.group(1)) if lo else '', + 'km': km.group(1) if km else '', + 'url': link.group(1) if link else '', + 'desc': clean(de.group(1), 160) if de else '', + 'img': norm_kj_img(ims.group(1)) if ims else '', + } + return out + + +def fetch_kijiji(src, pages): + got, per = {}, {} + for p in range(1, pages + 1): + url = src['base'] if p == 1 else src['page'].format(p) + items = None + for att in range(3): + try: + items = parse_kijiji(get(url)) + except Exception: + items = None + enough = bool(items) and (len(items) >= src['min'] or (p == 1 and len(items) >= 15)) + if enough: + break + if att < 2: + time.sleep(4) + per[str(p)] = None if items is None else len(items) + if items: + got.update(items) + time.sleep(SLEEP) + ok = bool(per.get('1')) + return ok, got, per + + +# ---------- AutoTrader ---------- + +def parse_at(raw): + m = re.search(r'', raw, re.S) + if not m: + return None + d = json.loads(m.group(1)) + pp = (d.get('props') or {}).get('pageProps') or {} + ls = pp.get('listings') + if ls is None: + return None + out = {} + for x in ls: + vid = str(x.get('id') or x.get('url') or '') + if not vid: + continue + price = x.get('price') or {} + loc = x.get('location') or {} + veh = x.get('vehicle') or {} + seller = x.get('seller') or {} + out[vid] = { + 'title': clean(x.get('title'), 90), + 'price': clean(price.get('priceFormatted')), + 'price_num': price.get('priceRaw'), + 'city': clean(loc.get('city')), + 'prov': clean(loc.get('provinceCode')), + 'km': clean(veh.get('mileageInKm')), + 'make': clean(veh.get('make')), + 'model': clean(veh.get('model')), + 'year': veh.get('modelYear'), + 'offer': clean(veh.get('offerType')), + 'seller': clean(seller.get('companyName')), + 'section': clean(x.get('searchResultSection')), + 'url': x.get('url') or '', + 'desc': clean(x.get('description'), 220), + 'img': norm_at_img((x.get('images') or [None])[0]), + } + return out + + +def fetch_at(src, pages): + got, per = {}, {} + for p in range(1, pages + 1): + url = src['tmpl'].format(src['reg'], p) + items = None + for att in range(2): + try: + items = parse_at(get(url)) + if items is not None: + break + except Exception: + items = None + time.sleep(3) + per[str(p)] = None if items is None else len(items) + if items: + got.update(items) + if items is not None and len(items) < 20: + break + time.sleep(SLEEP) + ok = bool(per.get('1')) + return ok, got, per + + +# ---------- Dealer Spike ---------- + +def parse_ds(raw, host): + out = {} + heads = list(re.finditer(r'vehicle-heading__link[^>]*?href="([^"]+)"[^>]*?title="([^"]*)"', raw)) + for i, h in enumerate(heads): + href, title = h.group(1), clean(h.group(2), 90) + seg_end = heads[i + 1].start() if i + 1 < len(heads) else min(len(raw), h.end() + 5000) + seg = raw[h.end():seg_end] + pm = re.search(r'vehicle-price__price[^>]*>\s*([^<]{1,24})', seg) + price = clean(pm.group(1)) if pm else '' + mm = re.search(r'-(\d+)(?:\?|$)', href) + vid = mm.group(1) if mm else href + cond = 'NEW' if href.startswith('/NEW-') else ('USED' if href.startswith('/USED-') else '') + # card media sits just above the heading link; keep the closest image + # that is real inventory or the model render (never logos/banners). + # The data-src URLs are HTML-escaped (/) — unescape before matching. + win = re.sub(r'[fF];', '/', raw[max(0, h.start() - 2500):h.start()]) + cands = [c for c in re.findall( + r'(?:https?:)?//cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/[^"\s|]+?\.(?:jpg|png)', + win, re.I) + if re.search(r'imglib/(?:assets/inventory|trimsdb)/', c, re.I)] + img = norm_ds_img(cands[-1]) if cands else '' + out[vid] = {'title': title, 'price': price, 'price_num': num_from_price(price), + 'cond': cond, 'url': host + href, 'img': img} + return out + + +def fetch_ds(src, pages): + got, per = {}, {} + for p in range(1, pages + 1): + url = src['tmpl'].format(p) + items = None + for att in range(2): + try: + items = parse_ds(get(url), src['host']) + if items: + break + except Exception: + items = None + time.sleep(3) + per[str(p)] = None if items is None else len(items) + if items: + got.update(items) + if not items: + break + time.sleep(SLEEP) + ok = bool(per.get('1')) + return ok, got, per + + +# ---------- State ---------- + +def state_path(name): + return os.path.join(STATE_DIR, name + '.json') + + +def load_state(name): + try: + with open(state_path(name), 'r', encoding='utf-8') as f: + return json.load(f) + except Exception: + return {} + + +def save_state(name, st): + os.makedirs(STATE_DIR, exist_ok=True) + tmp = state_path(name) + '.tmp' + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(st, f, ensure_ascii=False, indent=1) + os.replace(tmp, state_path(name)) + + +def line_for(src, vid, it): + def s(x): + return str(x if x is not None else '').replace('|', '/').strip() + + if src['kind'] == 'kijiji': + return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')), + s(it.get('title'))[:78], s(it.get('loc')), s(it.get('km'))) + if src['kind'] == 'autotrader': + return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')), + s(it.get('title'))[:78], s(it.get('city')), s(it.get('km'))) + return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')), + s(it.get('title'))[:78], s(it.get('cond')), '') + + +def run_source(src, pages_override=None): + name, kind = src['name'], src['kind'] + pages = pages_override or src['pages'] + prev = load_state(name) + if kind == 'kijiji': + ok, items, per = fetch_kijiji(src, pages) + elif kind == 'autotrader': + ok, items, per = fetch_at(src, pages) + else: + ok, items, per = fetch_ds(src, pages) + now = now_iso() + out = [] + if ok: + new_items = {} + for vid, it in items.items(): + p = (prev.get('items') or {}).get(vid) or ((prev.get('absent') or {}).get(vid) or {}).get('last') or {} + it2 = dict(it) + it2['img'] = p.get('img') or it.get('img') or '' + it2['first_seen'] = p.get('first_seen', now) + it2['last_seen'] = now + if p.get('price') is not None and it.get('price') and p.get('price') != it.get('price'): + it2['price_prev'] = p['price'] + new_items[vid] = it2 + absent = dict(prev.get('absent') or {}) + for vid in list(absent): + if vid in new_items: + del absent[vid] + for vid in (prev.get('items') or {}): + if vid not in new_items: + a = dict(absent.get(vid) or {}) + a['first_absent'] = a.get('first_absent', now) + a['times_missing'] = a.get('times_missing', 0) + 1 + a.setdefault('last', prev['items'][vid]) + absent[vid] = a + for vid in [v for v, a in absent.items() if a.get('times_missing', 0) > 60]: + del absent[vid] + partial = any(v is None or v == 0 or + (kind == 'kijiji' and v < src['min']) for v in per.values()) + st = {'source': name, 'fetched_at': now, 'ok': True, 'fail_streak': 0, + 'pages': per, 'partial': partial, 'items': new_items, 'absent': absent} + save_state(name, st) + if partial: + out.append('!partial|' + name) + for vid in sorted(new_items): + out.append(line_for(src, vid, new_items[vid])) + else: + streak = (prev.get('fail_streak') or 0) + 1 + st = dict(prev) if prev else {'source': name, 'items': {}, 'absent': {}} + st.update({'source': name, 'fetched_at': now, 'ok': False, + 'fail_streak': streak, 'pages': per}) + save_state(name, st) + if streak >= 2: + out.append('!fail|' + name) + for vid in sorted(st.get('items') or {}): + out.append(line_for(src, vid, st['items'][vid])) + return out, st + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--seed', '--full', action='store_true', help='full-depth sweep (daily coverage)') + ap.add_argument('--source', action='append', default=None) + ap.add_argument('--pages', type=int, default=None) + ap.add_argument('--verbose', action='store_true') + args = ap.parse_args() + os.makedirs(STATE_DIR, exist_ok=True) + lines = [] + for src in SOURCES: + if args.source and src['name'] not in args.source: + continue + pages = args.pages + if pages is None and args.seed: + pages = SEED_PAGES.get(src['name'], src['pages']) + t0 = time.time() + try: + block, st = run_source(src, pages_override=pages) + except Exception as e: + prev = load_state(src['name']) + streak = (prev.get('fail_streak') or 0) + 1 + prev['fail_streak'] = streak + prev['ok'] = False + prev['fetched_at'] = now_iso() + save_state(src['name'], prev) + block = (['!fail|' + src['name']] if streak >= 2 else []) + block += [line_for(src, vid, it) for vid, it in sorted((prev.get('items') or {}).items())] + st = prev + if args.verbose: + print('{}: EXCEPTION {}'.format(src['name'], e), file=sys.stderr) + lines.extend(block) + if args.verbose: + print('{}: ok={} items={} pages={} partial={} {:.0f}s'.format( + src['name'], st.get('ok'), len(st.get('items') or {}), st.get('pages'), + st.get('partial'), time.time() - t0), file=sys.stderr) + sys.stdout.write('\n'.join(lines) + ('\n' if lines else '')) + + +if __name__ == '__main__': + main()