docs: add SOURCES, DESIGN, DATA-MODEL; record v1 decisions (Kijiji source, rank-all deck, badge-only feed); vendor hunt reference modules

This commit is contained in:
AlexanderRozhenko
2026-11-10 17:48:32 -04:00
parent fc3aad1e93
commit 18bf31f361
10 changed files with 932 additions and 8 deletions

8
reference/README.md Normal file
View File

@@ -0,0 +1,8 @@
# reference/
Copies of the hunt-pipeline modules this app vendors from, taken 2026-11-10:
- `motorcycle-hunt-rank.py` — rubric tables (family tiers, extras, veto) ported into the app's scorer.
- `motorcycle-hunt-scan.py` — the Kijiji parser vendored behind the importer's `--direct` escape hatch.
Source of truth: `~/.hermes/scripts/` on the Rotarran (Alex's pipeline). If the originals change materially, refresh these copies deliberately — they are references, not live links.

View File

@@ -0,0 +1,231 @@
#!/usr/bin/env python3
"""
motorcycle-hunt-rank — score the current hunt board for Glenn's small-cruiser
search (NS + NB). Reads the state JSONs written by motorcycle-hunt-scan.py and
prints a ranked candidate list. Reusable on cron wakes.
Usage:
python3 motorcycle-hunt-rank.py [--top N] [--kind cruiser|style|all] [--src NAME]
Rubric (see the motorcycle-hunt skill for the full rationale):
base weight per model family (S = Indian Scout + Vulcan 900 Custom top tier,
T1 = strong fits, T2/style = wildcards), plus extras (MVI +6, whitewalls +5, windshield
/bags +3, spokes +3, ABS +2), mileage and year bonuses, price-fit vs the
budgets (used soft ~$5k / stretch ~$6.5k; new ~$10k / stretch ~$11.5k).
"""
import argparse
import glob
import html
import json
import os
import re
STATE = os.path.expanduser('~/.hermes/scripts/motorcycle-hunt.d')
FAMILIES = [
(r'\bscout\b', 92, 'S', 'Indian Scout — top priority (sport×cruise)'),
(r'vulcan\s*900\s*custom', 88, 'S', 'Vulcan 900 Custom — the dream spec'),
(r'vulcan\s*900', 62, 'T1', 'Vulcan 900'),
(r'vulcan\s*(classic|lt|nomad)', 58, 'T1', 'Vulcan Classic/LT'),
(r'vulcan\s*s\b', 50, 'T1', 'Vulcan S'),
(r'vulcan\s*500', 40, 'T2', 'Vulcan 500'),
(r'\bvulcan\b', 45, 'T1', 'Vulcan'),
(r'\bvolusia\b', 44, 'T1', 'Volusia (C50 predecessor)'),
(r'rebel\s*500', 50, 'T1', 'Rebel 500'),
(r'rebel\s*300', 44, 'T1', 'Rebel 300'),
(r'\brebel\b', 46, 'T1', 'Rebel'),
(r'boulevard\s*c50', 50, 'T1', 'Boulevard C50'),
(r'\bc50t?\b', 48, 'T1', 'Boulevard C50/C50T'),
(r'boulevard\s*(m50|s50)', 44, 'T1', 'Boulevard M50/S50'),
(r'boulevard\s*(c90|m90|m109)', 42, 'T2', 'Boulevard big twin'),
(r'm109r', 42, 'T2', 'Boulevard M109R'),
(r'\bboulevard\b', 40, 'T1', 'Boulevard'),
(r'\bintruder\b', 36, 'T2', 'Intruder'),
(r'shadow\s*(aero|phantom|spirit)', 48, 'T1', 'Shadow Aero/Phantom/Spirit'),
(r'shadow\s*750', 44, 'T1', 'Shadow 750'),
(r'\bshadow\b', 42, 'T1', 'Shadow'),
(r'\bsabre\b', 34, 'T2', 'Shadow Sabre'),
(r'v-?star\s*(650|950)', 46, 'T1', 'V-Star 650/950'),
(r'v-?star\s*1100', 40, 'T2', 'V-Star 1100'),
(r'v-?star', 42, 'T1', 'V-Star'),
(r'iron\s*883', 46, 'T1', 'Iron 883'),
(r'\bsportster\b', 40, 'T1', 'Sportster'),
(r'super\s*meteor', 48, 'T1', 'Super Meteor 650'),
(r'meteor\s*350', 42, 'T1', 'Meteor 350'),
(r'\bmeteor\b', 42, 'T1', 'Meteor'),
(r'hunter\s*350', 38, 'T2', 'Hunter 350'),
(r'classic\s*(350|650)', 36, 'T2', 'RE Classic'),
(r'royal\s*enfield', 34, 'T2', 'Royal Enfield'),
(r'\beliminator\b', 38, 'T2', 'Eliminator'),
(r'\bvirago\b', 34, 'T2', 'Virago'),
(r'\bmagna\b', 32, 'T2', 'Magna'),
(r'\bnightster\b', 32, 'T2', 'Nightster'),
(r'forty-?eight', 32, 'T2', 'Forty-Eight'),
(r'\bbonneville\b', 30, 'style', 'Bonneville (style-adjacent)'),
(r'\bw800\b', 28, 'style', 'W800'),
(r'z650rs', 28, 'style', 'Z650RS'),
(r'moto\s*guzzi|\bv7\b', 30, 'style', 'V7'),
(r'\bscrambler\b', 24, 'style', 'Scrambler'),
(r'\bcontinental\s*gt\b', 26, 'style', 'Continental GT'),
]
# category vetoes — not a small cruiser (dirt/sport/touring/powersports/gear)
VETO = [
(r'\b(atv|quad|side[\s-]?by[\s-]?side|utv|ranger|outlander|renegade|ryker|spyder|ski-doo|seadoo|sea-doo|snowmobile|snow\s*blower|lawn|generator|pump|watercraft|boat)\b', 'powersports'),
(r'\b(dirt\s*bike|motocross|enduro)\b', 'dirt'),
(r'\b(crf|kx\d|yz\d|rm-?z|klx|exc|xc-w|xc-f|fe\s*\d|fc\s*\d|mc\s*\d|sx\b|dr-?z|ttr|kdx)\b', 'dirt'),
(r'\b(ninja|zx-?\d|gsx-?[rs]|cbr|yzf|hayabusa|panigale|fireblade|gixxer|katana|r\d\s)\b', 'sport'),
(r'\b(v-?strom|versys|africa\s*twin|multistrada|t[eé]n[eé]r[eé]|transalp|super\s*adventure)\b', 'adv'),
(r'\b(gold\s*wing|voyager|electra\s*glide|road\s*king|ultra\s*classic|f6b)\b', 'tourer'),
(r'\b(helmet|half-?shell|jacket|glove|boots?\b|seat\b|exhaust|muffler|tire|tyre|rim\b|brake|lever|mirror|cover\b|battery|sprocket|for\s*parts|parts\b|project\s*bike|salvage|wreck)\b', 'gear'),
]
EXTRAS = [
(r'\b(mvi|safety|inspected|inspection|certified|e-?test)\b', 6, 'MVI/cert'),
(r'white\s*wall', 5, 'whitewalls'),
(r'\bspokes?\b|spoked', 3, 'spokes'),
(r'wind\s*screen|windshield', 3, 'windshield'),
(r'saddle\s*bags?|panniers?|\bbags\b', 3, 'bags'),
(r'\babs\b', 2, 'ABS'),
]
def load_sources():
out = {}
for f in sorted(glob.glob(os.path.join(STATE, '*.json'))):
name = os.path.basename(f)[:-5]
if name.endswith('.tmp'):
continue
try:
with open(f, encoding='utf-8') as fh:
out[name] = json.load(fh)
except Exception:
pass
return out
def km_num(it):
s = str(it.get('km') or '')
m = re.search(r'([\d,]+)', s)
return int(m.group(1).replace(',', '')) if m else None
def classify(title, desc):
text = (title + ' ' + desc).lower()
ttext = title.lower()
for pat, why in VETO:
if re.search(pat, ttext):
return None, None, None, why
best = None
for pat, weight, tier, label in FAMILIES:
if re.search(pat, text):
if best is None or weight > best[1]:
best = (pat, weight, tier, label)
if not best:
return None, None, None, None
return best[2], best[1], best[3], None
def score_item(src, it):
title = it.get('title') or ''
desc = it.get('desc') or ''
kind, base, label, veto = classify(title, desc)
if not kind:
return None
flags = []
s = base
text = (title + ' ' + desc).lower()
for pat, pts, fl in EXTRAS:
if re.search(pat, text):
s += pts
flags.append(fl)
price = it.get('price_num')
if price:
is_new = (it.get('cond') == 'NEW' or it.get('offer') == 'N')
if is_new:
if price <= 11500:
s += 3
elif price <= 13500:
s += 1
else:
s -= 1
flags.append('new')
else:
if price <= 5000:
s += 5
elif price <= 6500:
s += 3
elif price <= 8000:
s += 1
elif price <= 10000:
s -= 1
else:
s -= 3
else:
flags.append('no price')
km = km_num(it)
if km == 0:
km = None # 0 km on a listing is a placeholder, not delivery mileage
if km is not None:
if km < 5000:
s += 4
elif km < 15000:
s += 2
elif km > 90000:
s -= 5
elif km > 60000:
s -= 3
flags.append('{}km'.format(km))
ym = re.search(r'\b(19\d\d|20[0-2]\d)\b', title)
year = int(ym.group(1)) if ym else None
if year:
if year >= 2020:
s += 2
elif year >= 2015:
s += 1
elif year <= 2005:
s -= 1
return {'src': src, 'id': it.get('id') or '', 'kind': kind, 'label': label,
'score': s, 'price': it.get('price') or '', 'km': km, 'year': year,
'title': title, 'place': it.get('loc') or it.get('city') or '',
'url': it.get('url') or '', 'flags': flags, 'desc': desc,
'first_seen': it.get('first_seen', ''), 'prev': it.get('price_prev')}
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--top', type=int, default=80)
ap.add_argument('--kind', default='all')
ap.add_argument('--src', default=None)
args = ap.parse_args()
sources = load_sources()
rows, counts = [], {}
for src, st in sources.items():
if args.src and src != args.src:
continue
for vid, it in (st.get('items') or {}).items():
it = dict(it)
it.setdefault('id', vid)
r = score_item(src, it)
if r is None:
continue
counts[r['kind']] = counts.get(r['kind'], 0) + 1
if args.kind != 'all' and r['kind'] != args.kind:
continue
rows.append(r)
rows.sort(key=lambda r: (-r['score'], r['price'] or '$999999'))
print('=== motorcycle-hunt rank | board:', {k: v for k, v in sorted(counts.items())},
'| shown:', min(len(rows), args.top), '===')
for r in rows[:args.top]:
desc = re.sub(r'\s+', ' ', r['desc'])[:105]
print('[{}] {} | {} | {} | {} | {} | {} | km={} | {}'.format(
r['kind'], r['score'], r['label'], r['src'], r['id'], r['price'],
r['title'][:64], r['km'] if r['km'] is not None else '-', r['url'][:95]))
if desc:
print(' desc: {}'.format(desc))
print('=== families unmatched (not shown): kinds =', counts, '===')
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,448 @@
#!/usr/bin/env python3
"""
motorcycle-hunt — daily board scan for Glenn's small-cruiser hunt (NS + NB).
Sources (fixed URLs = stable diff baseline; calibrated live 2026-11-09):
kijiji-ns Kijiji motorcycles, Nova Scotia (path pagination /page-N/)
kijiji-nb Kijiji motorcycles, New Brunswick (path pagination /page-N/)
autotrader-ns AutoTrader cruiser body, NS (?sort=age&desc=1&page=N, stops at <20)
autotrader-nb AutoTrader cruiser body, NB
procycle Pro Cycle (Dartmouth NS) inventory, Dealer Spike (pg=N)
tbb Toys For Big Boys (NB) inventory, Dealer Spike (pg=N)
Prints one byte-stable line per listing:
src|id|price|title|place|km
sorted by source order then id. No timestamps in stdout — identical output
means nothing changed. A changed line set wakes the agent, which scores the
diff, updates the vault tracker, and alerts only on new-above-bar /
material price drop / genuine improvement.
State: ~/.hermes/scripts/motorcycle-hunt.d/<src>.json
items{id:{...,img?,first_seen,last_seen,price_prev?}}, absent{id:{first_absent,times_missing,last?}},
pages{page:cards}, ok, partial, fail_streak, fetched_at.
Each item may carry `img`, a hotlinkable picture URL (kijiji ?rule=kijijica-640-webp,
autotrader /720x540.jpg, dealerspike v1/640x480; a model render for new dealer
units is fine too). img never appears in stdout, so it cannot affect the change
gate; a previous img is kept on re-sight (curated pictures survive rescans —
only genuinely new items take theirs from the card).
Failure policy: per-page retries (Kijiji SSR flakes are common — retry then
flag). A failed source falls back to its last-known items so transient blocks
stay byte-stable (silent). Only at fail_streak>=2 does a `!fail|<src>` marker
appear (single wake to inform, no daily noise). `!partial|<src>` marks a day
where some pages failed or under-filled; expected cards are ~40 (Kijiji),
20 (AutoTrader), 20 (Dealer Spike).
Flags: --seed/--full (full-depth sweep; also the daily coverage), --source NAME (refresh one
source), --pages N (override depth), --verbose (summary to stderr).
"""
import argparse
import datetime
import html
import json
import os
import re
import sys
import time
import urllib.request
HOME = os.path.expanduser('~')
STATE_DIR = os.path.join(HOME, '.hermes', 'scripts', 'motorcycle-hunt.d')
UA = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
SLEEP = 2.0
KJ_NS_BASE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/c30l9002'
KJ_NS_PAGE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/page-{}/c30l9002'
KJ_NB_BASE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/c30l9005'
KJ_NB_PAGE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/page-{}/c30l9005'
AT_TMPL = ('https://www.autotrader.ca/motorcycles-atvs/lst/reg_{}/bt_cruiser'
'?sort=age&desc=1&page={}')
PC_TMPL = 'https://www.procycleonline.com/--inventory?category=motorcycle%20%2F%20scooter&pg={}'
TBB_TMPL = 'https://www.toysforbigboys.ca/--inventory?category=motorcycle+%2F+scooter&pg={}'
SOURCES = [
{'name': 'kijiji-ns', 'kind': 'kijiji', 'base': KJ_NS_BASE, 'page': KJ_NS_PAGE, 'pages': 4, 'min': 25},
{'name': 'kijiji-nb', 'kind': 'kijiji', 'base': KJ_NB_BASE, 'page': KJ_NB_PAGE, 'pages': 4, 'min': 25},
{'name': 'autotrader-ns', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'ns', 'pages': 4},
{'name': 'autotrader-nb', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'nb', 'pages': 4},
{'name': 'procycle', 'kind': 'dealerspike', 'tmpl': PC_TMPL, 'host': 'https://www.procycleonline.com', 'pages': 4},
{'name': 'tbb', 'kind': 'dealerspike', 'tmpl': TBB_TMPL, 'host': 'https://www.toysforbigboys.ca', 'pages': 4},
]
SEED_PAGES = {'kijiji-ns': 8, 'kijiji-nb': 8, 'autotrader-ns': 5,
'autotrader-nb': 5, 'procycle': 8, 'tbb': 8}
def now_iso():
return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
def clean(s, limit=None):
if s is None:
return ''
s = html.unescape(str(s))
s = re.sub(r'<[^>]+>', ' ', s)
s = re.sub(r'\s+', ' ', s).strip()
if limit and len(s) > limit:
s = s[:limit]
return s
def num_from_price(price):
if not price or '$' not in price:
return None
m = re.search(r'\d[\d,]*', price)
return int(m.group(0).replace(',', '')) if m else None
def norm_kj_img(u):
"""Kijiji CDN image URL -> 640px webp variant (token-free, hotlinkable)."""
if not u:
return ''
u = html.unescape(u)
if 'media.kijiji.ca' in u:
return u.split('?')[0] + '?rule=kijijica-640-webp'
return ''
def norm_at_img(u):
"""AutoScout24/AutoTrader picture -> 720x540 jpg variant."""
if not u:
return ''
u = html.unescape(u)
if 'pictures.autoscout24' in u:
return re.sub(r'/\d+x\d+\.(?:jpg|jpeg|webp|png)$', '/720x540.jpg', u)
return ''
def norm_ds_img(u):
"""Dealer Spike imglib -> 640x480 variant (case preserved on the path)."""
if not u:
return ''
u = html.unescape(u).replace('&#x2F;', '/')
if u.startswith('//'):
u = 'https:' + u
m = re.search(r'cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/'
r'([^"\s|]+?\.(?:jpg|png))', u, re.I)
if m:
return 'https://cdn.dealerspike.com/imglib/v1/640x480/imglib/' + m.group(1)
return ''
def get(url, timeout=35):
req = urllib.request.Request(url, headers={
'User-Agent': UA,
'Accept-Language': 'en-CA,en;q=0.9',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
})
with urllib.request.urlopen(req, timeout=timeout) as r:
data = r.read()
if len(data) < 2000:
raise ValueError('short body: %d bytes' % len(data))
return data.decode('utf-8', 'ignore')
# ---------- Kijiji ----------
def parse_kijiji(raw):
out = {}
for blk in raw.split('data-testid="listing-card"')[1:]:
blk = blk[:6000]
m = re.search(r'data-listingid="(\d+)"', blk)
if not m:
continue
vid = m.group(1)
link = re.search(r'data-testid="listing-link" href="([^"]+)"', blk)
t = re.search(r'data-testid="listing-link"[^>]*>(.*?)</a>', blk, re.S)
pr = re.search(r'data-testid="(?:autos-listing-price|listing-price)"[^>]*>(.*?)</', blk, re.S)
pre = blk.split('data-testid="listing-description"')[0]
km = re.search(r'>(\d[\d,]*)\s*km<', pre)
lo = re.search(r'data-testid="listing-location"[^>]*>(.*?)</', blk, re.S)
de = re.search(r'data-testid="listing-description"[^>]*>(.*?)</p>', blk, re.S)
imt = re.search(r'<img[^>]*data-testid="listing-card-image"[^>]*>', blk)
ims = re.search(r'src="([^"]+)"', imt.group(0)) if imt else None
price = clean(pr.group(1)) if pr else ''
out[vid] = {
'title': clean(t.group(1), 90) if t else '',
'price': price,
'price_num': num_from_price(price),
'loc': clean(lo.group(1)) if lo else '',
'km': km.group(1) if km else '',
'url': link.group(1) if link else '',
'desc': clean(de.group(1), 160) if de else '',
'img': norm_kj_img(ims.group(1)) if ims else '',
}
return out
def fetch_kijiji(src, pages):
got, per = {}, {}
for p in range(1, pages + 1):
url = src['base'] if p == 1 else src['page'].format(p)
items = None
for att in range(3):
try:
items = parse_kijiji(get(url))
except Exception:
items = None
enough = bool(items) and (len(items) >= src['min'] or (p == 1 and len(items) >= 15))
if enough:
break
if att < 2:
time.sleep(4)
per[str(p)] = None if items is None else len(items)
if items:
got.update(items)
time.sleep(SLEEP)
ok = bool(per.get('1'))
return ok, got, per
# ---------- AutoTrader ----------
def parse_at(raw):
m = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', raw, re.S)
if not m:
return None
d = json.loads(m.group(1))
pp = (d.get('props') or {}).get('pageProps') or {}
ls = pp.get('listings')
if ls is None:
return None
out = {}
for x in ls:
vid = str(x.get('id') or x.get('url') or '')
if not vid:
continue
price = x.get('price') or {}
loc = x.get('location') or {}
veh = x.get('vehicle') or {}
seller = x.get('seller') or {}
out[vid] = {
'title': clean(x.get('title'), 90),
'price': clean(price.get('priceFormatted')),
'price_num': price.get('priceRaw'),
'city': clean(loc.get('city')),
'prov': clean(loc.get('provinceCode')),
'km': clean(veh.get('mileageInKm')),
'make': clean(veh.get('make')),
'model': clean(veh.get('model')),
'year': veh.get('modelYear'),
'offer': clean(veh.get('offerType')),
'seller': clean(seller.get('companyName')),
'section': clean(x.get('searchResultSection')),
'url': x.get('url') or '',
'desc': clean(x.get('description'), 220),
'img': norm_at_img((x.get('images') or [None])[0]),
}
return out
def fetch_at(src, pages):
got, per = {}, {}
for p in range(1, pages + 1):
url = src['tmpl'].format(src['reg'], p)
items = None
for att in range(2):
try:
items = parse_at(get(url))
if items is not None:
break
except Exception:
items = None
time.sleep(3)
per[str(p)] = None if items is None else len(items)
if items:
got.update(items)
if items is not None and len(items) < 20:
break
time.sleep(SLEEP)
ok = bool(per.get('1'))
return ok, got, per
# ---------- Dealer Spike ----------
def parse_ds(raw, host):
out = {}
heads = list(re.finditer(r'vehicle-heading__link[^>]*?href="([^"]+)"[^>]*?title="([^"]*)"', raw))
for i, h in enumerate(heads):
href, title = h.group(1), clean(h.group(2), 90)
seg_end = heads[i + 1].start() if i + 1 < len(heads) else min(len(raw), h.end() + 5000)
seg = raw[h.end():seg_end]
pm = re.search(r'vehicle-price__price[^>]*>\s*([^<]{1,24})', seg)
price = clean(pm.group(1)) if pm else ''
mm = re.search(r'-(\d+)(?:\?|$)', href)
vid = mm.group(1) if mm else href
cond = 'NEW' if href.startswith('/NEW-') else ('USED' if href.startswith('/USED-') else '')
# card media sits just above the heading link; keep the closest image
# that is real inventory or the model render (never logos/banners).
# The data-src URLs are HTML-escaped (&#x2F;) — unescape before matching.
win = re.sub(r'&#x2[fF];', '/', raw[max(0, h.start() - 2500):h.start()])
cands = [c for c in re.findall(
r'(?:https?:)?//cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/[^"\s|]+?\.(?:jpg|png)',
win, re.I)
if re.search(r'imglib/(?:assets/inventory|trimsdb)/', c, re.I)]
img = norm_ds_img(cands[-1]) if cands else ''
out[vid] = {'title': title, 'price': price, 'price_num': num_from_price(price),
'cond': cond, 'url': host + href, 'img': img}
return out
def fetch_ds(src, pages):
got, per = {}, {}
for p in range(1, pages + 1):
url = src['tmpl'].format(p)
items = None
for att in range(2):
try:
items = parse_ds(get(url), src['host'])
if items:
break
except Exception:
items = None
time.sleep(3)
per[str(p)] = None if items is None else len(items)
if items:
got.update(items)
if not items:
break
time.sleep(SLEEP)
ok = bool(per.get('1'))
return ok, got, per
# ---------- State ----------
def state_path(name):
return os.path.join(STATE_DIR, name + '.json')
def load_state(name):
try:
with open(state_path(name), 'r', encoding='utf-8') as f:
return json.load(f)
except Exception:
return {}
def save_state(name, st):
os.makedirs(STATE_DIR, exist_ok=True)
tmp = state_path(name) + '.tmp'
with open(tmp, 'w', encoding='utf-8') as f:
json.dump(st, f, ensure_ascii=False, indent=1)
os.replace(tmp, state_path(name))
def line_for(src, vid, it):
def s(x):
return str(x if x is not None else '').replace('|', '/').strip()
if src['kind'] == 'kijiji':
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
s(it.get('title'))[:78], s(it.get('loc')), s(it.get('km')))
if src['kind'] == 'autotrader':
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
s(it.get('title'))[:78], s(it.get('city')), s(it.get('km')))
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
s(it.get('title'))[:78], s(it.get('cond')), '')
def run_source(src, pages_override=None):
name, kind = src['name'], src['kind']
pages = pages_override or src['pages']
prev = load_state(name)
if kind == 'kijiji':
ok, items, per = fetch_kijiji(src, pages)
elif kind == 'autotrader':
ok, items, per = fetch_at(src, pages)
else:
ok, items, per = fetch_ds(src, pages)
now = now_iso()
out = []
if ok:
new_items = {}
for vid, it in items.items():
p = (prev.get('items') or {}).get(vid) or ((prev.get('absent') or {}).get(vid) or {}).get('last') or {}
it2 = dict(it)
it2['img'] = p.get('img') or it.get('img') or ''
it2['first_seen'] = p.get('first_seen', now)
it2['last_seen'] = now
if p.get('price') is not None and it.get('price') and p.get('price') != it.get('price'):
it2['price_prev'] = p['price']
new_items[vid] = it2
absent = dict(prev.get('absent') or {})
for vid in list(absent):
if vid in new_items:
del absent[vid]
for vid in (prev.get('items') or {}):
if vid not in new_items:
a = dict(absent.get(vid) or {})
a['first_absent'] = a.get('first_absent', now)
a['times_missing'] = a.get('times_missing', 0) + 1
a.setdefault('last', prev['items'][vid])
absent[vid] = a
for vid in [v for v, a in absent.items() if a.get('times_missing', 0) > 60]:
del absent[vid]
partial = any(v is None or v == 0 or
(kind == 'kijiji' and v < src['min']) for v in per.values())
st = {'source': name, 'fetched_at': now, 'ok': True, 'fail_streak': 0,
'pages': per, 'partial': partial, 'items': new_items, 'absent': absent}
save_state(name, st)
if partial:
out.append('!partial|' + name)
for vid in sorted(new_items):
out.append(line_for(src, vid, new_items[vid]))
else:
streak = (prev.get('fail_streak') or 0) + 1
st = dict(prev) if prev else {'source': name, 'items': {}, 'absent': {}}
st.update({'source': name, 'fetched_at': now, 'ok': False,
'fail_streak': streak, 'pages': per})
save_state(name, st)
if streak >= 2:
out.append('!fail|' + name)
for vid in sorted(st.get('items') or {}):
out.append(line_for(src, vid, st['items'][vid]))
return out, st
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--seed', '--full', action='store_true', help='full-depth sweep (daily coverage)')
ap.add_argument('--source', action='append', default=None)
ap.add_argument('--pages', type=int, default=None)
ap.add_argument('--verbose', action='store_true')
args = ap.parse_args()
os.makedirs(STATE_DIR, exist_ok=True)
lines = []
for src in SOURCES:
if args.source and src['name'] not in args.source:
continue
pages = args.pages
if pages is None and args.seed:
pages = SEED_PAGES.get(src['name'], src['pages'])
t0 = time.time()
try:
block, st = run_source(src, pages_override=pages)
except Exception as e:
prev = load_state(src['name'])
streak = (prev.get('fail_streak') or 0) + 1
prev['fail_streak'] = streak
prev['ok'] = False
prev['fetched_at'] = now_iso()
save_state(src['name'], prev)
block = (['!fail|' + src['name']] if streak >= 2 else [])
block += [line_for(src, vid, it) for vid, it in sorted((prev.get('items') or {}).items())]
st = prev
if args.verbose:
print('{}: EXCEPTION {}'.format(src['name'], e), file=sys.stderr)
lines.extend(block)
if args.verbose:
print('{}: ok={} items={} pages={} partial={} {:.0f}s'.format(
src['name'], st.get('ok'), len(st.get('items') or {}), st.get('pages'),
st.get('partial'), time.time() - t0), file=sys.stderr)
sys.stdout.write('\n'.join(lines) + ('\n' if lines else ''))
if __name__ == '__main__':
main()