#!/usr/bin/env python3 """ motorcycle-hunt — daily board scan for Glenn's small-cruiser hunt (NS + NB). Sources (fixed URLs = stable diff baseline; calibrated live 2026-11-09): kijiji-ns Kijiji motorcycles, Nova Scotia (path pagination /page-N/) kijiji-nb Kijiji motorcycles, New Brunswick (path pagination /page-N/) autotrader-ns AutoTrader cruiser body, NS (?sort=age&desc=1&page=N, stops at <20) autotrader-nb AutoTrader cruiser body, NB procycle Pro Cycle (Dartmouth NS) inventory, Dealer Spike (pg=N) tbb Toys For Big Boys (NB) inventory, Dealer Spike (pg=N) Prints one byte-stable line per listing: src|id|price|title|place|km sorted by source order then id. No timestamps in stdout — identical output means nothing changed. A changed line set wakes the agent, which scores the diff, updates the vault tracker, and alerts only on new-above-bar / material price drop / genuine improvement. State: ~/.hermes/scripts/motorcycle-hunt.d/.json items{id:{...,img?,first_seen,last_seen,price_prev?}}, absent{id:{first_absent,times_missing,last?}}, pages{page:cards}, ok, partial, fail_streak, fetched_at. Each item may carry `img`, a hotlinkable picture URL (kijiji ?rule=kijijica-640-webp, autotrader /720x540.jpg, dealerspike v1/640x480; a model render for new dealer units is fine too). img never appears in stdout, so it cannot affect the change gate; a previous img is kept on re-sight (curated pictures survive rescans — only genuinely new items take theirs from the card). Failure policy: per-page retries (Kijiji SSR flakes are common — retry then flag). A failed source falls back to its last-known items so transient blocks stay byte-stable (silent). Only at fail_streak>=2 does a `!fail|` marker appear (single wake to inform, no daily noise). `!partial|` marks a day where some pages failed or under-filled; expected cards are ~40 (Kijiji), 20 (AutoTrader), 20 (Dealer Spike). Flags: --seed/--full (full-depth sweep; also the daily coverage), --source NAME (refresh one source), --pages N (override depth), --verbose (summary to stderr). """ import argparse import datetime import html import json import os import re import sys import time import urllib.request HOME = os.path.expanduser('~') STATE_DIR = os.path.join(HOME, '.hermes', 'scripts', 'motorcycle-hunt.d') UA = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36') SLEEP = 2.0 KJ_NS_BASE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/c30l9002' KJ_NS_PAGE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/page-{}/c30l9002' KJ_NB_BASE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/c30l9005' KJ_NB_PAGE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/page-{}/c30l9005' AT_TMPL = ('https://www.autotrader.ca/motorcycles-atvs/lst/reg_{}/bt_cruiser' '?sort=age&desc=1&page={}') PC_TMPL = 'https://www.procycleonline.com/--inventory?category=motorcycle%20%2F%20scooter&pg={}' TBB_TMPL = 'https://www.toysforbigboys.ca/--inventory?category=motorcycle+%2F+scooter&pg={}' SOURCES = [ {'name': 'kijiji-ns', 'kind': 'kijiji', 'base': KJ_NS_BASE, 'page': KJ_NS_PAGE, 'pages': 4, 'min': 25}, {'name': 'kijiji-nb', 'kind': 'kijiji', 'base': KJ_NB_BASE, 'page': KJ_NB_PAGE, 'pages': 4, 'min': 25}, {'name': 'autotrader-ns', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'ns', 'pages': 4}, {'name': 'autotrader-nb', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'nb', 'pages': 4}, {'name': 'procycle', 'kind': 'dealerspike', 'tmpl': PC_TMPL, 'host': 'https://www.procycleonline.com', 'pages': 4}, {'name': 'tbb', 'kind': 'dealerspike', 'tmpl': TBB_TMPL, 'host': 'https://www.toysforbigboys.ca', 'pages': 4}, ] SEED_PAGES = {'kijiji-ns': 8, 'kijiji-nb': 8, 'autotrader-ns': 5, 'autotrader-nb': 5, 'procycle': 8, 'tbb': 8} def now_iso(): return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') def clean(s, limit=None): if s is None: return '' s = html.unescape(str(s)) s = re.sub(r'<[^>]+>', ' ', s) s = re.sub(r'\s+', ' ', s).strip() if limit and len(s) > limit: s = s[:limit] return s def num_from_price(price): if not price or '$' not in price: return None m = re.search(r'\d[\d,]*', price) return int(m.group(0).replace(',', '')) if m else None def norm_kj_img(u): """Kijiji CDN image URL -> 640px webp variant (token-free, hotlinkable).""" if not u: return '' u = html.unescape(u) if 'media.kijiji.ca' in u: return u.split('?')[0] + '?rule=kijijica-640-webp' return '' def norm_at_img(u): """AutoScout24/AutoTrader picture -> 720x540 jpg variant.""" if not u: return '' u = html.unescape(u) if 'pictures.autoscout24' in u: return re.sub(r'/\d+x\d+\.(?:jpg|jpeg|webp|png)$', '/720x540.jpg', u) return '' def norm_ds_img(u): """Dealer Spike imglib -> 640x480 variant (case preserved on the path).""" if not u: return '' u = html.unescape(u).replace('/', '/') if u.startswith('//'): u = 'https:' + u m = re.search(r'cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/' r'([^"\s|]+?\.(?:jpg|png))', u, re.I) if m: return 'https://cdn.dealerspike.com/imglib/v1/640x480/imglib/' + m.group(1) return '' def get(url, timeout=35): req = urllib.request.Request(url, headers={ 'User-Agent': UA, 'Accept-Language': 'en-CA,en;q=0.9', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', }) with urllib.request.urlopen(req, timeout=timeout) as r: data = r.read() if len(data) < 2000: raise ValueError('short body: %d bytes' % len(data)) return data.decode('utf-8', 'ignore') # ---------- Kijiji ---------- def parse_kijiji(raw): out = {} for blk in raw.split('data-testid="listing-card"')[1:]: blk = blk[:6000] m = re.search(r'data-listingid="(\d+)"', blk) if not m: continue vid = m.group(1) link = re.search(r'data-testid="listing-link" href="([^"]+)"', blk) t = re.search(r'data-testid="listing-link"[^>]*>(.*?)', blk, re.S) pr = re.search(r'data-testid="(?:autos-listing-price|listing-price)"[^>]*>(.*?)(\d[\d,]*)\s*km<', pre) lo = re.search(r'data-testid="listing-location"[^>]*>(.*?)]*>(.*?)

', blk, re.S) imt = re.search(r']*data-testid="listing-card-image"[^>]*>', blk) ims = re.search(r'src="([^"]+)"', imt.group(0)) if imt else None price = clean(pr.group(1)) if pr else '' out[vid] = { 'title': clean(t.group(1), 90) if t else '', 'price': price, 'price_num': num_from_price(price), 'loc': clean(lo.group(1)) if lo else '', 'km': km.group(1) if km else '', 'url': link.group(1) if link else '', 'desc': clean(de.group(1), 160) if de else '', 'img': norm_kj_img(ims.group(1)) if ims else '', } return out def fetch_kijiji(src, pages): got, per = {}, {} for p in range(1, pages + 1): url = src['base'] if p == 1 else src['page'].format(p) items = None for att in range(3): try: items = parse_kijiji(get(url)) except Exception: items = None enough = bool(items) and (len(items) >= src['min'] or (p == 1 and len(items) >= 15)) if enough: break if att < 2: time.sleep(4) per[str(p)] = None if items is None else len(items) if items: got.update(items) time.sleep(SLEEP) ok = bool(per.get('1')) return ok, got, per # ---------- AutoTrader ---------- def parse_at(raw): m = re.search(r'', raw, re.S) if not m: return None d = json.loads(m.group(1)) pp = (d.get('props') or {}).get('pageProps') or {} ls = pp.get('listings') if ls is None: return None out = {} for x in ls: vid = str(x.get('id') or x.get('url') or '') if not vid: continue price = x.get('price') or {} loc = x.get('location') or {} veh = x.get('vehicle') or {} seller = x.get('seller') or {} out[vid] = { 'title': clean(x.get('title'), 90), 'price': clean(price.get('priceFormatted')), 'price_num': price.get('priceRaw'), 'city': clean(loc.get('city')), 'prov': clean(loc.get('provinceCode')), 'km': clean(veh.get('mileageInKm')), 'make': clean(veh.get('make')), 'model': clean(veh.get('model')), 'year': veh.get('modelYear'), 'offer': clean(veh.get('offerType')), 'seller': clean(seller.get('companyName')), 'section': clean(x.get('searchResultSection')), 'url': x.get('url') or '', 'desc': clean(x.get('description'), 220), 'img': norm_at_img((x.get('images') or [None])[0]), } return out def fetch_at(src, pages): got, per = {}, {} for p in range(1, pages + 1): url = src['tmpl'].format(src['reg'], p) items = None for att in range(2): try: items = parse_at(get(url)) if items is not None: break except Exception: items = None time.sleep(3) per[str(p)] = None if items is None else len(items) if items: got.update(items) if items is not None and len(items) < 20: break time.sleep(SLEEP) ok = bool(per.get('1')) return ok, got, per # ---------- Dealer Spike ---------- def parse_ds(raw, host): out = {} heads = list(re.finditer(r'vehicle-heading__link[^>]*?href="([^"]+)"[^>]*?title="([^"]*)"', raw)) for i, h in enumerate(heads): href, title = h.group(1), clean(h.group(2), 90) seg_end = heads[i + 1].start() if i + 1 < len(heads) else min(len(raw), h.end() + 5000) seg = raw[h.end():seg_end] pm = re.search(r'vehicle-price__price[^>]*>\s*([^<]{1,24})', seg) price = clean(pm.group(1)) if pm else '' mm = re.search(r'-(\d+)(?:\?|$)', href) vid = mm.group(1) if mm else href cond = 'NEW' if href.startswith('/NEW-') else ('USED' if href.startswith('/USED-') else '') # card media sits just above the heading link; keep the closest image # that is real inventory or the model render (never logos/banners). # The data-src URLs are HTML-escaped (/) — unescape before matching. win = re.sub(r'[fF];', '/', raw[max(0, h.start() - 2500):h.start()]) cands = [c for c in re.findall( r'(?:https?:)?//cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/[^"\s|]+?\.(?:jpg|png)', win, re.I) if re.search(r'imglib/(?:assets/inventory|trimsdb)/', c, re.I)] img = norm_ds_img(cands[-1]) if cands else '' out[vid] = {'title': title, 'price': price, 'price_num': num_from_price(price), 'cond': cond, 'url': host + href, 'img': img} return out def fetch_ds(src, pages): got, per = {}, {} for p in range(1, pages + 1): url = src['tmpl'].format(p) items = None for att in range(2): try: items = parse_ds(get(url), src['host']) if items: break except Exception: items = None time.sleep(3) per[str(p)] = None if items is None else len(items) if items: got.update(items) if not items: break time.sleep(SLEEP) ok = bool(per.get('1')) return ok, got, per # ---------- State ---------- def state_path(name): return os.path.join(STATE_DIR, name + '.json') def load_state(name): try: with open(state_path(name), 'r', encoding='utf-8') as f: return json.load(f) except Exception: return {} def save_state(name, st): os.makedirs(STATE_DIR, exist_ok=True) tmp = state_path(name) + '.tmp' with open(tmp, 'w', encoding='utf-8') as f: json.dump(st, f, ensure_ascii=False, indent=1) os.replace(tmp, state_path(name)) def line_for(src, vid, it): def s(x): return str(x if x is not None else '').replace('|', '/').strip() if src['kind'] == 'kijiji': return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')), s(it.get('title'))[:78], s(it.get('loc')), s(it.get('km'))) if src['kind'] == 'autotrader': return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')), s(it.get('title'))[:78], s(it.get('city')), s(it.get('km'))) return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')), s(it.get('title'))[:78], s(it.get('cond')), '') def run_source(src, pages_override=None): name, kind = src['name'], src['kind'] pages = pages_override or src['pages'] prev = load_state(name) if kind == 'kijiji': ok, items, per = fetch_kijiji(src, pages) elif kind == 'autotrader': ok, items, per = fetch_at(src, pages) else: ok, items, per = fetch_ds(src, pages) now = now_iso() out = [] if ok: new_items = {} for vid, it in items.items(): p = (prev.get('items') or {}).get(vid) or ((prev.get('absent') or {}).get(vid) or {}).get('last') or {} it2 = dict(it) it2['img'] = p.get('img') or it.get('img') or '' it2['first_seen'] = p.get('first_seen', now) it2['last_seen'] = now if p.get('price') is not None and it.get('price') and p.get('price') != it.get('price'): it2['price_prev'] = p['price'] new_items[vid] = it2 absent = dict(prev.get('absent') or {}) for vid in list(absent): if vid in new_items: del absent[vid] for vid in (prev.get('items') or {}): if vid not in new_items: a = dict(absent.get(vid) or {}) a['first_absent'] = a.get('first_absent', now) a['times_missing'] = a.get('times_missing', 0) + 1 a.setdefault('last', prev['items'][vid]) absent[vid] = a for vid in [v for v, a in absent.items() if a.get('times_missing', 0) > 60]: del absent[vid] partial = any(v is None or v == 0 or (kind == 'kijiji' and v < src['min']) for v in per.values()) st = {'source': name, 'fetched_at': now, 'ok': True, 'fail_streak': 0, 'pages': per, 'partial': partial, 'items': new_items, 'absent': absent} save_state(name, st) if partial: out.append('!partial|' + name) for vid in sorted(new_items): out.append(line_for(src, vid, new_items[vid])) else: streak = (prev.get('fail_streak') or 0) + 1 st = dict(prev) if prev else {'source': name, 'items': {}, 'absent': {}} st.update({'source': name, 'fetched_at': now, 'ok': False, 'fail_streak': streak, 'pages': per}) save_state(name, st) if streak >= 2: out.append('!fail|' + name) for vid in sorted(st.get('items') or {}): out.append(line_for(src, vid, st['items'][vid])) return out, st def main(): ap = argparse.ArgumentParser() ap.add_argument('--seed', '--full', action='store_true', help='full-depth sweep (daily coverage)') ap.add_argument('--source', action='append', default=None) ap.add_argument('--pages', type=int, default=None) ap.add_argument('--verbose', action='store_true') args = ap.parse_args() os.makedirs(STATE_DIR, exist_ok=True) lines = [] for src in SOURCES: if args.source and src['name'] not in args.source: continue pages = args.pages if pages is None and args.seed: pages = SEED_PAGES.get(src['name'], src['pages']) t0 = time.time() try: block, st = run_source(src, pages_override=pages) except Exception as e: prev = load_state(src['name']) streak = (prev.get('fail_streak') or 0) + 1 prev['fail_streak'] = streak prev['ok'] = False prev['fetched_at'] = now_iso() save_state(src['name'], prev) block = (['!fail|' + src['name']] if streak >= 2 else []) block += [line_for(src, vid, it) for vid, it in sorted((prev.get('items') or {}).items())] st = prev if args.verbose: print('{}: EXCEPTION {}'.format(src['name'], e), file=sys.stderr) lines.extend(block) if args.verbose: print('{}: ok={} items={} pages={} partial={} {:.0f}s'.format( src['name'], st.get('ok'), len(st.get('items') or {}), st.get('pages'), st.get('partial'), time.time() - t0), file=sys.stderr) sys.stdout.write('\n'.join(lines) + ('\n' if lines else '')) if __name__ == '__main__': main()