449 lines
17 KiB
Python
449 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
motorcycle-hunt — daily board scan for Glenn's small-cruiser hunt (NS + NB).
|
|
|
|
Sources (fixed URLs = stable diff baseline; calibrated live 2026-11-09):
|
|
kijiji-ns Kijiji motorcycles, Nova Scotia (path pagination /page-N/)
|
|
kijiji-nb Kijiji motorcycles, New Brunswick (path pagination /page-N/)
|
|
autotrader-ns AutoTrader cruiser body, NS (?sort=age&desc=1&page=N, stops at <20)
|
|
autotrader-nb AutoTrader cruiser body, NB
|
|
procycle Pro Cycle (Dartmouth NS) inventory, Dealer Spike (pg=N)
|
|
tbb Toys For Big Boys (NB) inventory, Dealer Spike (pg=N)
|
|
|
|
Prints one byte-stable line per listing:
|
|
src|id|price|title|place|km
|
|
sorted by source order then id. No timestamps in stdout — identical output
|
|
means nothing changed. A changed line set wakes the agent, which scores the
|
|
diff, updates the vault tracker, and alerts only on new-above-bar /
|
|
material price drop / genuine improvement.
|
|
|
|
State: ~/.hermes/scripts/motorcycle-hunt.d/<src>.json
|
|
items{id:{...,img?,first_seen,last_seen,price_prev?}}, absent{id:{first_absent,times_missing,last?}},
|
|
pages{page:cards}, ok, partial, fail_streak, fetched_at.
|
|
Each item may carry `img`, a hotlinkable picture URL (kijiji ?rule=kijijica-640-webp,
|
|
autotrader /720x540.jpg, dealerspike v1/640x480; a model render for new dealer
|
|
units is fine too). img never appears in stdout, so it cannot affect the change
|
|
gate; a previous img is kept on re-sight (curated pictures survive rescans —
|
|
only genuinely new items take theirs from the card).
|
|
|
|
Failure policy: per-page retries (Kijiji SSR flakes are common — retry then
|
|
flag). A failed source falls back to its last-known items so transient blocks
|
|
stay byte-stable (silent). Only at fail_streak>=2 does a `!fail|<src>` marker
|
|
appear (single wake to inform, no daily noise). `!partial|<src>` marks a day
|
|
where some pages failed or under-filled; expected cards are ~40 (Kijiji),
|
|
20 (AutoTrader), 20 (Dealer Spike).
|
|
|
|
Flags: --seed/--full (full-depth sweep; also the daily coverage), --source NAME (refresh one
|
|
source), --pages N (override depth), --verbose (summary to stderr).
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import html
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
HOME = os.path.expanduser('~')
|
|
STATE_DIR = os.path.join(HOME, '.hermes', 'scripts', 'motorcycle-hunt.d')
|
|
UA = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
|
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
|
|
SLEEP = 2.0
|
|
|
|
KJ_NS_BASE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/c30l9002'
|
|
KJ_NS_PAGE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/page-{}/c30l9002'
|
|
KJ_NB_BASE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/c30l9005'
|
|
KJ_NB_PAGE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/page-{}/c30l9005'
|
|
AT_TMPL = ('https://www.autotrader.ca/motorcycles-atvs/lst/reg_{}/bt_cruiser'
|
|
'?sort=age&desc=1&page={}')
|
|
PC_TMPL = 'https://www.procycleonline.com/--inventory?category=motorcycle%20%2F%20scooter&pg={}'
|
|
TBB_TMPL = 'https://www.toysforbigboys.ca/--inventory?category=motorcycle+%2F+scooter&pg={}'
|
|
|
|
SOURCES = [
|
|
{'name': 'kijiji-ns', 'kind': 'kijiji', 'base': KJ_NS_BASE, 'page': KJ_NS_PAGE, 'pages': 4, 'min': 25},
|
|
{'name': 'kijiji-nb', 'kind': 'kijiji', 'base': KJ_NB_BASE, 'page': KJ_NB_PAGE, 'pages': 4, 'min': 25},
|
|
{'name': 'autotrader-ns', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'ns', 'pages': 4},
|
|
{'name': 'autotrader-nb', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'nb', 'pages': 4},
|
|
{'name': 'procycle', 'kind': 'dealerspike', 'tmpl': PC_TMPL, 'host': 'https://www.procycleonline.com', 'pages': 4},
|
|
{'name': 'tbb', 'kind': 'dealerspike', 'tmpl': TBB_TMPL, 'host': 'https://www.toysforbigboys.ca', 'pages': 4},
|
|
]
|
|
SEED_PAGES = {'kijiji-ns': 8, 'kijiji-nb': 8, 'autotrader-ns': 5,
|
|
'autotrader-nb': 5, 'procycle': 8, 'tbb': 8}
|
|
|
|
|
|
def now_iso():
|
|
return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
|
|
def clean(s, limit=None):
|
|
if s is None:
|
|
return ''
|
|
s = html.unescape(str(s))
|
|
s = re.sub(r'<[^>]+>', ' ', s)
|
|
s = re.sub(r'\s+', ' ', s).strip()
|
|
if limit and len(s) > limit:
|
|
s = s[:limit]
|
|
return s
|
|
|
|
|
|
def num_from_price(price):
|
|
if not price or '$' not in price:
|
|
return None
|
|
m = re.search(r'\d[\d,]*', price)
|
|
return int(m.group(0).replace(',', '')) if m else None
|
|
|
|
|
|
def norm_kj_img(u):
|
|
"""Kijiji CDN image URL -> 640px webp variant (token-free, hotlinkable)."""
|
|
if not u:
|
|
return ''
|
|
u = html.unescape(u)
|
|
if 'media.kijiji.ca' in u:
|
|
return u.split('?')[0] + '?rule=kijijica-640-webp'
|
|
return ''
|
|
|
|
|
|
def norm_at_img(u):
|
|
"""AutoScout24/AutoTrader picture -> 720x540 jpg variant."""
|
|
if not u:
|
|
return ''
|
|
u = html.unescape(u)
|
|
if 'pictures.autoscout24' in u:
|
|
return re.sub(r'/\d+x\d+\.(?:jpg|jpeg|webp|png)$', '/720x540.jpg', u)
|
|
return ''
|
|
|
|
|
|
def norm_ds_img(u):
|
|
"""Dealer Spike imglib -> 640x480 variant (case preserved on the path)."""
|
|
if not u:
|
|
return ''
|
|
u = html.unescape(u).replace('/', '/')
|
|
if u.startswith('//'):
|
|
u = 'https:' + u
|
|
m = re.search(r'cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/'
|
|
r'([^"\s|]+?\.(?:jpg|png))', u, re.I)
|
|
if m:
|
|
return 'https://cdn.dealerspike.com/imglib/v1/640x480/imglib/' + m.group(1)
|
|
return ''
|
|
|
|
|
|
def get(url, timeout=35):
|
|
req = urllib.request.Request(url, headers={
|
|
'User-Agent': UA,
|
|
'Accept-Language': 'en-CA,en;q=0.9',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
data = r.read()
|
|
if len(data) < 2000:
|
|
raise ValueError('short body: %d bytes' % len(data))
|
|
return data.decode('utf-8', 'ignore')
|
|
|
|
|
|
# ---------- Kijiji ----------
|
|
|
|
def parse_kijiji(raw):
|
|
out = {}
|
|
for blk in raw.split('data-testid="listing-card"')[1:]:
|
|
blk = blk[:6000]
|
|
m = re.search(r'data-listingid="(\d+)"', blk)
|
|
if not m:
|
|
continue
|
|
vid = m.group(1)
|
|
link = re.search(r'data-testid="listing-link" href="([^"]+)"', blk)
|
|
t = re.search(r'data-testid="listing-link"[^>]*>(.*?)</a>', blk, re.S)
|
|
pr = re.search(r'data-testid="(?:autos-listing-price|listing-price)"[^>]*>(.*?)</', blk, re.S)
|
|
pre = blk.split('data-testid="listing-description"')[0]
|
|
km = re.search(r'>(\d[\d,]*)\s*km<', pre)
|
|
lo = re.search(r'data-testid="listing-location"[^>]*>(.*?)</', blk, re.S)
|
|
de = re.search(r'data-testid="listing-description"[^>]*>(.*?)</p>', blk, re.S)
|
|
imt = re.search(r'<img[^>]*data-testid="listing-card-image"[^>]*>', blk)
|
|
ims = re.search(r'src="([^"]+)"', imt.group(0)) if imt else None
|
|
price = clean(pr.group(1)) if pr else ''
|
|
out[vid] = {
|
|
'title': clean(t.group(1), 90) if t else '',
|
|
'price': price,
|
|
'price_num': num_from_price(price),
|
|
'loc': clean(lo.group(1)) if lo else '',
|
|
'km': km.group(1) if km else '',
|
|
'url': link.group(1) if link else '',
|
|
'desc': clean(de.group(1), 160) if de else '',
|
|
'img': norm_kj_img(ims.group(1)) if ims else '',
|
|
}
|
|
return out
|
|
|
|
|
|
def fetch_kijiji(src, pages):
|
|
got, per = {}, {}
|
|
for p in range(1, pages + 1):
|
|
url = src['base'] if p == 1 else src['page'].format(p)
|
|
items = None
|
|
for att in range(3):
|
|
try:
|
|
items = parse_kijiji(get(url))
|
|
except Exception:
|
|
items = None
|
|
enough = bool(items) and (len(items) >= src['min'] or (p == 1 and len(items) >= 15))
|
|
if enough:
|
|
break
|
|
if att < 2:
|
|
time.sleep(4)
|
|
per[str(p)] = None if items is None else len(items)
|
|
if items:
|
|
got.update(items)
|
|
time.sleep(SLEEP)
|
|
ok = bool(per.get('1'))
|
|
return ok, got, per
|
|
|
|
|
|
# ---------- AutoTrader ----------
|
|
|
|
def parse_at(raw):
|
|
m = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', raw, re.S)
|
|
if not m:
|
|
return None
|
|
d = json.loads(m.group(1))
|
|
pp = (d.get('props') or {}).get('pageProps') or {}
|
|
ls = pp.get('listings')
|
|
if ls is None:
|
|
return None
|
|
out = {}
|
|
for x in ls:
|
|
vid = str(x.get('id') or x.get('url') or '')
|
|
if not vid:
|
|
continue
|
|
price = x.get('price') or {}
|
|
loc = x.get('location') or {}
|
|
veh = x.get('vehicle') or {}
|
|
seller = x.get('seller') or {}
|
|
out[vid] = {
|
|
'title': clean(x.get('title'), 90),
|
|
'price': clean(price.get('priceFormatted')),
|
|
'price_num': price.get('priceRaw'),
|
|
'city': clean(loc.get('city')),
|
|
'prov': clean(loc.get('provinceCode')),
|
|
'km': clean(veh.get('mileageInKm')),
|
|
'make': clean(veh.get('make')),
|
|
'model': clean(veh.get('model')),
|
|
'year': veh.get('modelYear'),
|
|
'offer': clean(veh.get('offerType')),
|
|
'seller': clean(seller.get('companyName')),
|
|
'section': clean(x.get('searchResultSection')),
|
|
'url': x.get('url') or '',
|
|
'desc': clean(x.get('description'), 220),
|
|
'img': norm_at_img((x.get('images') or [None])[0]),
|
|
}
|
|
return out
|
|
|
|
|
|
def fetch_at(src, pages):
|
|
got, per = {}, {}
|
|
for p in range(1, pages + 1):
|
|
url = src['tmpl'].format(src['reg'], p)
|
|
items = None
|
|
for att in range(2):
|
|
try:
|
|
items = parse_at(get(url))
|
|
if items is not None:
|
|
break
|
|
except Exception:
|
|
items = None
|
|
time.sleep(3)
|
|
per[str(p)] = None if items is None else len(items)
|
|
if items:
|
|
got.update(items)
|
|
if items is not None and len(items) < 20:
|
|
break
|
|
time.sleep(SLEEP)
|
|
ok = bool(per.get('1'))
|
|
return ok, got, per
|
|
|
|
|
|
# ---------- Dealer Spike ----------
|
|
|
|
def parse_ds(raw, host):
|
|
out = {}
|
|
heads = list(re.finditer(r'vehicle-heading__link[^>]*?href="([^"]+)"[^>]*?title="([^"]*)"', raw))
|
|
for i, h in enumerate(heads):
|
|
href, title = h.group(1), clean(h.group(2), 90)
|
|
seg_end = heads[i + 1].start() if i + 1 < len(heads) else min(len(raw), h.end() + 5000)
|
|
seg = raw[h.end():seg_end]
|
|
pm = re.search(r'vehicle-price__price[^>]*>\s*([^<]{1,24})', seg)
|
|
price = clean(pm.group(1)) if pm else ''
|
|
mm = re.search(r'-(\d+)(?:\?|$)', href)
|
|
vid = mm.group(1) if mm else href
|
|
cond = 'NEW' if href.startswith('/NEW-') else ('USED' if href.startswith('/USED-') else '')
|
|
# card media sits just above the heading link; keep the closest image
|
|
# that is real inventory or the model render (never logos/banners).
|
|
# The data-src URLs are HTML-escaped (/) — unescape before matching.
|
|
win = re.sub(r'[fF];', '/', raw[max(0, h.start() - 2500):h.start()])
|
|
cands = [c for c in re.findall(
|
|
r'(?:https?:)?//cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/[^"\s|]+?\.(?:jpg|png)',
|
|
win, re.I)
|
|
if re.search(r'imglib/(?:assets/inventory|trimsdb)/', c, re.I)]
|
|
img = norm_ds_img(cands[-1]) if cands else ''
|
|
out[vid] = {'title': title, 'price': price, 'price_num': num_from_price(price),
|
|
'cond': cond, 'url': host + href, 'img': img}
|
|
return out
|
|
|
|
|
|
def fetch_ds(src, pages):
|
|
got, per = {}, {}
|
|
for p in range(1, pages + 1):
|
|
url = src['tmpl'].format(p)
|
|
items = None
|
|
for att in range(2):
|
|
try:
|
|
items = parse_ds(get(url), src['host'])
|
|
if items:
|
|
break
|
|
except Exception:
|
|
items = None
|
|
time.sleep(3)
|
|
per[str(p)] = None if items is None else len(items)
|
|
if items:
|
|
got.update(items)
|
|
if not items:
|
|
break
|
|
time.sleep(SLEEP)
|
|
ok = bool(per.get('1'))
|
|
return ok, got, per
|
|
|
|
|
|
# ---------- State ----------
|
|
|
|
def state_path(name):
|
|
return os.path.join(STATE_DIR, name + '.json')
|
|
|
|
|
|
def load_state(name):
|
|
try:
|
|
with open(state_path(name), 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def save_state(name, st):
|
|
os.makedirs(STATE_DIR, exist_ok=True)
|
|
tmp = state_path(name) + '.tmp'
|
|
with open(tmp, 'w', encoding='utf-8') as f:
|
|
json.dump(st, f, ensure_ascii=False, indent=1)
|
|
os.replace(tmp, state_path(name))
|
|
|
|
|
|
def line_for(src, vid, it):
|
|
def s(x):
|
|
return str(x if x is not None else '').replace('|', '/').strip()
|
|
|
|
if src['kind'] == 'kijiji':
|
|
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
|
|
s(it.get('title'))[:78], s(it.get('loc')), s(it.get('km')))
|
|
if src['kind'] == 'autotrader':
|
|
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
|
|
s(it.get('title'))[:78], s(it.get('city')), s(it.get('km')))
|
|
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
|
|
s(it.get('title'))[:78], s(it.get('cond')), '')
|
|
|
|
|
|
def run_source(src, pages_override=None):
|
|
name, kind = src['name'], src['kind']
|
|
pages = pages_override or src['pages']
|
|
prev = load_state(name)
|
|
if kind == 'kijiji':
|
|
ok, items, per = fetch_kijiji(src, pages)
|
|
elif kind == 'autotrader':
|
|
ok, items, per = fetch_at(src, pages)
|
|
else:
|
|
ok, items, per = fetch_ds(src, pages)
|
|
now = now_iso()
|
|
out = []
|
|
if ok:
|
|
new_items = {}
|
|
for vid, it in items.items():
|
|
p = (prev.get('items') or {}).get(vid) or ((prev.get('absent') or {}).get(vid) or {}).get('last') or {}
|
|
it2 = dict(it)
|
|
it2['img'] = p.get('img') or it.get('img') or ''
|
|
it2['first_seen'] = p.get('first_seen', now)
|
|
it2['last_seen'] = now
|
|
if p.get('price') is not None and it.get('price') and p.get('price') != it.get('price'):
|
|
it2['price_prev'] = p['price']
|
|
new_items[vid] = it2
|
|
absent = dict(prev.get('absent') or {})
|
|
for vid in list(absent):
|
|
if vid in new_items:
|
|
del absent[vid]
|
|
for vid in (prev.get('items') or {}):
|
|
if vid not in new_items:
|
|
a = dict(absent.get(vid) or {})
|
|
a['first_absent'] = a.get('first_absent', now)
|
|
a['times_missing'] = a.get('times_missing', 0) + 1
|
|
a.setdefault('last', prev['items'][vid])
|
|
absent[vid] = a
|
|
for vid in [v for v, a in absent.items() if a.get('times_missing', 0) > 60]:
|
|
del absent[vid]
|
|
partial = any(v is None or v == 0 or
|
|
(kind == 'kijiji' and v < src['min']) for v in per.values())
|
|
st = {'source': name, 'fetched_at': now, 'ok': True, 'fail_streak': 0,
|
|
'pages': per, 'partial': partial, 'items': new_items, 'absent': absent}
|
|
save_state(name, st)
|
|
if partial:
|
|
out.append('!partial|' + name)
|
|
for vid in sorted(new_items):
|
|
out.append(line_for(src, vid, new_items[vid]))
|
|
else:
|
|
streak = (prev.get('fail_streak') or 0) + 1
|
|
st = dict(prev) if prev else {'source': name, 'items': {}, 'absent': {}}
|
|
st.update({'source': name, 'fetched_at': now, 'ok': False,
|
|
'fail_streak': streak, 'pages': per})
|
|
save_state(name, st)
|
|
if streak >= 2:
|
|
out.append('!fail|' + name)
|
|
for vid in sorted(st.get('items') or {}):
|
|
out.append(line_for(src, vid, st['items'][vid]))
|
|
return out, st
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--seed', '--full', action='store_true', help='full-depth sweep (daily coverage)')
|
|
ap.add_argument('--source', action='append', default=None)
|
|
ap.add_argument('--pages', type=int, default=None)
|
|
ap.add_argument('--verbose', action='store_true')
|
|
args = ap.parse_args()
|
|
os.makedirs(STATE_DIR, exist_ok=True)
|
|
lines = []
|
|
for src in SOURCES:
|
|
if args.source and src['name'] not in args.source:
|
|
continue
|
|
pages = args.pages
|
|
if pages is None and args.seed:
|
|
pages = SEED_PAGES.get(src['name'], src['pages'])
|
|
t0 = time.time()
|
|
try:
|
|
block, st = run_source(src, pages_override=pages)
|
|
except Exception as e:
|
|
prev = load_state(src['name'])
|
|
streak = (prev.get('fail_streak') or 0) + 1
|
|
prev['fail_streak'] = streak
|
|
prev['ok'] = False
|
|
prev['fetched_at'] = now_iso()
|
|
save_state(src['name'], prev)
|
|
block = (['!fail|' + src['name']] if streak >= 2 else [])
|
|
block += [line_for(src, vid, it) for vid, it in sorted((prev.get('items') or {}).items())]
|
|
st = prev
|
|
if args.verbose:
|
|
print('{}: EXCEPTION {}'.format(src['name'], e), file=sys.stderr)
|
|
lines.extend(block)
|
|
if args.verbose:
|
|
print('{}: ok={} items={} pages={} partial={} {:.0f}s'.format(
|
|
src['name'], st.get('ok'), len(st.get('items') or {}), st.get('pages'),
|
|
st.get('partial'), time.time() - t0), file=sys.stderr)
|
|
sys.stdout.write('\n'.join(lines) + ('\n' if lines else ''))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|