docs: add SOURCES, DESIGN, DATA-MODEL; record v1 decisions (Kijiji source, rank-all deck, badge-only feed); vendor hunt reference modules
This commit is contained in:
8
reference/README.md
Normal file
8
reference/README.md
Normal file
@@ -0,0 +1,8 @@
|
||||
# reference/
|
||||
|
||||
Copies of the hunt-pipeline modules this app vendors from, taken 2026-11-10:
|
||||
|
||||
- `motorcycle-hunt-rank.py` — rubric tables (family tiers, extras, veto) ported into the app's scorer.
|
||||
- `motorcycle-hunt-scan.py` — the Kijiji parser vendored behind the importer's `--direct` escape hatch.
|
||||
|
||||
Source of truth: `~/.hermes/scripts/` on the Rotarran (Alex's pipeline). If the originals change materially, refresh these copies deliberately — they are references, not live links.
|
||||
231
reference/motorcycle-hunt-rank.py
Normal file
231
reference/motorcycle-hunt-rank.py
Normal file
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
motorcycle-hunt-rank — score the current hunt board for Glenn's small-cruiser
|
||||
search (NS + NB). Reads the state JSONs written by motorcycle-hunt-scan.py and
|
||||
prints a ranked candidate list. Reusable on cron wakes.
|
||||
|
||||
Usage:
|
||||
python3 motorcycle-hunt-rank.py [--top N] [--kind cruiser|style|all] [--src NAME]
|
||||
|
||||
Rubric (see the motorcycle-hunt skill for the full rationale):
|
||||
base weight per model family (S = Indian Scout + Vulcan 900 Custom top tier,
|
||||
T1 = strong fits, T2/style = wildcards), plus extras (MVI +6, whitewalls +5, windshield
|
||||
/bags +3, spokes +3, ABS +2), mileage and year bonuses, price-fit vs the
|
||||
budgets (used soft ~$5k / stretch ~$6.5k; new ~$10k / stretch ~$11.5k).
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
STATE = os.path.expanduser('~/.hermes/scripts/motorcycle-hunt.d')
|
||||
|
||||
FAMILIES = [
|
||||
(r'\bscout\b', 92, 'S', 'Indian Scout — top priority (sport×cruise)'),
|
||||
(r'vulcan\s*900\s*custom', 88, 'S', 'Vulcan 900 Custom — the dream spec'),
|
||||
(r'vulcan\s*900', 62, 'T1', 'Vulcan 900'),
|
||||
(r'vulcan\s*(classic|lt|nomad)', 58, 'T1', 'Vulcan Classic/LT'),
|
||||
(r'vulcan\s*s\b', 50, 'T1', 'Vulcan S'),
|
||||
(r'vulcan\s*500', 40, 'T2', 'Vulcan 500'),
|
||||
(r'\bvulcan\b', 45, 'T1', 'Vulcan'),
|
||||
(r'\bvolusia\b', 44, 'T1', 'Volusia (C50 predecessor)'),
|
||||
(r'rebel\s*500', 50, 'T1', 'Rebel 500'),
|
||||
(r'rebel\s*300', 44, 'T1', 'Rebel 300'),
|
||||
(r'\brebel\b', 46, 'T1', 'Rebel'),
|
||||
(r'boulevard\s*c50', 50, 'T1', 'Boulevard C50'),
|
||||
(r'\bc50t?\b', 48, 'T1', 'Boulevard C50/C50T'),
|
||||
(r'boulevard\s*(m50|s50)', 44, 'T1', 'Boulevard M50/S50'),
|
||||
(r'boulevard\s*(c90|m90|m109)', 42, 'T2', 'Boulevard big twin'),
|
||||
(r'm109r', 42, 'T2', 'Boulevard M109R'),
|
||||
(r'\bboulevard\b', 40, 'T1', 'Boulevard'),
|
||||
(r'\bintruder\b', 36, 'T2', 'Intruder'),
|
||||
(r'shadow\s*(aero|phantom|spirit)', 48, 'T1', 'Shadow Aero/Phantom/Spirit'),
|
||||
(r'shadow\s*750', 44, 'T1', 'Shadow 750'),
|
||||
(r'\bshadow\b', 42, 'T1', 'Shadow'),
|
||||
(r'\bsabre\b', 34, 'T2', 'Shadow Sabre'),
|
||||
(r'v-?star\s*(650|950)', 46, 'T1', 'V-Star 650/950'),
|
||||
(r'v-?star\s*1100', 40, 'T2', 'V-Star 1100'),
|
||||
(r'v-?star', 42, 'T1', 'V-Star'),
|
||||
(r'iron\s*883', 46, 'T1', 'Iron 883'),
|
||||
(r'\bsportster\b', 40, 'T1', 'Sportster'),
|
||||
(r'super\s*meteor', 48, 'T1', 'Super Meteor 650'),
|
||||
(r'meteor\s*350', 42, 'T1', 'Meteor 350'),
|
||||
(r'\bmeteor\b', 42, 'T1', 'Meteor'),
|
||||
(r'hunter\s*350', 38, 'T2', 'Hunter 350'),
|
||||
(r'classic\s*(350|650)', 36, 'T2', 'RE Classic'),
|
||||
(r'royal\s*enfield', 34, 'T2', 'Royal Enfield'),
|
||||
(r'\beliminator\b', 38, 'T2', 'Eliminator'),
|
||||
(r'\bvirago\b', 34, 'T2', 'Virago'),
|
||||
(r'\bmagna\b', 32, 'T2', 'Magna'),
|
||||
(r'\bnightster\b', 32, 'T2', 'Nightster'),
|
||||
(r'forty-?eight', 32, 'T2', 'Forty-Eight'),
|
||||
(r'\bbonneville\b', 30, 'style', 'Bonneville (style-adjacent)'),
|
||||
(r'\bw800\b', 28, 'style', 'W800'),
|
||||
(r'z650rs', 28, 'style', 'Z650RS'),
|
||||
(r'moto\s*guzzi|\bv7\b', 30, 'style', 'V7'),
|
||||
(r'\bscrambler\b', 24, 'style', 'Scrambler'),
|
||||
(r'\bcontinental\s*gt\b', 26, 'style', 'Continental GT'),
|
||||
]
|
||||
|
||||
# category vetoes — not a small cruiser (dirt/sport/touring/powersports/gear)
|
||||
VETO = [
|
||||
(r'\b(atv|quad|side[\s-]?by[\s-]?side|utv|ranger|outlander|renegade|ryker|spyder|ski-doo|seadoo|sea-doo|snowmobile|snow\s*blower|lawn|generator|pump|watercraft|boat)\b', 'powersports'),
|
||||
(r'\b(dirt\s*bike|motocross|enduro)\b', 'dirt'),
|
||||
(r'\b(crf|kx\d|yz\d|rm-?z|klx|exc|xc-w|xc-f|fe\s*\d|fc\s*\d|mc\s*\d|sx\b|dr-?z|ttr|kdx)\b', 'dirt'),
|
||||
(r'\b(ninja|zx-?\d|gsx-?[rs]|cbr|yzf|hayabusa|panigale|fireblade|gixxer|katana|r\d\s)\b', 'sport'),
|
||||
(r'\b(v-?strom|versys|africa\s*twin|multistrada|t[eé]n[eé]r[eé]|transalp|super\s*adventure)\b', 'adv'),
|
||||
(r'\b(gold\s*wing|voyager|electra\s*glide|road\s*king|ultra\s*classic|f6b)\b', 'tourer'),
|
||||
(r'\b(helmet|half-?shell|jacket|glove|boots?\b|seat\b|exhaust|muffler|tire|tyre|rim\b|brake|lever|mirror|cover\b|battery|sprocket|for\s*parts|parts\b|project\s*bike|salvage|wreck)\b', 'gear'),
|
||||
]
|
||||
|
||||
EXTRAS = [
|
||||
(r'\b(mvi|safety|inspected|inspection|certified|e-?test)\b', 6, 'MVI/cert'),
|
||||
(r'white\s*wall', 5, 'whitewalls'),
|
||||
(r'\bspokes?\b|spoked', 3, 'spokes'),
|
||||
(r'wind\s*screen|windshield', 3, 'windshield'),
|
||||
(r'saddle\s*bags?|panniers?|\bbags\b', 3, 'bags'),
|
||||
(r'\babs\b', 2, 'ABS'),
|
||||
]
|
||||
|
||||
|
||||
def load_sources():
|
||||
out = {}
|
||||
for f in sorted(glob.glob(os.path.join(STATE, '*.json'))):
|
||||
name = os.path.basename(f)[:-5]
|
||||
if name.endswith('.tmp'):
|
||||
continue
|
||||
try:
|
||||
with open(f, encoding='utf-8') as fh:
|
||||
out[name] = json.load(fh)
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def km_num(it):
|
||||
s = str(it.get('km') or '')
|
||||
m = re.search(r'([\d,]+)', s)
|
||||
return int(m.group(1).replace(',', '')) if m else None
|
||||
|
||||
|
||||
def classify(title, desc):
|
||||
text = (title + ' ' + desc).lower()
|
||||
ttext = title.lower()
|
||||
for pat, why in VETO:
|
||||
if re.search(pat, ttext):
|
||||
return None, None, None, why
|
||||
best = None
|
||||
for pat, weight, tier, label in FAMILIES:
|
||||
if re.search(pat, text):
|
||||
if best is None or weight > best[1]:
|
||||
best = (pat, weight, tier, label)
|
||||
if not best:
|
||||
return None, None, None, None
|
||||
return best[2], best[1], best[3], None
|
||||
|
||||
|
||||
def score_item(src, it):
|
||||
title = it.get('title') or ''
|
||||
desc = it.get('desc') or ''
|
||||
kind, base, label, veto = classify(title, desc)
|
||||
if not kind:
|
||||
return None
|
||||
flags = []
|
||||
s = base
|
||||
text = (title + ' ' + desc).lower()
|
||||
for pat, pts, fl in EXTRAS:
|
||||
if re.search(pat, text):
|
||||
s += pts
|
||||
flags.append(fl)
|
||||
price = it.get('price_num')
|
||||
if price:
|
||||
is_new = (it.get('cond') == 'NEW' or it.get('offer') == 'N')
|
||||
if is_new:
|
||||
if price <= 11500:
|
||||
s += 3
|
||||
elif price <= 13500:
|
||||
s += 1
|
||||
else:
|
||||
s -= 1
|
||||
flags.append('new')
|
||||
else:
|
||||
if price <= 5000:
|
||||
s += 5
|
||||
elif price <= 6500:
|
||||
s += 3
|
||||
elif price <= 8000:
|
||||
s += 1
|
||||
elif price <= 10000:
|
||||
s -= 1
|
||||
else:
|
||||
s -= 3
|
||||
else:
|
||||
flags.append('no price')
|
||||
km = km_num(it)
|
||||
if km == 0:
|
||||
km = None # 0 km on a listing is a placeholder, not delivery mileage
|
||||
if km is not None:
|
||||
if km < 5000:
|
||||
s += 4
|
||||
elif km < 15000:
|
||||
s += 2
|
||||
elif km > 90000:
|
||||
s -= 5
|
||||
elif km > 60000:
|
||||
s -= 3
|
||||
flags.append('{}km'.format(km))
|
||||
ym = re.search(r'\b(19\d\d|20[0-2]\d)\b', title)
|
||||
year = int(ym.group(1)) if ym else None
|
||||
if year:
|
||||
if year >= 2020:
|
||||
s += 2
|
||||
elif year >= 2015:
|
||||
s += 1
|
||||
elif year <= 2005:
|
||||
s -= 1
|
||||
return {'src': src, 'id': it.get('id') or '', 'kind': kind, 'label': label,
|
||||
'score': s, 'price': it.get('price') or '', 'km': km, 'year': year,
|
||||
'title': title, 'place': it.get('loc') or it.get('city') or '',
|
||||
'url': it.get('url') or '', 'flags': flags, 'desc': desc,
|
||||
'first_seen': it.get('first_seen', ''), 'prev': it.get('price_prev')}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--top', type=int, default=80)
|
||||
ap.add_argument('--kind', default='all')
|
||||
ap.add_argument('--src', default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
sources = load_sources()
|
||||
rows, counts = [], {}
|
||||
for src, st in sources.items():
|
||||
if args.src and src != args.src:
|
||||
continue
|
||||
for vid, it in (st.get('items') or {}).items():
|
||||
it = dict(it)
|
||||
it.setdefault('id', vid)
|
||||
r = score_item(src, it)
|
||||
if r is None:
|
||||
continue
|
||||
counts[r['kind']] = counts.get(r['kind'], 0) + 1
|
||||
if args.kind != 'all' and r['kind'] != args.kind:
|
||||
continue
|
||||
rows.append(r)
|
||||
rows.sort(key=lambda r: (-r['score'], r['price'] or '$999999'))
|
||||
print('=== motorcycle-hunt rank | board:', {k: v for k, v in sorted(counts.items())},
|
||||
'| shown:', min(len(rows), args.top), '===')
|
||||
for r in rows[:args.top]:
|
||||
desc = re.sub(r'\s+', ' ', r['desc'])[:105]
|
||||
print('[{}] {} | {} | {} | {} | {} | {} | km={} | {}'.format(
|
||||
r['kind'], r['score'], r['label'], r['src'], r['id'], r['price'],
|
||||
r['title'][:64], r['km'] if r['km'] is not None else '-', r['url'][:95]))
|
||||
if desc:
|
||||
print(' desc: {}'.format(desc))
|
||||
print('=== families unmatched (not shown): kinds =', counts, '===')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
448
reference/motorcycle-hunt-scan.py
Normal file
448
reference/motorcycle-hunt-scan.py
Normal file
@@ -0,0 +1,448 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
motorcycle-hunt — daily board scan for Glenn's small-cruiser hunt (NS + NB).
|
||||
|
||||
Sources (fixed URLs = stable diff baseline; calibrated live 2026-11-09):
|
||||
kijiji-ns Kijiji motorcycles, Nova Scotia (path pagination /page-N/)
|
||||
kijiji-nb Kijiji motorcycles, New Brunswick (path pagination /page-N/)
|
||||
autotrader-ns AutoTrader cruiser body, NS (?sort=age&desc=1&page=N, stops at <20)
|
||||
autotrader-nb AutoTrader cruiser body, NB
|
||||
procycle Pro Cycle (Dartmouth NS) inventory, Dealer Spike (pg=N)
|
||||
tbb Toys For Big Boys (NB) inventory, Dealer Spike (pg=N)
|
||||
|
||||
Prints one byte-stable line per listing:
|
||||
src|id|price|title|place|km
|
||||
sorted by source order then id. No timestamps in stdout — identical output
|
||||
means nothing changed. A changed line set wakes the agent, which scores the
|
||||
diff, updates the vault tracker, and alerts only on new-above-bar /
|
||||
material price drop / genuine improvement.
|
||||
|
||||
State: ~/.hermes/scripts/motorcycle-hunt.d/<src>.json
|
||||
items{id:{...,img?,first_seen,last_seen,price_prev?}}, absent{id:{first_absent,times_missing,last?}},
|
||||
pages{page:cards}, ok, partial, fail_streak, fetched_at.
|
||||
Each item may carry `img`, a hotlinkable picture URL (kijiji ?rule=kijijica-640-webp,
|
||||
autotrader /720x540.jpg, dealerspike v1/640x480; a model render for new dealer
|
||||
units is fine too). img never appears in stdout, so it cannot affect the change
|
||||
gate; a previous img is kept on re-sight (curated pictures survive rescans —
|
||||
only genuinely new items take theirs from the card).
|
||||
|
||||
Failure policy: per-page retries (Kijiji SSR flakes are common — retry then
|
||||
flag). A failed source falls back to its last-known items so transient blocks
|
||||
stay byte-stable (silent). Only at fail_streak>=2 does a `!fail|<src>` marker
|
||||
appear (single wake to inform, no daily noise). `!partial|<src>` marks a day
|
||||
where some pages failed or under-filled; expected cards are ~40 (Kijiji),
|
||||
20 (AutoTrader), 20 (Dealer Spike).
|
||||
|
||||
Flags: --seed/--full (full-depth sweep; also the daily coverage), --source NAME (refresh one
|
||||
source), --pages N (override depth), --verbose (summary to stderr).
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
HOME = os.path.expanduser('~')
|
||||
STATE_DIR = os.path.join(HOME, '.hermes', 'scripts', 'motorcycle-hunt.d')
|
||||
UA = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
|
||||
SLEEP = 2.0
|
||||
|
||||
KJ_NS_BASE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/c30l9002'
|
||||
KJ_NS_PAGE = 'https://www.kijiji.ca/b-motorcycles/nova-scotia/page-{}/c30l9002'
|
||||
KJ_NB_BASE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/c30l9005'
|
||||
KJ_NB_PAGE = 'https://www.kijiji.ca/b-motorcycles/new-brunswick/page-{}/c30l9005'
|
||||
AT_TMPL = ('https://www.autotrader.ca/motorcycles-atvs/lst/reg_{}/bt_cruiser'
|
||||
'?sort=age&desc=1&page={}')
|
||||
PC_TMPL = 'https://www.procycleonline.com/--inventory?category=motorcycle%20%2F%20scooter&pg={}'
|
||||
TBB_TMPL = 'https://www.toysforbigboys.ca/--inventory?category=motorcycle+%2F+scooter&pg={}'
|
||||
|
||||
SOURCES = [
|
||||
{'name': 'kijiji-ns', 'kind': 'kijiji', 'base': KJ_NS_BASE, 'page': KJ_NS_PAGE, 'pages': 4, 'min': 25},
|
||||
{'name': 'kijiji-nb', 'kind': 'kijiji', 'base': KJ_NB_BASE, 'page': KJ_NB_PAGE, 'pages': 4, 'min': 25},
|
||||
{'name': 'autotrader-ns', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'ns', 'pages': 4},
|
||||
{'name': 'autotrader-nb', 'kind': 'autotrader', 'tmpl': AT_TMPL, 'reg': 'nb', 'pages': 4},
|
||||
{'name': 'procycle', 'kind': 'dealerspike', 'tmpl': PC_TMPL, 'host': 'https://www.procycleonline.com', 'pages': 4},
|
||||
{'name': 'tbb', 'kind': 'dealerspike', 'tmpl': TBB_TMPL, 'host': 'https://www.toysforbigboys.ca', 'pages': 4},
|
||||
]
|
||||
SEED_PAGES = {'kijiji-ns': 8, 'kijiji-nb': 8, 'autotrader-ns': 5,
|
||||
'autotrader-nb': 5, 'procycle': 8, 'tbb': 8}
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
|
||||
def clean(s, limit=None):
|
||||
if s is None:
|
||||
return ''
|
||||
s = html.unescape(str(s))
|
||||
s = re.sub(r'<[^>]+>', ' ', s)
|
||||
s = re.sub(r'\s+', ' ', s).strip()
|
||||
if limit and len(s) > limit:
|
||||
s = s[:limit]
|
||||
return s
|
||||
|
||||
|
||||
def num_from_price(price):
|
||||
if not price or '$' not in price:
|
||||
return None
|
||||
m = re.search(r'\d[\d,]*', price)
|
||||
return int(m.group(0).replace(',', '')) if m else None
|
||||
|
||||
|
||||
def norm_kj_img(u):
|
||||
"""Kijiji CDN image URL -> 640px webp variant (token-free, hotlinkable)."""
|
||||
if not u:
|
||||
return ''
|
||||
u = html.unescape(u)
|
||||
if 'media.kijiji.ca' in u:
|
||||
return u.split('?')[0] + '?rule=kijijica-640-webp'
|
||||
return ''
|
||||
|
||||
|
||||
def norm_at_img(u):
|
||||
"""AutoScout24/AutoTrader picture -> 720x540 jpg variant."""
|
||||
if not u:
|
||||
return ''
|
||||
u = html.unescape(u)
|
||||
if 'pictures.autoscout24' in u:
|
||||
return re.sub(r'/\d+x\d+\.(?:jpg|jpeg|webp|png)$', '/720x540.jpg', u)
|
||||
return ''
|
||||
|
||||
|
||||
def norm_ds_img(u):
|
||||
"""Dealer Spike imglib -> 640x480 variant (case preserved on the path)."""
|
||||
if not u:
|
||||
return ''
|
||||
u = html.unescape(u).replace('/', '/')
|
||||
if u.startswith('//'):
|
||||
u = 'https:' + u
|
||||
m = re.search(r'cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/'
|
||||
r'([^"\s|]+?\.(?:jpg|png))', u, re.I)
|
||||
if m:
|
||||
return 'https://cdn.dealerspike.com/imglib/v1/640x480/imglib/' + m.group(1)
|
||||
return ''
|
||||
|
||||
|
||||
def get(url, timeout=35):
|
||||
req = urllib.request.Request(url, headers={
|
||||
'User-Agent': UA,
|
||||
'Accept-Language': 'en-CA,en;q=0.9',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
data = r.read()
|
||||
if len(data) < 2000:
|
||||
raise ValueError('short body: %d bytes' % len(data))
|
||||
return data.decode('utf-8', 'ignore')
|
||||
|
||||
|
||||
# ---------- Kijiji ----------
|
||||
|
||||
def parse_kijiji(raw):
|
||||
out = {}
|
||||
for blk in raw.split('data-testid="listing-card"')[1:]:
|
||||
blk = blk[:6000]
|
||||
m = re.search(r'data-listingid="(\d+)"', blk)
|
||||
if not m:
|
||||
continue
|
||||
vid = m.group(1)
|
||||
link = re.search(r'data-testid="listing-link" href="([^"]+)"', blk)
|
||||
t = re.search(r'data-testid="listing-link"[^>]*>(.*?)</a>', blk, re.S)
|
||||
pr = re.search(r'data-testid="(?:autos-listing-price|listing-price)"[^>]*>(.*?)</', blk, re.S)
|
||||
pre = blk.split('data-testid="listing-description"')[0]
|
||||
km = re.search(r'>(\d[\d,]*)\s*km<', pre)
|
||||
lo = re.search(r'data-testid="listing-location"[^>]*>(.*?)</', blk, re.S)
|
||||
de = re.search(r'data-testid="listing-description"[^>]*>(.*?)</p>', blk, re.S)
|
||||
imt = re.search(r'<img[^>]*data-testid="listing-card-image"[^>]*>', blk)
|
||||
ims = re.search(r'src="([^"]+)"', imt.group(0)) if imt else None
|
||||
price = clean(pr.group(1)) if pr else ''
|
||||
out[vid] = {
|
||||
'title': clean(t.group(1), 90) if t else '',
|
||||
'price': price,
|
||||
'price_num': num_from_price(price),
|
||||
'loc': clean(lo.group(1)) if lo else '',
|
||||
'km': km.group(1) if km else '',
|
||||
'url': link.group(1) if link else '',
|
||||
'desc': clean(de.group(1), 160) if de else '',
|
||||
'img': norm_kj_img(ims.group(1)) if ims else '',
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def fetch_kijiji(src, pages):
|
||||
got, per = {}, {}
|
||||
for p in range(1, pages + 1):
|
||||
url = src['base'] if p == 1 else src['page'].format(p)
|
||||
items = None
|
||||
for att in range(3):
|
||||
try:
|
||||
items = parse_kijiji(get(url))
|
||||
except Exception:
|
||||
items = None
|
||||
enough = bool(items) and (len(items) >= src['min'] or (p == 1 and len(items) >= 15))
|
||||
if enough:
|
||||
break
|
||||
if att < 2:
|
||||
time.sleep(4)
|
||||
per[str(p)] = None if items is None else len(items)
|
||||
if items:
|
||||
got.update(items)
|
||||
time.sleep(SLEEP)
|
||||
ok = bool(per.get('1'))
|
||||
return ok, got, per
|
||||
|
||||
|
||||
# ---------- AutoTrader ----------
|
||||
|
||||
def parse_at(raw):
|
||||
m = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', raw, re.S)
|
||||
if not m:
|
||||
return None
|
||||
d = json.loads(m.group(1))
|
||||
pp = (d.get('props') or {}).get('pageProps') or {}
|
||||
ls = pp.get('listings')
|
||||
if ls is None:
|
||||
return None
|
||||
out = {}
|
||||
for x in ls:
|
||||
vid = str(x.get('id') or x.get('url') or '')
|
||||
if not vid:
|
||||
continue
|
||||
price = x.get('price') or {}
|
||||
loc = x.get('location') or {}
|
||||
veh = x.get('vehicle') or {}
|
||||
seller = x.get('seller') or {}
|
||||
out[vid] = {
|
||||
'title': clean(x.get('title'), 90),
|
||||
'price': clean(price.get('priceFormatted')),
|
||||
'price_num': price.get('priceRaw'),
|
||||
'city': clean(loc.get('city')),
|
||||
'prov': clean(loc.get('provinceCode')),
|
||||
'km': clean(veh.get('mileageInKm')),
|
||||
'make': clean(veh.get('make')),
|
||||
'model': clean(veh.get('model')),
|
||||
'year': veh.get('modelYear'),
|
||||
'offer': clean(veh.get('offerType')),
|
||||
'seller': clean(seller.get('companyName')),
|
||||
'section': clean(x.get('searchResultSection')),
|
||||
'url': x.get('url') or '',
|
||||
'desc': clean(x.get('description'), 220),
|
||||
'img': norm_at_img((x.get('images') or [None])[0]),
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def fetch_at(src, pages):
|
||||
got, per = {}, {}
|
||||
for p in range(1, pages + 1):
|
||||
url = src['tmpl'].format(src['reg'], p)
|
||||
items = None
|
||||
for att in range(2):
|
||||
try:
|
||||
items = parse_at(get(url))
|
||||
if items is not None:
|
||||
break
|
||||
except Exception:
|
||||
items = None
|
||||
time.sleep(3)
|
||||
per[str(p)] = None if items is None else len(items)
|
||||
if items:
|
||||
got.update(items)
|
||||
if items is not None and len(items) < 20:
|
||||
break
|
||||
time.sleep(SLEEP)
|
||||
ok = bool(per.get('1'))
|
||||
return ok, got, per
|
||||
|
||||
|
||||
# ---------- Dealer Spike ----------
|
||||
|
||||
def parse_ds(raw, host):
|
||||
out = {}
|
||||
heads = list(re.finditer(r'vehicle-heading__link[^>]*?href="([^"]+)"[^>]*?title="([^"]*)"', raw))
|
||||
for i, h in enumerate(heads):
|
||||
href, title = h.group(1), clean(h.group(2), 90)
|
||||
seg_end = heads[i + 1].start() if i + 1 < len(heads) else min(len(raw), h.end() + 5000)
|
||||
seg = raw[h.end():seg_end]
|
||||
pm = re.search(r'vehicle-price__price[^>]*>\s*([^<]{1,24})', seg)
|
||||
price = clean(pm.group(1)) if pm else ''
|
||||
mm = re.search(r'-(\d+)(?:\?|$)', href)
|
||||
vid = mm.group(1) if mm else href
|
||||
cond = 'NEW' if href.startswith('/NEW-') else ('USED' if href.startswith('/USED-') else '')
|
||||
# card media sits just above the heading link; keep the closest image
|
||||
# that is real inventory or the model render (never logos/banners).
|
||||
# The data-src URLs are HTML-escaped (/) — unescape before matching.
|
||||
win = re.sub(r'[fF];', '/', raw[max(0, h.start() - 2500):h.start()])
|
||||
cands = [c for c in re.findall(
|
||||
r'(?:https?:)?//cdn\.dealerspike\.com/imglib/v1/\d+x\d+/imglib/[^"\s|]+?\.(?:jpg|png)',
|
||||
win, re.I)
|
||||
if re.search(r'imglib/(?:assets/inventory|trimsdb)/', c, re.I)]
|
||||
img = norm_ds_img(cands[-1]) if cands else ''
|
||||
out[vid] = {'title': title, 'price': price, 'price_num': num_from_price(price),
|
||||
'cond': cond, 'url': host + href, 'img': img}
|
||||
return out
|
||||
|
||||
|
||||
def fetch_ds(src, pages):
|
||||
got, per = {}, {}
|
||||
for p in range(1, pages + 1):
|
||||
url = src['tmpl'].format(p)
|
||||
items = None
|
||||
for att in range(2):
|
||||
try:
|
||||
items = parse_ds(get(url), src['host'])
|
||||
if items:
|
||||
break
|
||||
except Exception:
|
||||
items = None
|
||||
time.sleep(3)
|
||||
per[str(p)] = None if items is None else len(items)
|
||||
if items:
|
||||
got.update(items)
|
||||
if not items:
|
||||
break
|
||||
time.sleep(SLEEP)
|
||||
ok = bool(per.get('1'))
|
||||
return ok, got, per
|
||||
|
||||
|
||||
# ---------- State ----------
|
||||
|
||||
def state_path(name):
|
||||
return os.path.join(STATE_DIR, name + '.json')
|
||||
|
||||
|
||||
def load_state(name):
|
||||
try:
|
||||
with open(state_path(name), 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(name, st):
|
||||
os.makedirs(STATE_DIR, exist_ok=True)
|
||||
tmp = state_path(name) + '.tmp'
|
||||
with open(tmp, 'w', encoding='utf-8') as f:
|
||||
json.dump(st, f, ensure_ascii=False, indent=1)
|
||||
os.replace(tmp, state_path(name))
|
||||
|
||||
|
||||
def line_for(src, vid, it):
|
||||
def s(x):
|
||||
return str(x if x is not None else '').replace('|', '/').strip()
|
||||
|
||||
if src['kind'] == 'kijiji':
|
||||
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
|
||||
s(it.get('title'))[:78], s(it.get('loc')), s(it.get('km')))
|
||||
if src['kind'] == 'autotrader':
|
||||
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
|
||||
s(it.get('title'))[:78], s(it.get('city')), s(it.get('km')))
|
||||
return '{}|{}|{}|{}|{}|{}'.format(src['name'], vid, s(it.get('price')),
|
||||
s(it.get('title'))[:78], s(it.get('cond')), '')
|
||||
|
||||
|
||||
def run_source(src, pages_override=None):
|
||||
name, kind = src['name'], src['kind']
|
||||
pages = pages_override or src['pages']
|
||||
prev = load_state(name)
|
||||
if kind == 'kijiji':
|
||||
ok, items, per = fetch_kijiji(src, pages)
|
||||
elif kind == 'autotrader':
|
||||
ok, items, per = fetch_at(src, pages)
|
||||
else:
|
||||
ok, items, per = fetch_ds(src, pages)
|
||||
now = now_iso()
|
||||
out = []
|
||||
if ok:
|
||||
new_items = {}
|
||||
for vid, it in items.items():
|
||||
p = (prev.get('items') or {}).get(vid) or ((prev.get('absent') or {}).get(vid) or {}).get('last') or {}
|
||||
it2 = dict(it)
|
||||
it2['img'] = p.get('img') or it.get('img') or ''
|
||||
it2['first_seen'] = p.get('first_seen', now)
|
||||
it2['last_seen'] = now
|
||||
if p.get('price') is not None and it.get('price') and p.get('price') != it.get('price'):
|
||||
it2['price_prev'] = p['price']
|
||||
new_items[vid] = it2
|
||||
absent = dict(prev.get('absent') or {})
|
||||
for vid in list(absent):
|
||||
if vid in new_items:
|
||||
del absent[vid]
|
||||
for vid in (prev.get('items') or {}):
|
||||
if vid not in new_items:
|
||||
a = dict(absent.get(vid) or {})
|
||||
a['first_absent'] = a.get('first_absent', now)
|
||||
a['times_missing'] = a.get('times_missing', 0) + 1
|
||||
a.setdefault('last', prev['items'][vid])
|
||||
absent[vid] = a
|
||||
for vid in [v for v, a in absent.items() if a.get('times_missing', 0) > 60]:
|
||||
del absent[vid]
|
||||
partial = any(v is None or v == 0 or
|
||||
(kind == 'kijiji' and v < src['min']) for v in per.values())
|
||||
st = {'source': name, 'fetched_at': now, 'ok': True, 'fail_streak': 0,
|
||||
'pages': per, 'partial': partial, 'items': new_items, 'absent': absent}
|
||||
save_state(name, st)
|
||||
if partial:
|
||||
out.append('!partial|' + name)
|
||||
for vid in sorted(new_items):
|
||||
out.append(line_for(src, vid, new_items[vid]))
|
||||
else:
|
||||
streak = (prev.get('fail_streak') or 0) + 1
|
||||
st = dict(prev) if prev else {'source': name, 'items': {}, 'absent': {}}
|
||||
st.update({'source': name, 'fetched_at': now, 'ok': False,
|
||||
'fail_streak': streak, 'pages': per})
|
||||
save_state(name, st)
|
||||
if streak >= 2:
|
||||
out.append('!fail|' + name)
|
||||
for vid in sorted(st.get('items') or {}):
|
||||
out.append(line_for(src, vid, st['items'][vid]))
|
||||
return out, st
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--seed', '--full', action='store_true', help='full-depth sweep (daily coverage)')
|
||||
ap.add_argument('--source', action='append', default=None)
|
||||
ap.add_argument('--pages', type=int, default=None)
|
||||
ap.add_argument('--verbose', action='store_true')
|
||||
args = ap.parse_args()
|
||||
os.makedirs(STATE_DIR, exist_ok=True)
|
||||
lines = []
|
||||
for src in SOURCES:
|
||||
if args.source and src['name'] not in args.source:
|
||||
continue
|
||||
pages = args.pages
|
||||
if pages is None and args.seed:
|
||||
pages = SEED_PAGES.get(src['name'], src['pages'])
|
||||
t0 = time.time()
|
||||
try:
|
||||
block, st = run_source(src, pages_override=pages)
|
||||
except Exception as e:
|
||||
prev = load_state(src['name'])
|
||||
streak = (prev.get('fail_streak') or 0) + 1
|
||||
prev['fail_streak'] = streak
|
||||
prev['ok'] = False
|
||||
prev['fetched_at'] = now_iso()
|
||||
save_state(src['name'], prev)
|
||||
block = (['!fail|' + src['name']] if streak >= 2 else [])
|
||||
block += [line_for(src, vid, it) for vid, it in sorted((prev.get('items') or {}).items())]
|
||||
st = prev
|
||||
if args.verbose:
|
||||
print('{}: EXCEPTION {}'.format(src['name'], e), file=sys.stderr)
|
||||
lines.extend(block)
|
||||
if args.verbose:
|
||||
print('{}: ok={} items={} pages={} partial={} {:.0f}s'.format(
|
||||
src['name'], st.get('ok'), len(st.get('items') or {}), st.get('pages'),
|
||||
st.get('partial'), time.time() - t0), file=sys.stderr)
|
||||
sys.stdout.write('\n'.join(lines) + ('\n' if lines else ''))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user