From 19e46cd11b2c39becb9d8b1d1489aaad77a7f026 Mon Sep 17 00:00:00 2001 From: Torres Date: Tue, 10 Nov 2026 18:57:17 -0400 Subject: [PATCH] app: add proof deck CLI, dependency-free selfcheck, gitignore --- .gitignore | 12 +++ app/deck.py | 65 +++++++++++++ app/selfcheck.py | 249 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 326 insertions(+) create mode 100644 .gitignore create mode 100644 app/deck.py create mode 100644 app/selfcheck.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f564ead --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# runtime state +data/ +*.sqlite3 +*.sqlite3-wal +*.sqlite3-shm + +# python +__pycache__/ +*.pyc + +# scratch +scratch/ diff --git a/app/deck.py b/app/deck.py new file mode 100644 index 0000000..1ca3ed2 --- /dev/null +++ b/app/deck.py @@ -0,0 +1,65 @@ +"""Read-only proof CLI: print the ranked deck and DB counts. + + python3 -m app.deck --db data/bike-tinder.sqlite3 --top 15 +""" +import argparse +import sys + +from . import db + +DECK_QUERY = """ +SELECT * FROM listings +WHERE status = 'active' + AND id NOT IN (SELECT listing_id FROM swipes WHERE action = 'nope') + AND id NOT IN (SELECT listing_id FROM swipes WHERE action IN ('like','maybe') + AND (action = 'like' OR updated_at > datetime('now','-7 days'))) +ORDER BY score DESC, first_seen DESC +LIMIT ? +""" + + +def fmt_price(p): + return p if p else '-' + + +def fmt_km(k): + return ('%d km' % k) if k is not None else '-' + + +def main(argv=None): + ap = argparse.ArgumentParser(description='Bike-Tinder deck (read-only)') + ap.add_argument('--db', default='data/bike-tinder.sqlite3') + ap.add_argument('--top', type=int, default=15) + args = ap.parse_args(argv) + + conn = db.connect(args.db) + db.init_db(conn) + + rows = conn.execute(DECK_QUERY, (args.top,)).fetchall() + print('=== Bike-Tinder deck (top %d) ===' % len(rows)) + for i, r in enumerate(rows, 1): + print('%3d. %s · %s · %s · %s · %d' % ( + i, r['title'], fmt_price(r['price_display']), fmt_km(r['km']), + r['region'], r['score'])) + + counts = {} + for status in ('active', 'gone', 'dup'): + counts[status] = conn.execute( + 'SELECT COUNT(*) AS n FROM listings WHERE status=?', (status,)).fetchone()['n'] + total = conn.execute('SELECT COUNT(*) AS n FROM listings').fetchone()['n'] + last = conn.execute( + 'SELECT new FROM import_runs ORDER BY id DESC LIMIT 1').fetchone() + new = last['new'] if last else 0 + + print('=== counts ===') + print('total: %d' % total) + print('active: %d' % counts['active']) + print('gone: %d' % counts['gone']) + print('dup: %d' % counts['dup']) + print('new: %d' % new) + conn.close() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/app/selfcheck.py b/app/selfcheck.py new file mode 100644 index 0000000..3abe981 --- /dev/null +++ b/app/selfcheck.py @@ -0,0 +1,249 @@ +"""Dependency-free self-check for the Phase 1 data spine. + + python3 -m app.selfcheck + +Covers: gate rejections, dedupe (incl. false-positive sanity), scoring +spot-checks against the reference rubric, and gone-after-2-misses. +""" +import datetime +import json +import os +import shutil +import sys +import tempfile + +from . import db, rubric +from . import importer as imp + + +def recent_iso(): + return datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def make_item(title, price='$5,000', price_num=5000, km='10000', loc='Halifax', + url='https://www.kijiji.ca/v-x/1', desc='', img=''): + return {'title': title, 'price': price, 'price_num': price_num, 'loc': loc, + 'km': km, 'url': url, 'desc': desc, 'img': img, + 'first_seen': '2026-01-01T00:00:00Z', 'last_seen': '2026-01-01T00:00:00Z'} + + +def make_state(items, ok=True, partial=False, fetched_at=None, absent=None): + return {'source': 'kijiji-ns', 'fetched_at': fetched_at or recent_iso(), + 'ok': ok, 'fail_streak': 0, 'pages': {'1': len(items)}, + 'partial': partial, 'items': items, 'absent': absent or {}} + + +def write_states(state_dir, ns, nb): + os.makedirs(state_dir, exist_ok=True) + for name, st in (('kijiji-ns', ns), ('kijiji-nb', nb)): + with open(os.path.join(state_dir, name + '.json'), 'w', encoding='utf-8') as fh: + json.dump(st, fh) + + +def fresh_db(): + d = tempfile.mkdtemp(prefix='bt-selfcheck-') + return d, os.path.join(d, 'test.sqlite3') + + +# --- gate rejections ------------------------------------------------------- + +def test_gate_rejections(): + ok, err = imp.gate_region(make_state({'1': make_item('X')}, ok=False)) + assert not ok and 'ok' in (err or ''), err + ok, err = imp.gate_region(make_state({'1': make_item('X')}, partial=True)) + assert not ok and 'partial' in (err or ''), err + stale = (datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(hours=30)).strftime('%Y-%m-%dT%H:%M:%SZ') + ok, err = imp.gate_region(make_state({'1': make_item('X')}, fetched_at=stale)) + assert not ok and '26h' in (err or ''), err + ok, err = imp.gate_region(make_state({})) + assert not ok and 'empty' in (err or ''), err + ok, err = imp.gate_region(make_state({'1': make_item('X')})) + assert ok and err is None, err + + +def test_gate_abort_leaves_db(): + d, db_path = fresh_db() + try: + state_dir = os.path.join(d, 'state') + write_states(state_dir, + make_state({'1': make_item('X')}, ok=False), # ns fails + make_state({'2': make_item('Y')})) # nb ok + res = imp.run_import(state_dir, db_path) + assert res['ok'] is False + conn = db.connect(db_path) + db.init_db(conn) + n = conn.execute('SELECT COUNT(*) AS n FROM listings').fetchone()['n'] + runs = conn.execute('SELECT ok, error FROM import_runs').fetchall() + conn.close() + assert n == 0, 'listings must be untouched on gate failure' + assert len(runs) == 1 and runs[0]['ok'] == 0 and 'ns' in runs[0]['error'] + finally: + shutil.rmtree(d, ignore_errors=True) + + +# --- dedupe ---------------------------------------------------------------- + +def test_dedupe_twins(): + d, db_path = fresh_db() + try: + state_dir = os.path.join(d, 'state') + twin = make_item('2021 Indian Scout Bobber', price_num=6500, km='8000') + write_states(state_dir, + make_state({'ns1': twin}), + make_state({'nb1': dict(twin)})) + res = imp.run_import(state_dir, db_path) + assert res['ok'] and res['total'] == 2 and res['dup'] == 1, res + conn = db.connect(db_path) + db.init_db(conn) + rows = conn.execute( + 'SELECT id, region, status, dup_of FROM listings ORDER BY region').fetchall() + conn.close() + by_status = {r['status'] for r in rows} + assert by_status == {'active', 'dup'}, rows + dup = [r for r in rows if r['status'] == 'dup'][0] + active = [r for r in rows if r['status'] == 'active'][0] + assert dup['dup_of'] == active['id'], (dup, active) + finally: + shutil.rmtree(d, ignore_errors=True) + + +def test_dedupe_false_positive_same_province(): + # Same title+price but both in ONE province -> different bikes, not merged. + d, db_path = fresh_db() + try: + state_dir = os.path.join(d, 'state') + write_states(state_dir, + make_state({'ns1': make_item('2025 BMW G 310 R', price_num=5444), + 'ns2': make_item('2025 BMW G 310 R', price_num=5444)}), + make_state({'nb_dummy': make_item('Some Other Bike', price_num=9999)})) + res = imp.run_import(state_dir, db_path) + assert res['ok'] and res['total'] == 3 and res['dup'] == 0, res + finally: + shutil.rmtree(d, ignore_errors=True) + + +def test_dedupe_false_positive_price_diff(): + # Same title, different price, cross-province -> not merged. + d, db_path = fresh_db() + try: + state_dir = os.path.join(d, 'state') + write_states(state_dir, + make_state({'ns1': make_item('2021 Indian Scout', price_num=6500)}), + make_state({'nb1': make_item('2021 Indian Scout', price_num=7000)})) + res = imp.run_import(state_dir, db_path) + assert res['ok'] and res['total'] == 2 and res['dup'] == 0, res + finally: + shutil.rmtree(d, ignore_errors=True) + + +# --- scoring spot-checks --------------------------------------------------- + +def test_scoring(): + s, fam = rubric.score_item('2021 Indian Scout Bobber', 'MVI done, low km', + 6500, 8000, 2021) + assert s == 105 and fam and 'Scout' in fam, (s, fam) + s, fam = rubric.score_item('Kawasaki Vulcan 900 Custom', 'whitewalls, windshield', + 5000, 20000, 2018) + assert s == 102 and 'Vulcan 900 Custom' in fam, (s, fam) + s, fam = rubric.score_item('Kawasaki Ninja 400', 'sport bike', 5000, 10000, 2020) + assert s == 0 and fam is None, (s, fam) # vetoed -> unscored + s, fam = rubric.score_item('Some random bike', '', 3000, None, None) + assert s == 0 and fam is None, (s, fam) # no family -> unscored + # km=0 placeholder -> treated as no km + s, _ = rubric.score_item('2021 Indian Scout', '', 5000, 0, 2021) + assert s == 92 + 5 + 2, s # base + price(<=5000) + year(>=2020), no km bonus + + +# --- gone after 2 misses --------------------------------------------------- + +def test_gone_after_two_misses(): + d, db_path = fresh_db() + try: + conn = db.connect(db_path) + db.init_db(conn) + conn.execute( + "INSERT INTO listings (region, src_id, title, url, status, last_seen) " + "VALUES ('ns','gone1','X','http://x/1','active','2026-01-01T00:00:01Z')") + conn.commit() + # single miss: last_seen == prev_good -> not gone + gone = imp.mark_gone(conn, set(), '2026-01-01T00:00:01Z') + assert gone == 0, gone + st = conn.execute("SELECT status FROM listings WHERE src_id='gone1'").fetchone()['status'] + assert st == 'active', st + # second miss: last_seen < prev_good -> gone + gone = imp.mark_gone(conn, set(), '2026-01-01T00:00:02Z') + assert gone == 1, gone + st = conn.execute("SELECT status FROM listings WHERE src_id='gone1'").fetchone()['status'] + assert st == 'gone', st + conn.close() + finally: + shutil.rmtree(d, ignore_errors=True) + + +def test_gone_full_flow(): + # End-to-end: present -> absent (single miss) -> absent (gone). + d, db_path = fresh_db() + try: + state_dir = os.path.join(d, 'state') + orig = imp.now_iso + clock = {'now': '2026-01-01T00:00:01Z'} + imp.now_iso = lambda: clock['now'] + try: + write_states(state_dir, + make_state({'a1': make_item('2021 Indian Scout', price_num=6500)}), + make_state({'nb_dummy': make_item('Some Other Bike', price_num=9999)})) + imp.run_import(state_dir, db_path) # import 1: present + + clock['now'] = '2026-01-01T00:00:02Z' + write_states(state_dir, + make_state({'ns_other': make_item('Another Bike', price_num=8888)}), + make_state({'nb_dummy': make_item('Some Other Bike', price_num=9999)})) + imp.run_import(state_dir, db_path) # import 2: absent (single miss) + conn = db.connect(db_path) + db.init_db(conn) + st = conn.execute("SELECT status FROM listings WHERE src_id='a1'").fetchone()['status'] + conn.close() + assert st == 'active', 'single miss must not mark gone' + + clock['now'] = '2026-01-01T00:00:03Z' + imp.run_import(state_dir, db_path) # import 3: absent (2nd miss) + conn = db.connect(db_path) + db.init_db(conn) + st = conn.execute("SELECT status FROM listings WHERE src_id='a1'").fetchone()['status'] + conn.close() + assert st == 'gone', '2nd consecutive miss must mark gone' + finally: + imp.now_iso = orig + finally: + shutil.rmtree(d, ignore_errors=True) + + +TESTS = [ + ('gate rejections', test_gate_rejections), + ('gate abort leaves DB untouched', test_gate_abort_leaves_db), + ('dedupe cross-province twins', test_dedupe_twins), + ('dedupe false-positive (same province)', test_dedupe_false_positive_same_province), + ('dedupe false-positive (price differs)', test_dedupe_false_positive_price_diff), + ('scoring spot-checks', test_scoring), + ('gone after 2 misses (unit)', test_gone_after_two_misses), + ('gone after 2 misses (full flow)', test_gone_full_flow), +] + + +def main(): + failures = 0 + for name, fn in TESTS: + try: + fn() + print('PASS %s' % name) + except Exception as e: + failures += 1 + print('FAIL %s: %s' % (name, e)) + print('---') + print('%d/%d passed' % (len(TESTS) - failures, len(TESTS))) + return 1 if failures else 0 + + +if __name__ == '__main__': + sys.exit(main())