P3: personalization core — event log + taste engine + section engine + agent pass + API + frontend
- db.py: guarded P3 tables (swipe_events/sections/section_items/taste_profile/ agent_runs), one-time event seeding from swipes, append_event round math - taste.py: deterministic taste engine (action x round x recency weights, n/(n+3) shrink, ramp w=min(.85,N/60), percentile norm, agent adjustments +/-10, points + source-labeled breakdown + tank bands) - sections.py: compose/read/exhaust/forge — 2 wildcards + <=5 returns (<=2 nopes, surprising only) + remainder new, <=4/family, >=3 price bands w/ repair, returns maybe>like>nope, last-two-section exclusion, ondemand/strict modes - agentpass.py: mock (deterministic) / live (OpenRouter, one retry) / fallback; strict payload schema; --print-summary/--dry-run - api.py: GET /api/section, POST /api/section/forge, GET /api/trends, POST /api/agent/run, POST /api/prefs; P3 payload fields (points, tank, slot, round_next, points_breakdown); events stamped with section_id - frontend: section buffer + progress pill, report screen + forge + mode toggle, WILDCARD/SECOND LOOK bands, tank gauges (list + dossier), dossier Why-this-rank, real Trends tab, /api/deck fallback for pre-P3 API, sw v2 - selfcheck: 9 new P3 checks (25/25 green)
This commit is contained in:
488
app/sections.py
Normal file
488
app/sections.py
Normal file
@@ -0,0 +1,488 @@
|
||||
"""Bike-Tinder P3 section engine — compose / read / exhaust / forge.
|
||||
|
||||
The deck is one finite SECTION at a time (default 25 cards):
|
||||
2 wildcards + <=5 returns (<=2 nopes, "surprising" only) + remainder new,
|
||||
per docs/RANKING.md + docs/DATA-MODEL.md "P3 additions".
|
||||
|
||||
Pinned order within a section (position 1 first):
|
||||
new (points DESC) -> returns (maybe > like > nope, points DESC within
|
||||
role) -> wildcards (wildcard score DESC). "The best suggestions lead"
|
||||
(docs/PLAN.md); returns form their own sub-sequence in role order;
|
||||
wildcard stretches sit at the back. (Wildcard placement inside the deck
|
||||
is not pinned anywhere in the frozen docs — noted in the P3 report.)
|
||||
|
||||
Constraints: <=4 per non-NULL family per section; >=3 price bands when the
|
||||
pool allows (soft repair swaps new-slot items within the final list).
|
||||
A re-nope (round>=2 nope event) rests an item: no more second looks.
|
||||
Rounds cap at 2 for influence (taste.py); returns are offered once.
|
||||
|
||||
Modes: 'ondemand' (default) — forge runs the agent pass then composes;
|
||||
'strict' — forge refuses (409) unless a taste_profile strictly newer than
|
||||
the current section's profile_version exists.
|
||||
"""
|
||||
import json
|
||||
|
||||
from . import taste as tastemod
|
||||
from .taste import NOPE
|
||||
|
||||
FAMILY_CAP = 4
|
||||
MAX_RETURNS = 5
|
||||
MAX_RETURN_NOPES = 2
|
||||
WILDCARDS = 2
|
||||
DEFAULT_SIZE = 25
|
||||
MIN_BANDS = 3
|
||||
|
||||
LAST_TWO_SQL = """
|
||||
SELECT si.listing_id FROM section_items si
|
||||
WHERE si.section_id IN (SELECT id FROM sections ORDER BY id DESC LIMIT 2)
|
||||
"""
|
||||
|
||||
NEW_POOL_SQL = """
|
||||
SELECT l.* FROM listings l
|
||||
WHERE l.status = 'active'
|
||||
AND NOT EXISTS (SELECT 1 FROM swipes s WHERE s.listing_id = l.id)
|
||||
"""
|
||||
|
||||
RETURNS_SQL = """
|
||||
SELECT l.*, s.action AS verdict FROM listings l
|
||||
JOIN swipes s ON s.listing_id = l.id
|
||||
WHERE l.status = 'active'
|
||||
AND NOT EXISTS (SELECT 1 FROM swipe_events e
|
||||
WHERE e.listing_id = l.id AND e.round >= 2
|
||||
AND e.action = 'nope')
|
||||
AND NOT EXISTS (SELECT 1 FROM section_items si
|
||||
WHERE si.listing_id = l.id AND si.slot = 'return')
|
||||
"""
|
||||
|
||||
WILDCARD_SQL = """
|
||||
SELECT l.* FROM listings l
|
||||
WHERE l.status = 'active'
|
||||
AND NOT EXISTS (SELECT 1 FROM swipe_events e WHERE e.listing_id = l.id)
|
||||
"""
|
||||
|
||||
|
||||
# --- helpers ----------------------------------------------------------------
|
||||
|
||||
def _iso(now):
|
||||
return now.strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
|
||||
def last_two_listing_ids(conn):
|
||||
return {r['listing_id'] for r in conn.execute(LAST_TWO_SQL)}
|
||||
|
||||
|
||||
def get_current_section(conn):
|
||||
"""The newest open/report section, or None."""
|
||||
return conn.execute(
|
||||
"SELECT * FROM sections WHERE status IN ('open','report')"
|
||||
' ORDER BY id DESC LIMIT 1').fetchone()
|
||||
|
||||
|
||||
def latest_profile(conn):
|
||||
return conn.execute(
|
||||
'SELECT * FROM taste_profile ORDER BY version DESC LIMIT 1'
|
||||
).fetchone()
|
||||
|
||||
|
||||
def build_engine(conn, profile=None, now=None):
|
||||
"""TasteEngine over active listings + every listing an event touches,
|
||||
with the given profile payload (default: the latest version)."""
|
||||
rows = [r for r in conn.execute(
|
||||
"SELECT * FROM listings WHERE status='active'")]
|
||||
ev_ids = {r['listing_id'] for r in conn.execute(
|
||||
'SELECT DISTINCT listing_id FROM swipe_events')}
|
||||
missing = ev_ids - {r['id'] for r in rows}
|
||||
if missing:
|
||||
qmarks = ','.join('?' * len(missing))
|
||||
rows += [r for r in conn.execute(
|
||||
'SELECT * FROM listings WHERE id IN (%s)' % qmarks,
|
||||
tuple(missing))]
|
||||
events = conn.execute('SELECT * FROM swipe_events ORDER BY id').fetchall()
|
||||
payload = None
|
||||
if profile is None:
|
||||
profile = latest_profile(conn)
|
||||
if profile is not None and profile['payload']:
|
||||
try:
|
||||
payload = json.loads(profile['payload'])
|
||||
except ValueError:
|
||||
payload = None
|
||||
return tastemod.TasteEngine(rows, events, payload, now=now)
|
||||
|
||||
|
||||
def fresh_profile_exists(conn, section):
|
||||
"""Strict-mode freshness: a profile version newer than the section's."""
|
||||
cur = section['profile_version'] if section else None
|
||||
if latest_profile(conn) is None:
|
||||
return False
|
||||
if cur is None:
|
||||
return True
|
||||
return latest_profile(conn)['version'] > cur
|
||||
|
||||
|
||||
# --- wildcard novelty --------------------------------------------------------
|
||||
|
||||
def _cosine(a, b):
|
||||
dot = sum(a.get(k, 0.0) * b.get(k, 0.0) for k in (a.keys() | b.keys()))
|
||||
na = sum(x * x for x in a.values()) ** 0.5
|
||||
nb = sum(x * x for x in b.values()) ** 0.5
|
||||
if not na or not nb:
|
||||
return 0.0
|
||||
return dot / (na * nb)
|
||||
|
||||
|
||||
def novelty(rows_by_id, liked_ids, candidate_ids):
|
||||
"""0..1 distance from the liked-profile centroid (1 = very novel)."""
|
||||
if not liked_ids:
|
||||
return {lid: 0.5 for lid in candidate_ids}
|
||||
centroid = {}
|
||||
liked = [l for l in liked_ids if l in rows_by_id]
|
||||
for lid in liked:
|
||||
for d, v in tastemod.extract_features(rows_by_id[lid]):
|
||||
centroid[(d, v)] = centroid.get((d, v), 0.0) + 1.0
|
||||
n = len(liked) or 1
|
||||
centroid = {k: v / n for k, v in centroid.items()}
|
||||
out = {}
|
||||
for lid in candidate_ids:
|
||||
vec = {(d, v): 1.0 for d, v in tastemod.extract_features(rows_by_id[lid])}
|
||||
out[lid] = 1.0 - _cosine(vec, centroid)
|
||||
return out
|
||||
|
||||
|
||||
# --- compose -----------------------------------------------------------------
|
||||
|
||||
def _points_key(engine, row):
|
||||
"""Deterministic sort key: points, rubric score, first_seen, id.
|
||||
Use with reverse=True for descending order."""
|
||||
return (engine.points(row['id']), row['score'] or 0,
|
||||
row['first_seen'] or '', row['id'])
|
||||
|
||||
|
||||
def pick_new(engine, candidates, want, fam_count, picked_ids):
|
||||
"""Greedy new picks by points DESC honoring the family cap."""
|
||||
out = []
|
||||
ordered = sorted(candidates, key=lambda r: _points_key(engine, r),
|
||||
reverse=True)
|
||||
for cand in ordered:
|
||||
if len(out) >= want:
|
||||
break
|
||||
lid = cand['id']
|
||||
if lid in picked_ids:
|
||||
continue
|
||||
fam = cand['family']
|
||||
if fam and fam_count.get(fam, 0) >= FAMILY_CAP:
|
||||
continue
|
||||
out.append(cand)
|
||||
picked_ids.add(lid)
|
||||
if fam:
|
||||
fam_count[fam] = fam_count.get(fam, 0) + 1
|
||||
return out
|
||||
|
||||
|
||||
def repair_bands(engine, picked, candidates, fam_count, picked_ids):
|
||||
"""Swap new-slot items to reach >=3 price bands when the pool allows.
|
||||
|
||||
Only swaps INTO slots of new items in over-represented bands; returns
|
||||
and wildcard slots are never touched. Deterministic; bounded passes.
|
||||
"""
|
||||
if len(picked) < MIN_BANDS:
|
||||
return picked
|
||||
bands_of = {r['id']: tastemod.price_band(r['price_num']) for r in candidates}
|
||||
bands_of.update({r['id']: tastemod.price_band(r['price_num'])
|
||||
for r in picked})
|
||||
cur_bands = {bands_of[r['id']] for r in picked
|
||||
if bands_of[r['id']] is not None}
|
||||
if len(cur_bands) >= MIN_BANDS:
|
||||
return picked
|
||||
pool_bands = {bands_of[r['id']] for r in candidates
|
||||
if bands_of[r['id']] is not None}
|
||||
if len(pool_bands) < MIN_BANDS:
|
||||
return picked # the pool does not allow 3 bands
|
||||
picked = list(picked)
|
||||
attempts = len(candidates)
|
||||
for _ in range(attempts):
|
||||
cur_bands = {bands_of[r['id']] for r in picked
|
||||
if bands_of[r['id']] is not None}
|
||||
if len(cur_bands) >= MIN_BANDS:
|
||||
break
|
||||
missing = [b for b in pool_bands if b not in cur_bands]
|
||||
if not missing:
|
||||
break
|
||||
want_band = missing[0]
|
||||
best = None
|
||||
best_key = None
|
||||
for cand in candidates:
|
||||
if cand['id'] in picked_ids:
|
||||
continue
|
||||
if bands_of[cand['id']] != want_band:
|
||||
continue
|
||||
fam = cand['family']
|
||||
if fam and fam_count.get(fam, 0) >= FAMILY_CAP:
|
||||
continue
|
||||
key = _points_key(engine, cand)
|
||||
if best_key is None or key > best_key:
|
||||
best, best_key = cand, key
|
||||
if best is None:
|
||||
pool_bands = pool_bands - {want_band}
|
||||
if len(pool_bands) < MIN_BANDS:
|
||||
break
|
||||
continue
|
||||
# victim: lowest-points new-slot item from a band with >=2 reps
|
||||
counts = {}
|
||||
for r in picked:
|
||||
b = bands_of[r['id']]
|
||||
if b is not None:
|
||||
counts[b] = counts.get(b, 0) + 1
|
||||
victim = None
|
||||
worst_first = sorted(picked, key=lambda r: _points_key(engine, r),
|
||||
reverse=True)
|
||||
for band in sorted(counts, key=lambda b: -counts[b]):
|
||||
if counts[band] < 2:
|
||||
continue
|
||||
in_band = [r for r in worst_first if bands_of[r['id']] == band]
|
||||
if in_band:
|
||||
victim = in_band[-1]
|
||||
break
|
||||
if victim is None:
|
||||
break
|
||||
victim_fam = victim['family']
|
||||
idx = next(i for i, r in enumerate(picked) if r['id'] == victim['id'])
|
||||
picked[idx] = best
|
||||
if victim_fam and victim_fam in fam_count:
|
||||
fam_count[victim_fam] = max(0, fam_count[victim_fam] - 1)
|
||||
if best['family']:
|
||||
fam_count[best['family']] = fam_count.get(best['family'], 0) + 1
|
||||
picked_ids.remove(victim['id'])
|
||||
picked_ids.add(best['id'])
|
||||
return picked
|
||||
|
||||
|
||||
def compose_section(conn, engine, size=DEFAULT_SIZE, profile_version=None,
|
||||
note='', now=None):
|
||||
"""Compose and persist a new open section; returns the section row.
|
||||
|
||||
Caller closes/reports any prior section first. `engine` scores the
|
||||
candidates (a TasteEngine over active listings + events + profile).
|
||||
"""
|
||||
import datetime
|
||||
now = now or datetime.datetime.now(datetime.timezone.utc).replace(
|
||||
tzinfo=None)
|
||||
size = int(size)
|
||||
rows_by_id = {r['id']: r for r in engine.listings}
|
||||
last_two = last_two_listing_ids(conn)
|
||||
|
||||
new_cands = [r for r in conn.execute(NEW_POOL_SQL)
|
||||
if r['id'] not in last_two]
|
||||
ret_cands = [r for r in conn.execute(RETURNS_SQL)
|
||||
if r['id'] not in last_two]
|
||||
wild_cands = [r for r in conn.execute(WILDCARD_SQL)
|
||||
if r['id'] not in last_two]
|
||||
|
||||
# wildcard family exclusions: >=3 cumulative nopes AND net-negative.
|
||||
# Stats are computed for every family the engine knows (active + event
|
||||
# listings), not just current candidates — a hard-noped family is
|
||||
# excluded even when its rows are gone/cleared.
|
||||
fam_stats = {f: engine.family_net(f)
|
||||
for f in {r['family'] for r in engine.listings if r['family']}}
|
||||
excluded_fams = {f for f, (net, nopes) in fam_stats.items()
|
||||
if nopes >= 3 and net < 0}
|
||||
wild_cands = [r for r in wild_cands
|
||||
if not (r['family'] and r['family'] in excluded_fams)]
|
||||
|
||||
fam_count = {}
|
||||
picked_ids = set()
|
||||
|
||||
# 1) preliminary top-3 new (uncommitted) — keys the wildcard preference
|
||||
top3 = pick_new(engine, new_cands, 3, {}, set())
|
||||
top_fams = {r['family'] for r in top3 if r['family']}
|
||||
top_bands = {tastemod.price_band(r['price_num']) for r in top3
|
||||
if tastemod.price_band(r['price_num']) is not None}
|
||||
|
||||
# 2) returns: maybe > like > nope, points DESC within role; nopes must be
|
||||
# "surprising" (points >=55 or net-positive family); <=5, <=2 nopes.
|
||||
# Budget: wildcards keep their 2 slots; returns never push the
|
||||
# section past `size`.
|
||||
wanted_rets = min(MAX_RETURNS, max(0, size - min(WILDCARDS, size)))
|
||||
role_order = {'maybe': 0, 'like': 1, 'nope': 2}
|
||||
rets = []
|
||||
nope_count = 0
|
||||
for cand in sorted(ret_cands, reverse=True,
|
||||
key=lambda r: (-role_order.get(r['verdict'], 9),
|
||||
_points_key(engine, r))):
|
||||
if len(rets) >= wanted_rets:
|
||||
break
|
||||
fam = cand['family']
|
||||
if fam and fam_count.get(fam, 0) >= FAMILY_CAP:
|
||||
continue
|
||||
if cand['verdict'] == NOPE:
|
||||
if nope_count >= MAX_RETURN_NOPES:
|
||||
continue
|
||||
net, _nopes = fam_stats.get(fam, (0.0, 0))
|
||||
if engine.points(cand['id']) < 55 and not (fam and net > 0):
|
||||
continue
|
||||
nope_count += 1
|
||||
rets.append(cand)
|
||||
picked_ids.add(cand['id'])
|
||||
if fam:
|
||||
fam_count[fam] = fam_count.get(fam, 0) + 1
|
||||
|
||||
# 3) wildcards: min(2, size), preferred different family AND price band
|
||||
liked_ids = [r['listing_id'] for r in conn.execute(
|
||||
"SELECT listing_id FROM swipes WHERE action='like'")]
|
||||
nov = novelty(rows_by_id, liked_ids,
|
||||
[r['id'] for r in wild_cands])
|
||||
wild_key = lambda r: (
|
||||
-(0.5 * engine.points(r['id']) + 50.0 * nov[r['id']]), r['id'])
|
||||
|
||||
def _wild_preferred(rs):
|
||||
return [r for r in rs
|
||||
if (not r['family'] or r['family'] not in top_fams)
|
||||
and (tastemod.price_band(r['price_num']) is None
|
||||
or tastemod.price_band(r['price_num']) not in top_bands)]
|
||||
|
||||
n_wilds = min(WILDCARDS, size)
|
||||
wilds = []
|
||||
avail = sorted([r for r in wild_cands if r['id'] not in picked_ids],
|
||||
key=wild_key)
|
||||
preferred = sorted(_wild_preferred(avail), key=wild_key)
|
||||
pref_ids = {r['id'] for r in preferred}
|
||||
for cand in preferred + [r for r in avail if r['id'] not in pref_ids]:
|
||||
if len(wilds) >= n_wilds:
|
||||
break
|
||||
fam = cand['family']
|
||||
if fam and fam_count.get(fam, 0) >= FAMILY_CAP:
|
||||
continue
|
||||
wilds.append(cand)
|
||||
picked_ids.add(cand['id'])
|
||||
if fam:
|
||||
fam_count[fam] = fam_count.get(fam, 0) + 1
|
||||
n_wilds = len(wilds) # pool shortfall reverts to new
|
||||
|
||||
# 4) new fills the remainder of the section (size - returns - wildcards)
|
||||
want_new = max(0, size - len(rets) - n_wilds)
|
||||
picked_new = pick_new(engine, new_cands, want_new, fam_count, picked_ids)
|
||||
|
||||
# final pinned order: new -> returns -> wildcards
|
||||
ordered = list(picked_new) + rets + wilds
|
||||
|
||||
# price-band repair (new slots only — ordered[:len(picked_new)])
|
||||
n_new = len(picked_new)
|
||||
before = len({tastemod.price_band(r['price_num']) for r in ordered
|
||||
if tastemod.price_band(r['price_num']) is not None})
|
||||
if n_new:
|
||||
ordered[:n_new] = repair_bands(engine, ordered[:n_new], new_cands,
|
||||
fam_count, picked_ids)
|
||||
after = len({tastemod.price_band(r['price_num']) for r in ordered
|
||||
if tastemod.price_band(r['price_num']) is not None})
|
||||
|
||||
recipe = {
|
||||
'new': len(picked_new),
|
||||
'return': len(rets),
|
||||
'wildcard': len(wilds),
|
||||
'bands': before if before == after else after,
|
||||
'caps_applied': {f: fam_count[f] for f in fam_count
|
||||
if fam_count[f] >= FAMILY_CAP},
|
||||
'note': note or ('bands %d->%d repaired'
|
||||
% (before, after) if before != after else ''),
|
||||
}
|
||||
|
||||
cur = conn.execute(
|
||||
"""INSERT INTO sections (status, size, profile_version, recipe,
|
||||
created_at)
|
||||
VALUES ('open', ?, ?, ?, ?)""",
|
||||
(size, profile_version, json.dumps(recipe), _iso(now)))
|
||||
section_id = cur.lastrowid
|
||||
conn.executemany(
|
||||
"""INSERT INTO section_items (section_id, listing_id, slot, position)
|
||||
VALUES (?, ?, ?, ?)""",
|
||||
[(section_id, r['id'], slot, pos)
|
||||
for pos, (r, slot) in enumerate(
|
||||
[(r, 'new') for r in ordered[:n_new]]
|
||||
+ [(r, 'return') for r in rets]
|
||||
+ [(r, 'wildcard') for r in wilds], 1)])
|
||||
conn.commit()
|
||||
return conn.execute('SELECT * FROM sections WHERE id=?',
|
||||
(section_id,)).fetchone()
|
||||
|
||||
|
||||
# --- read / exhaust ----------------------------------------------------------
|
||||
|
||||
def close_section(conn, section, now, status='closed',
|
||||
created_at_col='closed_at'):
|
||||
conn.execute(
|
||||
'UPDATE sections SET status=?, %s=? WHERE id=?' % created_at_col,
|
||||
(status, _iso(now), section['id']))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def section_items(conn, section):
|
||||
return conn.execute(
|
||||
"""SELECT l.*, s.action AS decision, si.slot, si.position
|
||||
FROM section_items si
|
||||
JOIN listings l ON l.id = si.listing_id
|
||||
LEFT JOIN swipes s ON s.listing_id = l.id
|
||||
WHERE si.section_id = ?
|
||||
ORDER BY si.position ASC""", (section['id'],)).fetchall()
|
||||
|
||||
|
||||
def section_counts(conn, section):
|
||||
"""Per-action counts + second-look / wildcard engagement for this
|
||||
section (events stamped with this section_id only)."""
|
||||
counts = {'likes': 0, 'maybes': 0, 'nopes': 0}
|
||||
for item in section_items(conn, section):
|
||||
if item['decision'] == 'like':
|
||||
counts['likes'] += 1
|
||||
elif item['decision'] == 'maybe':
|
||||
counts['maybes'] += 1
|
||||
elif item['decision'] == 'nope':
|
||||
counts['nopes'] += 1
|
||||
restruck = conn.execute(
|
||||
"""SELECT COUNT(DISTINCT e.listing_id) AS n FROM swipe_events e
|
||||
JOIN section_items si
|
||||
ON si.section_id = e.section_id AND si.listing_id = e.listing_id
|
||||
WHERE e.section_id = ? AND si.slot = 'return'""",
|
||||
(section['id'],)).fetchone()['n']
|
||||
wild_hits = conn.execute(
|
||||
"""SELECT COUNT(DISTINCT e.listing_id) AS n FROM swipe_events e
|
||||
JOIN section_items si
|
||||
ON si.section_id = e.section_id AND si.listing_id = e.listing_id
|
||||
WHERE e.section_id = ? AND si.slot = 'wildcard'
|
||||
AND e.action IN ('like','maybe')""",
|
||||
(section['id'],)).fetchone()['n']
|
||||
counts['restruck'] = restruck
|
||||
counts['wildcard_hits'] = wild_hits
|
||||
return counts
|
||||
|
||||
|
||||
def maybe_exhaust(conn, section, now):
|
||||
"""Flip an all-decided open section to 'report' (returns the fresh row)."""
|
||||
if section['status'] != 'open':
|
||||
return section
|
||||
items = section_items(conn, section)
|
||||
if all(it['decision'] is not None for it in items):
|
||||
close_section(conn, section, now, status='report')
|
||||
return conn.execute('SELECT * FROM sections WHERE id=?',
|
||||
(section['id'],)).fetchone()
|
||||
return section
|
||||
|
||||
|
||||
def read_section(conn, section, now):
|
||||
"""Meta + items + progress + exhausted for GET /api/section."""
|
||||
section = maybe_exhaust(conn, section, now)
|
||||
items = section_items(conn, section)
|
||||
decided = sum(1 for it in items if it['decision'] is not None)
|
||||
return {
|
||||
'section': {
|
||||
'id': section['id'],
|
||||
'status': section['status'],
|
||||
'size': section['size'],
|
||||
'profile_version': section['profile_version'],
|
||||
'recipe': json.loads(section['recipe']) if section['recipe']
|
||||
else {},
|
||||
'created_at': section['created_at'],
|
||||
'closed_at': section['closed_at'],
|
||||
},
|
||||
'items': items,
|
||||
'progress': {'decided': decided, 'undecided': len(items) - decided,
|
||||
'total': len(items)},
|
||||
'exhausted': section['status'] != 'open',
|
||||
}
|
||||
Reference in New Issue
Block a user