Ask the write watch once per run of regions in the texture pool's check, and pass released entries by.

At every submission that binds the texture pools the replay checks each listed texture for CPU writes, and takes the write-watch snapshot of the 2 MiB regions that have none yet as the list comes to them: one query a region, since the list is in no address order. Standing still that is 5 to 10 queries and the check takes 0.11 ms. While the player runs through Kakariko or Castle Town the game writes about 32 textures into new storage every four or five frames, a region the CPU wrote is not frozen again for 1,800 submissions, and the check asked for 100 to 500 regions one by one, 2.7 times a frame: 0.5 to 1.5 ms a check, about 3 us a query, in five sessions of one player. The regions are now collected for the whole list first and asked in one query per run of adjacent regions. On this machine 400 queries of 2 MiB take 727 us and one query over the same 800 MiB takes 360 us. The same regions are queried and the same pages stamped, all of them before the list is looked at, as each entry's were before its own look; a run across the edge of a watched reservation is left to the queries by range. pool_presnap_runs = 0 asks region by region as before.

A released entry stays in the list until it is destroyed, the check does nothing for it, and the stamp it compares stays as it was at the release, so one whose region took a write later missed the region check at every submission from then on and was fetched only to be passed by: 1,000 to 6,300 entries a check, with up to 8,192 released entries kept. The list now carries the released state, set at the release and cleared at the take-back, and the check passes such an entry by without touching it or asking for its regions.

The pool log's freshness line counts the released entries passed by, and its snapshot queries count the queries either way. --probe-vkmem has a case for the queries by run.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-12 10:23:10 -05:00
parent 0cc247b513
commit 65dacfa7c7
7 changed files with 143 additions and 9 deletions

View File

@@ -1440,10 +1440,15 @@ static void rel_unlink(Tex *e) {
// that the check of an entry whose 2 MiB regions took no write since (vk_mem_unchanged_locked, one shared lock for the
// whole list) touches neither the entry nor the lock: 5,600 entries took 0.77 ms a submission one by one. Texture lock
// held.
typedef struct { Tex *e; uint64_t addr, size; uint32_t since; int ok; } PEnt; // ok: uploaded as of since
// rel is the entry's released state (mark_released, entry_of's take-back): the check does nothing for a released
// entry, and with the state here it does not touch the entry to find that out. On a card with room the sweep keeps up
// to 8,192 released entries, they stay listed, and since stays as it was at their release, so each one whose regions
// took a write later missed the region check at every submission from then on: 1,000 to 6,300 entries a check in
// Kakariko and Castle Town, a third to a half of the list.
typedef struct { Tex *e; uint64_t addr, size; uint32_t since; uint8_t ok, rel; } PEnt; // ok: uploaded as of since
static PEnt *pent;
static int npent, cpent, *pq, cpq;
static uint64_t n_pool_checks, n_pool_skips;
static uint64_t n_pool_checks, n_pool_skips, n_pool_rel;
// The region check's misses (vk_mem_unchanged_why_locked): not uploaded, a region without this epoch's snapshot yet, one
// untracked, one written since, nothing to check; and the snapshot queries its own pass made (NX2NAVI_POOL_PRESNAP).
static uint64_t n_pool_miss[5], n_presnap_q;
@@ -1534,7 +1539,7 @@ int vk_texture_pool_note_locked(int id, uint32_t *ent) {
if (!p || !q) rt_fatal("vk: out of memory for the texture pool's entry list");
pent = p, pq = q, cpent = cpq = n;
}
pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0};
pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0, (uint8_t)e->released};
e->pidx = npent;
}
*ent = pool_ref(e);
@@ -1553,6 +1558,14 @@ static int pool_presnap(void) {
if (mode < 0) mode = rt_knob("POOL_PRESNAP", "1");
return RT_KNOB_ON(mode);
}
// NX2NAVI_POOL_PRESNAP_RUNS=1 (default), 0, ab: those snapshots are taken for the whole list first, one write-watch
// query per run of adjacent regions (vk_memory.c vk_mem_snap_noted_locked), instead of one per region as the list comes
// to it. The log's "snapshot queries" of the pool freshness check count either.
static int pool_presnap_runs(void) {
static int mode = -1;
if (mode < 0) mode = rt_knob("POOL_PRESNAP_RUNS", "1");
return RT_KNOB_ON(mode);
}
static void pool_made(uint64_t t0); // below, with the pool's creation budget
// The submission being recorded checked the pool's entries by this list: for the idle clock it walked the pool
@@ -1561,13 +1574,24 @@ static int pool_by_list;
int vk_texture_pool_fresh_locked(void) {
int nq = 0, ps = pool_presnap();
uint64_t rel0 = n_pool_rel;
pool_by_list = 1;
if (ps) vk_mem_gen_lock();
else vk_mem_gen_lock_shared();
int by_range = ps; // each entry's regions are asked for as the list comes to it
if (ps && pool_presnap_runs()) {
for (int i = 0; i < npent; i++)
if (pent[i].ok && !pent[i].rel) vk_mem_snap_note_locked(pent[i].addr, pent[i].size, submit_epoch);
n_presnap_q += vk_mem_snap_noted_locked(submit_epoch, &by_range);
}
for (int i = 0; i < npent; i++) {
const PEnt *p = &pent[i];
if (p->rel) { // released: nothing to check (the loop below would pass it by)
n_pool_rel++;
continue;
}
if (p->ok) {
if (ps) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch);
if (by_range) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch);
int why = vk_mem_unchanged_why_locked(p->addr, p->size, p->since, submit_epoch);
if (!why) continue;
n_pool_miss[why]++;
@@ -1579,6 +1603,7 @@ int vk_texture_pool_fresh_locked(void) {
if (ps) vk_mem_gen_unlock();
else vk_mem_gen_unlock_shared();
n_pool_skips += (uint64_t)(npent - nq);
n_pool_skips -= n_pool_rel - rel0;
for (int k = nq; k-- > 0;) { // from the end: an entry leaving swaps the last one into its place
int i = pq[k];
PEnt *p = &pent[i];
@@ -3909,6 +3934,7 @@ static Tex *entry_of(const tex_t *t) {
uint64_t gap = e->samp_frame - e->rel_frame;
if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap;
e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten);
if (e->pidx) pent[e->pidx - 1].rel = 0; // the pool's check looks at it again
e->overwritten = 0;
e->og_backs++, e->og_back_frame = e->samp_frame;
if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work
@@ -3971,6 +3997,7 @@ static int wb_wanted(const Tex *e) { return e->t.L.size <= wb_limit || e->run <
static void mark_released(Tex *e) { // tex_lock held; e is not released
flight_ev(e, FLIGHT_TEX_RELEASE, 0, 0);
e->released = 1;
if (e->pidx) pent[e->pidx - 1].rel = 1; // the pool's check passes it by (vk_texture_pool_fresh_locked)
e->rel_epoch = submit_epoch;
e->rel_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
nreleased++;
@@ -4365,11 +4392,11 @@ void vk_texture_bucket_stats(char *out, size_t n) {
if (used < n)
snprintf(out + used, n - used, "; pool freshness check: %llu entries checked, %llu passed by their regions (missed: "
"%llu not uploaded, %llu a region without a snapshot yet, %llu untracked, %llu written since, %llu "
"nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP)",
"nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP; %llu released entries passed by)",
(unsigned long long)n_pool_checks, (unsigned long long)n_pool_skips,
(unsigned long long)n_pool_miss[0], (unsigned long long)n_pool_miss[1],
(unsigned long long)n_pool_miss[2], (unsigned long long)n_pool_miss[3],
(unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q);
(unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q, (unsigned long long)n_pool_rel);
used = strlen(out);
if (used < n)
snprintf(out + used, n - used, "; %d slot groups holding %llu slots", ngrp_live, (unsigned long long)n_grouped);
@@ -4380,7 +4407,7 @@ void vk_texture_bucket_stats(char *out, size_t n) {
if (used < n)
snprintf(out + used, n - used, "; pool slots hold %d entries, %d of them GPU-owned (stamped in each submission "
"checked by list)", held, gown);
n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = 0;
n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = n_pool_rel = 0;
memset(n_pool_miss, 0, sizeof n_pool_miss);
memset(n_site, 0, sizeof n_site);
memset(bump_count, 0, sizeof bump_count);