From 65dacfa7c7ee4a4b7e104097ac547e89d5357dec Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 10:23:10 -0500 Subject: [PATCH] Ask the write watch once per run of regions in the texture pool's check, and pass released entries by. At every submission that binds the texture pools the replay checks each listed texture for CPU writes, and takes the write-watch snapshot of the 2 MiB regions that have none yet as the list comes to them: one query a region, since the list is in no address order. Standing still that is 5 to 10 queries and the check takes 0.11 ms. While the player runs through Kakariko or Castle Town the game writes about 32 textures into new storage every four or five frames, a region the CPU wrote is not frozen again for 1,800 submissions, and the check asked for 100 to 500 regions one by one, 2.7 times a frame: 0.5 to 1.5 ms a check, about 3 us a query, in five sessions of one player. The regions are now collected for the whole list first and asked in one query per run of adjacent regions. On this machine 400 queries of 2 MiB take 727 us and one query over the same 800 MiB takes 360 us. The same regions are queried and the same pages stamped, all of them before the list is looked at, as each entry's were before its own look; a run across the edge of a watched reservation is left to the queries by range. pool_presnap_runs = 0 asks region by region as before. A released entry stays in the list until it is destroyed, the check does nothing for it, and the stamp it compares stays as it was at the release, so one whose region took a write later missed the region check at every submission from then on and was fetched only to be passed by: 1,000 to 6,300 entries a check, with up to 8,192 released entries kept. The list now carries the released state, set at the release and cleared at the take-back, and the check passes such an entry by without touching it or asking for its regions. The pool log's freshness line counts the released entries passed by, and its snapshot queries count the queries either way. --probe-vkmem has a case for the queries by run. Co-Authored-By: Claude Fable 5.2 --- runtime/cfg.c | 2 +- runtime/host.h | 2 ++ runtime/memory.c | 2 ++ runtime/tex_probe.c | 47 ++++++++++++++++++++++++++++++++++++++- runtime/vk.h | 5 +++++ runtime/vk_memory.c | 53 ++++++++++++++++++++++++++++++++++++++++++++ runtime/vk_texture.c | 41 ++++++++++++++++++++++++++++------ 7 files changed, 143 insertions(+), 9 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index a1057dd9..3c6ffad9 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -60,7 +60,7 @@ static const char *const known[] = { "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC", "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", - "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", + "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "POOL_PRESNAP_RUNS", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", diff --git a/runtime/host.h b/runtime/host.h index 3b5928db..63fa2b9a 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -220,6 +220,8 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64 uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max); // the same without the reset: the bits stay for the next // mem_written_pages (verify checks only) +int mem_watched(uint64_t addr, uint64_t size); // [addr, addr+size) lies inside one of those two ranges, so + // one mem_written_pages call can answer for all of it int mem_freeze(uint64_t a); // makes the 2 MiB region at a (2 MiB aligned, watched, // committed PAGE_READWRITE) read-only until a write or // mem_thaw thaws it; 1 frozen (or already), 0 refused diff --git a/runtime/memory.c b/runtime/memory.c index be2e0839..fa010e77 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -1203,6 +1203,8 @@ uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_ return written_pages(addr, size, pages, max, 0); } +int mem_watched(uint64_t addr, uint64_t size) { return size && watched(addr, size); } + static uint64_t image_bytes(void) { uint64_t n = 0; for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size; diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 3905c530..224ab2cb 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -411,6 +411,51 @@ static int test_import(pool_t *pool) { return 0; } +// The write-watch snapshot of many ranges at once (vk_memory.c vk_mem_snap_note_locked, vk_mem_snap_noted_locked), as +// the texture pool's freshness check asks for it: ranges in three adjacent 2 MiB regions of the heap and in one apart, +// noted in no address order, take one query per run of adjacent regions; afterwards every region has the epoch's +// snapshot, the regions with a CPU write since `since` say so, the one without does not, and the pages are exact. +// The bytes written are the ones already there, so the tests after this one find their memory as it was. +static int test_snap_runs(uint8_t *h) { + static const uint32_t off[4] = {0xe10000, 0xa80000, 0x603000, 0x9f0000}; // regions at 14, 10, 6 and 8 MiB + uint64_t base = (uintptr_t)h; + GCHECK(!(base & 0x1fffff), "the heap at %#llx does not start on a 2 MiB region", (unsigned long long)base); + int left = -1; + uint32_t g0, now; + vk_texture_new_submit(); + uint32_t ep = vk_texture_epoch(); + vk_mem_gen_lock(); + for (int i = 0; i < 4; i++) vk_mem_snap_note_locked(base + off[i], 0x2000, ep); + vk_mem_snap_noted_locked(ep, &left); // what was written before this test + vk_mem_gen_unlock(); + GCHECK(!left, "a run inside the heap was left to the queries by range"); + vk_mem_changed(base + 0x200000, 0x1000, 0, &g0); + volatile uint8_t *w[3] = {h + 0xe10010, h + 0xa80800, h + 0x604000}; + for (int i = 0; i < 3; i++) *w[i] = *w[i]; + vk_texture_new_submit(); + ep = vk_texture_epoch(); + vk_mem_gen_lock(); + for (int i = 0; i < 4; i++) vk_mem_snap_note_locked(base + off[i], 0x2000, ep); + for (int i = 0; i < 4; i++) vk_mem_snap_note_locked(base + off[i], 0x2000, ep); // a region noted twice is asked once + uint64_t q = vk_mem_snap_noted_locked(ep, &left), again = 0; + int why[4]; + for (int i = 0; i < 4; i++) { + why[i] = vk_mem_unchanged_why_locked(base + off[i], 0x2000, g0, ep); + again += vk_mem_snap_range_locked(base + off[i], 0x2000, ep); + } + vk_mem_gen_unlock(); + GCHECK(q == 2 && !left, "four regions in two runs took %llu queries (left %d)", (unsigned long long)q, left); + GCHECK(!again, "%llu regions had no snapshot after the queries by run", (unsigned long long)again); + GCHECK(why[0] == 3 && why[1] == 3 && why[2] == 3 && why[3] == 0, "regions written, written, written, not written " + "answer %d %d %d %d", why[0], why[1], why[2], why[3]); + GCHECK(vk_mem_changed_snap(base + 0x604000, 0x1000, g0, ep, &now) == 1 && + vk_mem_changed_snap(base + 0x603000, 0x1000, g0, ep, &now) == 0 && + vk_mem_changed_snap(base + 0x9f0000, 0x2000, g0, ep, &now) == 0, "the pages written are not reported exactly"); + printf("probe-vkmem: write-watch snapshot by runs: 4 regions in 2 runs took 2 queries; written regions and pages exact\n"); + g_gpu_checks += 5; + return 0; +} + // A GPU write is noted on a pool, and the pool is finalized and made again at its address before the write is // reported: the report must leave alone what the CPU has written to the new pool since. With shadow storage the new // pool's shadow is a copy of guest memory from before those writes and holds no byte of the old write @@ -1184,7 +1229,7 @@ static int test_gpu(void) { .size = 0x40000, .game_mem = h + 0x800000, .span = 0x40000}; pool_t virt = {.magic = NVN_POOL_MAGIC, .flags = 0x2a1, .mem = (uint8_t *)(uintptr_t)(RT_GPU_VA_BASE + 0x200000000ull), .size = 0x80000, .span = 0x80000}; - if (test_import(&pool) || test_readfile(pool.mem + 0x38000) || test_virtual(&phys, &virt)) return 1; + if (test_import(&pool) || test_readfile(pool.mem + 0x38000) || test_snap_runs(h) || test_virtual(&phys, &virt)) return 1; GCHECK(vk_pool_init(&tpool), "vk_pool_init of the texture pool"); if (test_textures(&tpool, &phys)) return 1; pool_t dpool = {.magic = NVN_POOL_MAGIC, .flags = 0x22, .mem = h + 0x500000, .size = 0x100000, .game_mem = h + 0x500000}; diff --git a/runtime/vk.h b/runtime/vk.h index 5c858e9f..001493dd 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -670,6 +670,11 @@ int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint32_t since, ui void vk_mem_gen_lock(void); void vk_mem_gen_unlock(void); uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch); +// The same snapshot for many ranges, with the lock held exclusively: note each range, then one call queries the noted +// regions that have no snapshot in epoch, one query per run of adjacent regions (the number of queries). *left: 1 +// when it left regions without one, for vk_mem_snap_range_locked. +void vk_mem_snap_note_locked(uint64_t addr, uint64_t size, uint32_t epoch); +uint64_t vk_mem_snap_noted_locked(uint32_t epoch, int *left); void vk_sampler_dlss_bias(float bias); // texture LOD bias added for DLSS upscaling (material samplers) void vk_sampler_aniso_live(void); // each present: NX2NAVI_ANISO after a settings change (tiled samplers) uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 1571b8b8..05b50924 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1058,6 +1058,59 @@ uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch) return 0; } +// The snapshot of many ranges at once, for the texture pool's freshness check (vk_texture.c +// vk_texture_pool_fresh_locked): vk_mem_snap_note_locked notes the regions of a range that have no snapshot in epoch +// yet, and vk_mem_snap_noted_locked queries what was noted, one query per run of adjacent regions. +// Asked range by range (vk_mem_snap_range_locked), every such region got a query of its own, because a texture rarely +// spans two regions and the list is in no address order. While the player runs through a town the game writes about 32 +// textures into new storage every four or five frames, a region the CPU wrote is not frozen again for freeze_idle +// submissions, and so 100 to 500 regions of the texture pools were queried one by one at every check, 2.7 checks a +// frame: the check took 0.5 to 1.5 ms against 0.11 ms standing still (5 to 10 queries), about 3 us a query (five +// sessions of one player, RTX 5090, 2026-11-12). About half of a 2 MiB query is the call itself: in a process of its +// own, 400 of them took 727 us and one query over the same 800 MiB 360 us, with 0, 8 or 64 pages written. +// The regions queried and the pages stamped are the same, and every one is stamped before any entry of the list is +// looked at, as before: each entry's regions were queried ahead of its own look. A run is asked in one query only where +// it lies inside one watched reservation (mem_watched): a query across the edge of one fails as a whole, so the regions +// of such a run are left to the queries by range, and *left says that there were some. +static uint32_t snap_noted[SNAP_REGIONS]; // gen_lock: snap_pass when the region is in snap_want +static uint32_t snap_pass = 1; +static uint32_t *snap_want; // gen_lock: the regions noted since the last vk_mem_snap_noted_locked +static int nsnap_want, csnap_want; + +void vk_mem_snap_note_locked(uint64_t addr, uint64_t size, uint32_t epoch) { + if (!size || !epoch) return; + uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; + if (r1 >= SNAP_REGIONS) return; + for (uint64_t r = r0; r <= r1; r++) { + if (snap_at[r] == epoch || snap_noted[r] == snap_pass) continue; + snap_noted[r] = snap_pass; + if (nsnap_want == csnap_want) grow((void **)&snap_want, &csnap_want, sizeof *snap_want); + snap_want[nsnap_want++] = (uint32_t)r; + } +} + +static int snap_want_cmp(const void *a, const void *b) { + uint32_t x = *(const uint32_t *)a, y = *(const uint32_t *)b; + return x < y ? -1 : x > y; +} + +uint64_t vk_mem_snap_noted_locked(uint32_t epoch, int *left) { + uint64_t calls = 0; + *left = 0; + if (nsnap_want > 1) qsort(snap_want, (size_t)nsnap_want, sizeof *snap_want, snap_want_cmp); + for (int i = 0, k; i < nsnap_want; i = k + 1) { + for (k = i; k + 1 < nsnap_want && snap_want[k + 1] == snap_want[k] + 1;) k++; + uint64_t r0 = snap_want[i], r1 = snap_want[k]; + if (r1 > r0 && !mem_watched(r0 << SNAP_SHIFT, (r1 - r0 + 1) << SNAP_SHIFT)) *left = 1; + else calls += snap_regions(r0, r1, epoch); + } + nsnap_want = 0; + if (!++snap_pass) memset(snap_noted, 0, sizeof snap_noted), snap_pass = 1; + n_snap_calls += calls; + return calls; +} + void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) { *checks = __atomic_exchange_n(&n_snap_checks, 0, __ATOMIC_RELAXED); *calls = __atomic_exchange_n(&n_snap_calls, 0, __ATOMIC_RELAXED); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index a033c995..0554dae7 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1440,10 +1440,15 @@ static void rel_unlink(Tex *e) { // that the check of an entry whose 2 MiB regions took no write since (vk_mem_unchanged_locked, one shared lock for the // whole list) touches neither the entry nor the lock: 5,600 entries took 0.77 ms a submission one by one. Texture lock // held. -typedef struct { Tex *e; uint64_t addr, size; uint32_t since; int ok; } PEnt; // ok: uploaded as of since +// rel is the entry's released state (mark_released, entry_of's take-back): the check does nothing for a released +// entry, and with the state here it does not touch the entry to find that out. On a card with room the sweep keeps up +// to 8,192 released entries, they stay listed, and since stays as it was at their release, so each one whose regions +// took a write later missed the region check at every submission from then on: 1,000 to 6,300 entries a check in +// Kakariko and Castle Town, a third to a half of the list. +typedef struct { Tex *e; uint64_t addr, size; uint32_t since; uint8_t ok, rel; } PEnt; // ok: uploaded as of since static PEnt *pent; static int npent, cpent, *pq, cpq; -static uint64_t n_pool_checks, n_pool_skips; +static uint64_t n_pool_checks, n_pool_skips, n_pool_rel; // The region check's misses (vk_mem_unchanged_why_locked): not uploaded, a region without this epoch's snapshot yet, one // untracked, one written since, nothing to check; and the snapshot queries its own pass made (NX2NAVI_POOL_PRESNAP). static uint64_t n_pool_miss[5], n_presnap_q; @@ -1534,7 +1539,7 @@ int vk_texture_pool_note_locked(int id, uint32_t *ent) { if (!p || !q) rt_fatal("vk: out of memory for the texture pool's entry list"); pent = p, pq = q, cpent = cpq = n; } - pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0}; + pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0, (uint8_t)e->released}; e->pidx = npent; } *ent = pool_ref(e); @@ -1553,6 +1558,14 @@ static int pool_presnap(void) { if (mode < 0) mode = rt_knob("POOL_PRESNAP", "1"); return RT_KNOB_ON(mode); } +// NX2NAVI_POOL_PRESNAP_RUNS=1 (default), 0, ab: those snapshots are taken for the whole list first, one write-watch +// query per run of adjacent regions (vk_memory.c vk_mem_snap_noted_locked), instead of one per region as the list comes +// to it. The log's "snapshot queries" of the pool freshness check count either. +static int pool_presnap_runs(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("POOL_PRESNAP_RUNS", "1"); + return RT_KNOB_ON(mode); +} static void pool_made(uint64_t t0); // below, with the pool's creation budget // The submission being recorded checked the pool's entries by this list: for the idle clock it walked the pool @@ -1561,13 +1574,24 @@ static int pool_by_list; int vk_texture_pool_fresh_locked(void) { int nq = 0, ps = pool_presnap(); + uint64_t rel0 = n_pool_rel; pool_by_list = 1; if (ps) vk_mem_gen_lock(); else vk_mem_gen_lock_shared(); + int by_range = ps; // each entry's regions are asked for as the list comes to it + if (ps && pool_presnap_runs()) { + for (int i = 0; i < npent; i++) + if (pent[i].ok && !pent[i].rel) vk_mem_snap_note_locked(pent[i].addr, pent[i].size, submit_epoch); + n_presnap_q += vk_mem_snap_noted_locked(submit_epoch, &by_range); + } for (int i = 0; i < npent; i++) { const PEnt *p = &pent[i]; + if (p->rel) { // released: nothing to check (the loop below would pass it by) + n_pool_rel++; + continue; + } if (p->ok) { - if (ps) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch); + if (by_range) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch); int why = vk_mem_unchanged_why_locked(p->addr, p->size, p->since, submit_epoch); if (!why) continue; n_pool_miss[why]++; @@ -1579,6 +1603,7 @@ int vk_texture_pool_fresh_locked(void) { if (ps) vk_mem_gen_unlock(); else vk_mem_gen_unlock_shared(); n_pool_skips += (uint64_t)(npent - nq); + n_pool_skips -= n_pool_rel - rel0; for (int k = nq; k-- > 0;) { // from the end: an entry leaving swaps the last one into its place int i = pq[k]; PEnt *p = &pent[i]; @@ -3909,6 +3934,7 @@ static Tex *entry_of(const tex_t *t) { uint64_t gap = e->samp_frame - e->rel_frame; if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); + if (e->pidx) pent[e->pidx - 1].rel = 0; // the pool's check looks at it again e->overwritten = 0; e->og_backs++, e->og_back_frame = e->samp_frame; if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work @@ -3971,6 +3997,7 @@ static int wb_wanted(const Tex *e) { return e->t.L.size <= wb_limit || e->run < static void mark_released(Tex *e) { // tex_lock held; e is not released flight_ev(e, FLIGHT_TEX_RELEASE, 0, 0); e->released = 1; + if (e->pidx) pent[e->pidx - 1].rel = 1; // the pool's check passes it by (vk_texture_pool_fresh_locked) e->rel_epoch = submit_epoch; e->rel_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); nreleased++; @@ -4365,11 +4392,11 @@ void vk_texture_bucket_stats(char *out, size_t n) { if (used < n) snprintf(out + used, n - used, "; pool freshness check: %llu entries checked, %llu passed by their regions (missed: " "%llu not uploaded, %llu a region without a snapshot yet, %llu untracked, %llu written since, %llu " - "nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP)", + "nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP; %llu released entries passed by)", (unsigned long long)n_pool_checks, (unsigned long long)n_pool_skips, (unsigned long long)n_pool_miss[0], (unsigned long long)n_pool_miss[1], (unsigned long long)n_pool_miss[2], (unsigned long long)n_pool_miss[3], - (unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q); + (unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q, (unsigned long long)n_pool_rel); used = strlen(out); if (used < n) snprintf(out + used, n - used, "; %d slot groups holding %llu slots", ngrp_live, (unsigned long long)n_grouped); @@ -4380,7 +4407,7 @@ void vk_texture_bucket_stats(char *out, size_t n) { if (used < n) snprintf(out + used, n - used, "; pool slots hold %d entries, %d of them GPU-owned (stamped in each submission " "checked by list)", held, gown); - n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = 0; + n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = n_pool_rel = 0; memset(n_pool_miss, 0, sizeof n_pool_miss); memset(n_site, 0, sizeof n_site); memset(bump_count, 0, sizeof bump_count);