diff --git a/runtime/cfg.c b/runtime/cfg.c index a1057dd9..3c6ffad9 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -60,7 +60,7 @@ static const char *const known[] = { "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC", "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", - "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", + "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "POOL_PRESNAP_RUNS", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", diff --git a/runtime/host.h b/runtime/host.h index 3b5928db..63fa2b9a 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -220,6 +220,8 @@ uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64 uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max); // the same without the reset: the bits stay for the next // mem_written_pages (verify checks only) +int mem_watched(uint64_t addr, uint64_t size); // [addr, addr+size) lies inside one of those two ranges, so + // one mem_written_pages call can answer for all of it int mem_freeze(uint64_t a); // makes the 2 MiB region at a (2 MiB aligned, watched, // committed PAGE_READWRITE) read-only until a write or // mem_thaw thaws it; 1 frozen (or already), 0 refused diff --git a/runtime/memory.c b/runtime/memory.c index be2e0839..fa010e77 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -1203,6 +1203,8 @@ uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_ return written_pages(addr, size, pages, max, 0); } +int mem_watched(uint64_t addr, uint64_t size) { return size && watched(addr, size); } + static uint64_t image_bytes(void) { uint64_t n = 0; for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size; diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 3905c530..224ab2cb 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -411,6 +411,51 @@ static int test_import(pool_t *pool) { return 0; } +// The write-watch snapshot of many ranges at once (vk_memory.c vk_mem_snap_note_locked, vk_mem_snap_noted_locked), as +// the texture pool's freshness check asks for it: ranges in three adjacent 2 MiB regions of the heap and in one apart, +// noted in no address order, take one query per run of adjacent regions; afterwards every region has the epoch's +// snapshot, the regions with a CPU write since `since` say so, the one without does not, and the pages are exact. +// The bytes written are the ones already there, so the tests after this one find their memory as it was. +static int test_snap_runs(uint8_t *h) { + static const uint32_t off[4] = {0xe10000, 0xa80000, 0x603000, 0x9f0000}; // regions at 14, 10, 6 and 8 MiB + uint64_t base = (uintptr_t)h; + GCHECK(!(base & 0x1fffff), "the heap at %#llx does not start on a 2 MiB region", (unsigned long long)base); + int left = -1; + uint32_t g0, now; + vk_texture_new_submit(); + uint32_t ep = vk_texture_epoch(); + vk_mem_gen_lock(); + for (int i = 0; i < 4; i++) vk_mem_snap_note_locked(base + off[i], 0x2000, ep); + vk_mem_snap_noted_locked(ep, &left); // what was written before this test + vk_mem_gen_unlock(); + GCHECK(!left, "a run inside the heap was left to the queries by range"); + vk_mem_changed(base + 0x200000, 0x1000, 0, &g0); + volatile uint8_t *w[3] = {h + 0xe10010, h + 0xa80800, h + 0x604000}; + for (int i = 0; i < 3; i++) *w[i] = *w[i]; + vk_texture_new_submit(); + ep = vk_texture_epoch(); + vk_mem_gen_lock(); + for (int i = 0; i < 4; i++) vk_mem_snap_note_locked(base + off[i], 0x2000, ep); + for (int i = 0; i < 4; i++) vk_mem_snap_note_locked(base + off[i], 0x2000, ep); // a region noted twice is asked once + uint64_t q = vk_mem_snap_noted_locked(ep, &left), again = 0; + int why[4]; + for (int i = 0; i < 4; i++) { + why[i] = vk_mem_unchanged_why_locked(base + off[i], 0x2000, g0, ep); + again += vk_mem_snap_range_locked(base + off[i], 0x2000, ep); + } + vk_mem_gen_unlock(); + GCHECK(q == 2 && !left, "four regions in two runs took %llu queries (left %d)", (unsigned long long)q, left); + GCHECK(!again, "%llu regions had no snapshot after the queries by run", (unsigned long long)again); + GCHECK(why[0] == 3 && why[1] == 3 && why[2] == 3 && why[3] == 0, "regions written, written, written, not written " + "answer %d %d %d %d", why[0], why[1], why[2], why[3]); + GCHECK(vk_mem_changed_snap(base + 0x604000, 0x1000, g0, ep, &now) == 1 && + vk_mem_changed_snap(base + 0x603000, 0x1000, g0, ep, &now) == 0 && + vk_mem_changed_snap(base + 0x9f0000, 0x2000, g0, ep, &now) == 0, "the pages written are not reported exactly"); + printf("probe-vkmem: write-watch snapshot by runs: 4 regions in 2 runs took 2 queries; written regions and pages exact\n"); + g_gpu_checks += 5; + return 0; +} + // A GPU write is noted on a pool, and the pool is finalized and made again at its address before the write is // reported: the report must leave alone what the CPU has written to the new pool since. With shadow storage the new // pool's shadow is a copy of guest memory from before those writes and holds no byte of the old write @@ -1184,7 +1229,7 @@ static int test_gpu(void) { .size = 0x40000, .game_mem = h + 0x800000, .span = 0x40000}; pool_t virt = {.magic = NVN_POOL_MAGIC, .flags = 0x2a1, .mem = (uint8_t *)(uintptr_t)(RT_GPU_VA_BASE + 0x200000000ull), .size = 0x80000, .span = 0x80000}; - if (test_import(&pool) || test_readfile(pool.mem + 0x38000) || test_virtual(&phys, &virt)) return 1; + if (test_import(&pool) || test_readfile(pool.mem + 0x38000) || test_snap_runs(h) || test_virtual(&phys, &virt)) return 1; GCHECK(vk_pool_init(&tpool), "vk_pool_init of the texture pool"); if (test_textures(&tpool, &phys)) return 1; pool_t dpool = {.magic = NVN_POOL_MAGIC, .flags = 0x22, .mem = h + 0x500000, .size = 0x100000, .game_mem = h + 0x500000}; diff --git a/runtime/vk.h b/runtime/vk.h index 5c858e9f..001493dd 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -670,6 +670,11 @@ int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint32_t since, ui void vk_mem_gen_lock(void); void vk_mem_gen_unlock(void); uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch); +// The same snapshot for many ranges, with the lock held exclusively: note each range, then one call queries the noted +// regions that have no snapshot in epoch, one query per run of adjacent regions (the number of queries). *left: 1 +// when it left regions without one, for vk_mem_snap_range_locked. +void vk_mem_snap_note_locked(uint64_t addr, uint64_t size, uint32_t epoch); +uint64_t vk_mem_snap_noted_locked(uint32_t epoch, int *left); void vk_sampler_dlss_bias(float bias); // texture LOD bias added for DLSS upscaling (material samplers) void vk_sampler_aniso_live(void); // each present: NX2NAVI_ANISO after a settings change (tiled samplers) uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 1571b8b8..05b50924 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1058,6 +1058,59 @@ uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch) return 0; } +// The snapshot of many ranges at once, for the texture pool's freshness check (vk_texture.c +// vk_texture_pool_fresh_locked): vk_mem_snap_note_locked notes the regions of a range that have no snapshot in epoch +// yet, and vk_mem_snap_noted_locked queries what was noted, one query per run of adjacent regions. +// Asked range by range (vk_mem_snap_range_locked), every such region got a query of its own, because a texture rarely +// spans two regions and the list is in no address order. While the player runs through a town the game writes about 32 +// textures into new storage every four or five frames, a region the CPU wrote is not frozen again for freeze_idle +// submissions, and so 100 to 500 regions of the texture pools were queried one by one at every check, 2.7 checks a +// frame: the check took 0.5 to 1.5 ms against 0.11 ms standing still (5 to 10 queries), about 3 us a query (five +// sessions of one player, RTX 5090, 2026-11-12). About half of a 2 MiB query is the call itself: in a process of its +// own, 400 of them took 727 us and one query over the same 800 MiB 360 us, with 0, 8 or 64 pages written. +// The regions queried and the pages stamped are the same, and every one is stamped before any entry of the list is +// looked at, as before: each entry's regions were queried ahead of its own look. A run is asked in one query only where +// it lies inside one watched reservation (mem_watched): a query across the edge of one fails as a whole, so the regions +// of such a run are left to the queries by range, and *left says that there were some. +static uint32_t snap_noted[SNAP_REGIONS]; // gen_lock: snap_pass when the region is in snap_want +static uint32_t snap_pass = 1; +static uint32_t *snap_want; // gen_lock: the regions noted since the last vk_mem_snap_noted_locked +static int nsnap_want, csnap_want; + +void vk_mem_snap_note_locked(uint64_t addr, uint64_t size, uint32_t epoch) { + if (!size || !epoch) return; + uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; + if (r1 >= SNAP_REGIONS) return; + for (uint64_t r = r0; r <= r1; r++) { + if (snap_at[r] == epoch || snap_noted[r] == snap_pass) continue; + snap_noted[r] = snap_pass; + if (nsnap_want == csnap_want) grow((void **)&snap_want, &csnap_want, sizeof *snap_want); + snap_want[nsnap_want++] = (uint32_t)r; + } +} + +static int snap_want_cmp(const void *a, const void *b) { + uint32_t x = *(const uint32_t *)a, y = *(const uint32_t *)b; + return x < y ? -1 : x > y; +} + +uint64_t vk_mem_snap_noted_locked(uint32_t epoch, int *left) { + uint64_t calls = 0; + *left = 0; + if (nsnap_want > 1) qsort(snap_want, (size_t)nsnap_want, sizeof *snap_want, snap_want_cmp); + for (int i = 0, k; i < nsnap_want; i = k + 1) { + for (k = i; k + 1 < nsnap_want && snap_want[k + 1] == snap_want[k] + 1;) k++; + uint64_t r0 = snap_want[i], r1 = snap_want[k]; + if (r1 > r0 && !mem_watched(r0 << SNAP_SHIFT, (r1 - r0 + 1) << SNAP_SHIFT)) *left = 1; + else calls += snap_regions(r0, r1, epoch); + } + nsnap_want = 0; + if (!++snap_pass) memset(snap_noted, 0, sizeof snap_noted), snap_pass = 1; + n_snap_calls += calls; + return calls; +} + void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) { *checks = __atomic_exchange_n(&n_snap_checks, 0, __ATOMIC_RELAXED); *calls = __atomic_exchange_n(&n_snap_calls, 0, __ATOMIC_RELAXED); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index a033c995..0554dae7 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1440,10 +1440,15 @@ static void rel_unlink(Tex *e) { // that the check of an entry whose 2 MiB regions took no write since (vk_mem_unchanged_locked, one shared lock for the // whole list) touches neither the entry nor the lock: 5,600 entries took 0.77 ms a submission one by one. Texture lock // held. -typedef struct { Tex *e; uint64_t addr, size; uint32_t since; int ok; } PEnt; // ok: uploaded as of since +// rel is the entry's released state (mark_released, entry_of's take-back): the check does nothing for a released +// entry, and with the state here it does not touch the entry to find that out. On a card with room the sweep keeps up +// to 8,192 released entries, they stay listed, and since stays as it was at their release, so each one whose regions +// took a write later missed the region check at every submission from then on: 1,000 to 6,300 entries a check in +// Kakariko and Castle Town, a third to a half of the list. +typedef struct { Tex *e; uint64_t addr, size; uint32_t since; uint8_t ok, rel; } PEnt; // ok: uploaded as of since static PEnt *pent; static int npent, cpent, *pq, cpq; -static uint64_t n_pool_checks, n_pool_skips; +static uint64_t n_pool_checks, n_pool_skips, n_pool_rel; // The region check's misses (vk_mem_unchanged_why_locked): not uploaded, a region without this epoch's snapshot yet, one // untracked, one written since, nothing to check; and the snapshot queries its own pass made (NX2NAVI_POOL_PRESNAP). static uint64_t n_pool_miss[5], n_presnap_q; @@ -1534,7 +1539,7 @@ int vk_texture_pool_note_locked(int id, uint32_t *ent) { if (!p || !q) rt_fatal("vk: out of memory for the texture pool's entry list"); pent = p, pq = q, cpent = cpq = n; } - pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0}; + pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0, (uint8_t)e->released}; e->pidx = npent; } *ent = pool_ref(e); @@ -1553,6 +1558,14 @@ static int pool_presnap(void) { if (mode < 0) mode = rt_knob("POOL_PRESNAP", "1"); return RT_KNOB_ON(mode); } +// NX2NAVI_POOL_PRESNAP_RUNS=1 (default), 0, ab: those snapshots are taken for the whole list first, one write-watch +// query per run of adjacent regions (vk_memory.c vk_mem_snap_noted_locked), instead of one per region as the list comes +// to it. The log's "snapshot queries" of the pool freshness check count either. +static int pool_presnap_runs(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("POOL_PRESNAP_RUNS", "1"); + return RT_KNOB_ON(mode); +} static void pool_made(uint64_t t0); // below, with the pool's creation budget // The submission being recorded checked the pool's entries by this list: for the idle clock it walked the pool @@ -1561,13 +1574,24 @@ static int pool_by_list; int vk_texture_pool_fresh_locked(void) { int nq = 0, ps = pool_presnap(); + uint64_t rel0 = n_pool_rel; pool_by_list = 1; if (ps) vk_mem_gen_lock(); else vk_mem_gen_lock_shared(); + int by_range = ps; // each entry's regions are asked for as the list comes to it + if (ps && pool_presnap_runs()) { + for (int i = 0; i < npent; i++) + if (pent[i].ok && !pent[i].rel) vk_mem_snap_note_locked(pent[i].addr, pent[i].size, submit_epoch); + n_presnap_q += vk_mem_snap_noted_locked(submit_epoch, &by_range); + } for (int i = 0; i < npent; i++) { const PEnt *p = &pent[i]; + if (p->rel) { // released: nothing to check (the loop below would pass it by) + n_pool_rel++; + continue; + } if (p->ok) { - if (ps) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch); + if (by_range) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch); int why = vk_mem_unchanged_why_locked(p->addr, p->size, p->since, submit_epoch); if (!why) continue; n_pool_miss[why]++; @@ -1579,6 +1603,7 @@ int vk_texture_pool_fresh_locked(void) { if (ps) vk_mem_gen_unlock(); else vk_mem_gen_unlock_shared(); n_pool_skips += (uint64_t)(npent - nq); + n_pool_skips -= n_pool_rel - rel0; for (int k = nq; k-- > 0;) { // from the end: an entry leaving swaps the last one into its place int i = pq[k]; PEnt *p = &pent[i]; @@ -3909,6 +3934,7 @@ static Tex *entry_of(const tex_t *t) { uint64_t gap = e->samp_frame - e->rel_frame; if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); + if (e->pidx) pent[e->pidx - 1].rel = 0; // the pool's check looks at it again e->overwritten = 0; e->og_backs++, e->og_back_frame = e->samp_frame; if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work @@ -3971,6 +3997,7 @@ static int wb_wanted(const Tex *e) { return e->t.L.size <= wb_limit || e->run < static void mark_released(Tex *e) { // tex_lock held; e is not released flight_ev(e, FLIGHT_TEX_RELEASE, 0, 0); e->released = 1; + if (e->pidx) pent[e->pidx - 1].rel = 1; // the pool's check passes it by (vk_texture_pool_fresh_locked) e->rel_epoch = submit_epoch; e->rel_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); nreleased++; @@ -4365,11 +4392,11 @@ void vk_texture_bucket_stats(char *out, size_t n) { if (used < n) snprintf(out + used, n - used, "; pool freshness check: %llu entries checked, %llu passed by their regions (missed: " "%llu not uploaded, %llu a region without a snapshot yet, %llu untracked, %llu written since, %llu " - "nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP)", + "nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP; %llu released entries passed by)", (unsigned long long)n_pool_checks, (unsigned long long)n_pool_skips, (unsigned long long)n_pool_miss[0], (unsigned long long)n_pool_miss[1], (unsigned long long)n_pool_miss[2], (unsigned long long)n_pool_miss[3], - (unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q); + (unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q, (unsigned long long)n_pool_rel); used = strlen(out); if (used < n) snprintf(out + used, n - used, "; %d slot groups holding %llu slots", ngrp_live, (unsigned long long)n_grouped); @@ -4380,7 +4407,7 @@ void vk_texture_bucket_stats(char *out, size_t n) { if (used < n) snprintf(out + used, n - used, "; pool slots hold %d entries, %d of them GPU-owned (stamped in each submission " "checked by list)", held, gown); - n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = 0; + n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = n_pool_rel = 0; memset(n_pool_miss, 0, sizeof n_pool_miss); memset(n_site, 0, sizeof n_site); memset(bump_count, 0, sizeof bump_count);