From 7b009ffc648efa13894ed6dd4b563be3f0d903b0 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 23:14:03 -0500 Subject: [PATCH 1/4] Let the wait after a frame's first submission cover the write-backs under shadow storage. With shadow storage (the Linux default) what the GPU wrote reaches guest memory in the fence's completion callback, later than an import's store on the GPU. The wait after a frame's first submission returned at the executed timeline, before that, so the game rewrote what those passes had written and the write-back then put the GPU's older bytes back over the rewrite. The whole 3D scene came out brighter for single frames, each of them a frame whose main work was held at its CPU event wait and replayed late. Standing outdoors with NX2NAVI_SHADOW=1 on an RTX 5090: 695, 624, 443, 323 and 2 flashed frames of 1,800 in five runs with the split, none in eight runs without it. The frame rate did not drop (74.6 fps median with the fix; runs of the build before it gave 71 to 72). NX2NAVI_WB_SPLIT=2 keeps the split with shadow storage, to measure the fault. That this is the flicker a player reported on Linux (RX 7800 XT, RADV) is not confirmed on their machine; NX2NAVI_WB_SPLIT=0 gives 0.7.1 the same wait there. Co-Authored-By: Claude Fable 5.2 --- docs/building.md | 5 ++++- runtime/cfg.c | 2 +- runtime/vk_sync.c | 25 ++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/building.md b/docs/building.md index d41af214..6aae4898 100644 --- a/docs/building.md +++ b/docs/building.md @@ -272,7 +272,10 @@ writes come back at its fence. Pages both sides write keep a base copy, so each a VRAM mirror's write-back merge marks the pages it changed, and only those are compared at the fence; a lookup that reads guest memory itself (the uniform snapshot, `vk_mem_lookup`) does not queue its pool for the copy-in. In Kokiri Forest (1.1.1, RTX 4090) that costs about 1 ms a frame (copy-in 0.5 ms, write-back 0.6 ms), and the copies hold about -5 GB. `NX2NAVI_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as +5 GB. The GPU's bytes reach guest memory in the fence's completion callback, later than an import's, so the wait +after a frame's first submission covers the write-backs there (`runtime/vk_sync.c`, "the executed timeline"): the game +rewrites what those passes wrote without a fence, and a write-back that landed after the rewrite left the GPU adding +light onto its own last output, the whole scene brighter for a frame. `NX2NAVI_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as well: the shadow copies are chosen for the measured cost above, not for a missing extension. The device address table (`runtime/vk_gpa.c`) names a pool's shadow copy as it names an import, so shaders follow guest diff --git a/runtime/cfg.c b/runtime/cfg.c index 167c8a64..2e16200a 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -279,7 +279,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 9e7ee07a..d28dadf3 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -82,12 +82,35 @@ static int hold_max = -1; // write-backs follow in a part of their own, which signals the timeline as before. A submission without that split // signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among // them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's. +// Not with shadow storage (vk_memory.c): there the game does need the write-backs before it goes on. With an import a +// write-back stores into guest memory on the GPU, right behind the passes. With a shadow copy it stores into the copy, +// and the bytes reach guest memory in the fence's completion callback (shadow_wb_cb), after the write-back part, the +// fence and the callbacks queued before it, while the game is already rewriting for its next frame what those passes +// wrote. A write-back that lands after such a rewrite puts the GPU's older bytes over it in guest memory and moves the +// page's base copy along, so the next copy-in finds nothing the CPU changed; and a rewrite with the value the base +// copy holds (a reset to zero) is no difference at any time. The GPU then goes on from its own last output, as the +// game's lighting accumulators do, which the shaders add to and the CPU resets. +// Measured: with the split the whole 3D scene came out brighter for single frames (3.8 of 255, the interface as it +// was), each of them a frame whose main work was held at its CPU event wait and replayed late, together with the next +// frame's: standing outdoors, 695, 624, 443, 323 and 2 of 1,800 frames in five runs, and none in eight runs without +// the split (RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10). A player on Linux (RX 7800 XT, RADV) saw the open field lit +// in one frame and flat in the next; that it is the same fault is not confirmed on their machine. Which words the +// write-backs overtook was not traced: NX2NAVI_SHADOW_CHECK counts no byte that both sides changed to different +// values, which fits a reset to the base copy's value. +// NX2NAVI_WB_SPLIT=2 keeps the split with shadow storage too, to measure that fault. static VkSemaphore exec_tl; static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl static int wb_split = -1; int vk_sync_wb_split(void) { - if (wb_split < 0) wb_split = cfg_bool("NX2NAVI_WB_SPLIT"); + if (wb_split < 0) { + const char *e = cfg_get("NX2NAVI_WB_SPLIT"); + int forced = e && !strcmp(e, "2"); + wb_split = cfg_bool("NX2NAVI_WB_SPLIT") && (forced || !vk_mem_shadow_storage()); + if (vk_mem_shadow_storage()) + rt_log("vk: the wait after a frame's first submission %s (shadow storage; NX2NAVI_WB_SPLIT=2 leaves them out)", + wb_split ? "leaves out its write-backs" : "covers its write-backs and their copies into guest memory"); + } return wb_split && exec_tl != VK_NULL_HANDLE; } From 2f6ff17651e93feb0e4b0a949f444f4f4ed5e57d Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 23:14:10 -0500 Subject: [PATCH 2/4] Give each part of the import check in --probe-vkmem its own write-watch epoch. Under shadow storage the copy-in reads each region's write-watch snapshot once per submission. The check ran its two parts in one epoch, so the second part's copy-in did not see the CPU writes made after the first, and the probe failed with NX2NAVI_SHADOW=1 ("GPU read of CPU-written heap differs"). Each part now starts a new epoch, as a submission does. Co-Authored-By: Claude Fable 5.2 --- runtime/tex_probe.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index b568e790..1bee0375 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -352,7 +352,11 @@ static int test_import(pool_t *pool) { vk_mem_resolve(a + 0x1000, 0x20000, &sp); mem_written_pages(a, pool->size, pages, 64); vk_mem_gpu_pending(a + 0x1000, 0x20000); - vk_mem_shadow_sync(); // as a submit does: the part's CPU inputs go in before its GPU work + // As a submit does: a new write-watch epoch (the copy-in reads each region's snapshot once per submission, so a + // second sync in the same epoch would not see the CPU writes made since the first), then the part's CPU inputs + // go in before its GPU work. + vk_texture_new_submit(); + vk_mem_shadow_sync(); copy_wait(sb, 0, sp.buffer, sp.offset, 0x20000); vk_gpu_written(a + 0x1000, 0x20000); GCHECK(!memcmp(m + 0x1000, s, 0x20000), "GPU copy into the imported heap range differs from the source"); @@ -370,6 +374,7 @@ static int test_import(pool_t *pool) { // A GPU write noted for the part being submitted must not keep the part's CPU inputs out: a buffer the shaders // add to and the CPU resets (the game's lighting accumulators) depends on the reset reaching the GPU first. vk_mem_gpu_pending(a + 0x30000, 0x4000); + vk_texture_new_submit(); vk_mem_shadow_sync(); copy_wait(sp.buffer, sp.offset, sb, 0, 0x4000); vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); From 175c1947deef6bc2faef6c9c59c373fe2fd9bb20 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 23:14:42 -0500 Subject: [PATCH 3/4] Pass over the pools with nothing written when a part that uses the device address table syncs the shadows. Under shadow storage the prologue of every part whose shaders use the table brings the shadow of every pool up to date. It ran the full per-pool sync for each of them, 10,700 times a frame outdoors (3,772 pools, about three such parts), though the write-watch snapshots of all pools are taken at once just before. A pool whose regions hold no stamp newer than its shadow is now passed over with a compare per region: 1,500 pool syncs a frame are left, and the copy-ins went from 1.32 to 1.01 ms a frame of the replay (RTX 5090 with NX2NAVI_SHADOW=1, one standing run each way). Co-Authored-By: Claude Fable 5.2 --- runtime/vk_memory.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 1c17fd59..fb1f6e3b 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2074,6 +2074,22 @@ int vk_mem_shadow_storage(void) { return shadow_on; } static int gpa_shadow_pool(const PoolRec *r) { return r->kind == POOL_IMPORT && r->gpa_rec && r->smap; } +// gen_lock held: nothing of the pool was stamped since its shadow's last copy-in. Its regions have this epoch's +// write-watch snapshot (vk_mem_gpa_sync takes them for all pools at once), none of them is untracked, and none holds a +// stamp newer than the shadow's sgen, which then stays as it is. With cpu the stamps are the CPU-side tables', as +// shadow_sync_pool reads them. A table sync ran shadow_sync_pool for every pool at every part that uses the table, +// 10,700 times a frame outdoors (3,772 pools, about three such parts), each with its own timer reads, snapshot walk +// and region loop; with this test 1,500 pool syncs a frame are left, and the copy-ins went from 1.32 to 1.01 ms a +// frame (RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10). +static int shadow_pool_clean(const PoolRec *r, uint32_t epoch, int cpu) { + uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT; + if (!epoch || r1 >= SNAP_REGIONS) return 0; + const uint32_t *mx = cpu ? creg_max : reg_max; + for (uint64_t k = r0; k <= r1; k++) + if (snap_at[k] != epoch || snap_untracked[k] || mx[k] > r->sgen) return 0; + return 1; +} + void vk_mem_gpa_sync(void) { if (!gpa_part_used) return; gpa_part_used = 0; @@ -2101,7 +2117,7 @@ void vk_mem_gpa_sync(void) { if (have) s_queries += snap_regions(run0, run1, epoch); for (int i = 0; i < pool_index.n; i++) { PoolRec *r = pool_index.e[i].buf; - if (gpa_shadow_pool(r)) shadow_sync_pool(r, epoch, cpu), s_syncs++; + if (gpa_shadow_pool(r) && !shadow_pool_clean(r, epoch, cpu)) shadow_sync_pool(r, epoch, cpu), s_syncs++; } plat_mutex_unlock(&gen_lock); // The uploads read the shadows just brought up to date, and the prologue records them ahead of the part, after From 5b1413d8175c574a510980cc64c1c33e8597ddd3 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 23:15:02 -0500 Subject: [PATCH 4/4] Count what the shadow copy-ins overwrite and cost, and keep stores through the device address table behind a switch. NX2NAVI_SHADOW_CHECK=1 logs with the shadow line, every 60 frames: the whole copy-ins that overwrote a store a shader made through the device address table, and the pools they fell on; the write-backs that moved bytes into a page the CPU had written since its copy-in; the bytes both sides changed since their base copy; what the copy-ins cost by kind, and which pools and flag sets took them. The stores are found by content: a whole copy-in leaves a hash of the page, and the next one compares the shadow page with it first. NX2NAVI_SHADOW_TABLE_STORES=1 keeps such stores: a page found stored into gets a base copy, so copy-ins take only the CPU's bytes, and the fence of every submission that uses the table writes the stores back to guest memory. --probe-vkmem checks it with a GPU write that nothing notes. It is off by default. The overwrite it removes was the suspected cause of the flicker on Linux and is not: standing outdoors (RTX 5090 with NX2NAVI_SHADOW=1) it happens exactly four times a frame, on ten pages in pools of 20 and 44 KiB, with the copy that precedes the frame's stores, and no frame came out wrong with it or without it in 1,800 frames each; the hashes cost 1.8 ms a frame of the replay (67 against 75 fps); and a count the CPU resets before the write-back has landed would keep counting, where the whole copy-in resets it every time. Co-Authored-By: Claude Fable 5.2 --- docs/building.md | 17 ++- runtime/cfg.c | 4 +- runtime/tex_probe.c | 96 ++++++++++++- runtime/vk.h | 6 +- runtime/vk_gpa.c | 7 +- runtime/vk_memory.c | 331 +++++++++++++++++++++++++++++++++++++++++++- runtime/vk_sync.c | 2 +- 7 files changed, 449 insertions(+), 14 deletions(-) diff --git a/docs/building.md b/docs/building.md index 6aae4898..558368e6 100644 --- a/docs/building.md +++ b/docs/building.md @@ -283,9 +283,20 @@ pointers into either. A shader that reads through the table resolves nothing, so the copy-in: a part whose pipelines declare the table is marked at its draws and dispatches, and its prologue brings the shadow of every pool in the table up to date with the pages the CPU wrote and queues the same pages for those pools' VRAM mirrors (`vk_memory.c` `vk_mem_gpa_sync`). That copies the written pages of every CPU-visible pool in -each such part, which the cost above does not include. What a shader stores through the table into a shadow copy does -not come back to guest memory. `--probe-vkmem` checks the read: a shader follows the table to a word of a pool that is -never bound, and must read `0x13579BDF` and, after a plain CPU write, `0x2468ACE0`. On Linux, run it for each backing: +each such part, which the cost above does not include. What a shader stores through the table lands in the shadow copy +and in the pool's VRAM mirror, and no binding names it, so nothing notes it as a GPU write: it does not come back to +guest memory, and the next whole copy-in of its page (the CPU rewrites other words of the same records) puts guest +memory's stale bytes back over it in both copies. In the runs measured so far that copy-in always came before the +frame's stores, which does no harm. `NX2NAVI_SHADOW_TABLE_STORES=1` keeps the stores instead: the copy-ins then keep a +hash of each page they write, a page whose shadow no longer matches its hash was stored into, keeps a base copy from +then on like a page with a noted GPU write, and has its stores written back to guest memory at the fence of every +submission that uses the table. It is off by default (`runtime/vk_memory.c`, "Pages a shader stores into", says why). +`NX2NAVI_SHADOW_CHECK=1` logs every 60 frames how often a copy-in overwrote such a store and for which pools, how many +write-backs landed on pages the CPU had written since their copy-in, and what the copy-ins cost and where they went. +`--probe-vkmem` checks the read: a shader follows the table to a word of a pool that is never bound, and must read +`0x13579BDF` and, after a plain CPU write, `0x2468ACE0`; and with `NX2NAVI_SHADOW_TABLE_STORES=1` the store: a GPU +write into a shadow page that nothing notes must survive the CPU's writes of that page and reach guest memory. On +Linux, run it for each backing: ```sh for shadow in 0 1; do for memory in import mirror; do diff --git a/runtime/cfg.c b/runtime/cfg.c index 2e16200a..4a6d0a1c 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -60,7 +60,7 @@ static const char *const known[] = { "PRESNAP_PLACE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", - "SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_BULK_WRITE", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", + "SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_BULK_WRITE", "SHADOW_CHECK", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHADOW_TABLE_STORES", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SVC_SLOW_MS", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", @@ -192,6 +192,8 @@ static const CfgMeta metas[] = { {"SHADOW_RES", "game", "game|256|512|768|1024|1536|2048", RESTART}, {"SHADOW_SCALE", "1", "1..4", RESTART}, {"SHADOW_COMPARE", "1", "0|1", RESTART}, // vk_texture.c {"SHADOW_BULK_WRITE", "0", "0|1", RESTART}, // vk_memory.c shadow_write_back + // vk_memory.c "Pages a shader stores into through the device address table" + {"SHADOW_TABLE_STORES", "0", "0|1", RESTART}, {"SHADOW_CHECK", "0", "0|1", RESTART}, {"SVC_SLOW_MS", "0", "0..60000", RESTART}, // kernel.c kern_svc: log services slower than this {"SHADOW_RANGE", "100", "50..400", RESTART}, {"CONTACT_SHADOWS", "0", "0..100", RESTART}, {"PCSS", "0", "0..200", RESTART}, diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 1bee0375..769081eb 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -520,6 +520,100 @@ done: return bad; } +// test_table_store: a submission whose one part uses the device address table and records nothing else, as a part of +// the replay with such a shader ends: its prologue brings every pool of the table up to date (vk_mem_gpa_sync). +static void table_part_record(VkCommandBuffer cb, void *arg) { + (void)arg; + vk_texture_new_submit(); + vk_mem_mirror_begin(); + vk_mem_gpa_use(); + vk_mem_mirror_end(cb, vk_sync_issued() + 1); +} + +static int table_part(void) { + uint64_t fence = vk_sync_probe_submit(table_part_record, NULL); + return vk_sync_fence_wait(fence, 5000000000ull); // its completion callbacks have run when the wait returns +} + +// Shadow storage: what a shader stores through the device address table lands in the pool's shadow copy and nothing +// notes it (vk_memory.c "Pages a shader stores into"). A GPU copy into the shadow buffer that is not noted stands in +// for the store. Word X of a page takes the stores, the CPU rewrites word Y of the same page before every part: +// - the store made after the page's first whole copy-in is overwritten by the second, which finds it (nothing can +// keep it); +// - every later store stays in the shadow through the copy-ins that bring Y, and is in guest memory after the fence; +// - a CPU write of X after that reaches the shadow. +static int test_table_store(uint8_t *guest) { + VkCtx *c = vk_ctx(); + if (!vk_mem_shadow_storage() || !vk_gpa_wanted() || !vk_mem_shadow_table_stores()) { + printf("probe-vkmem: stores through the device address table into a shadow copy: skipped (%s)\n", + !vk_mem_shadow_storage() ? "no shadow storage" : !vk_gpa_wanted() ? "no device address table" + : "NX2NAVI_SHADOW_TABLE_STORES is not 1"); + return 0; + } + pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xB00000, .size = 0x10000, + .game_mem = guest + 0xB00000}; + volatile uint32_t *x = (volatile uint32_t *)(pool.mem + 0x200), *y = (volatile uint32_t *)(pool.mem + 0x100); + *x = 0, *y = 1; + GCHECK(vk_gpa_on() && vk_pool_init(&pool), "a CPU-visible pool in the device address table"); + VkBuffer sb; + VmaAllocation sa; + volatile uint32_t *s = vk_staging(0x1000, &sb, &sa); + VkMemSpan sp; + int bad = 1; +#define TS_CHECK(cond, ...) do { if (!(cond)) { printf("probe-vkmem: FAIL table store: "); printf(__VA_ARGS__); printf("\n"); goto done; } } while (0) +#define TS_STORE(v) \ + do { \ + s[0] = (v); \ + vmaFlushAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); \ + TS_CHECK(vk_mem_lookup((uintptr_t)x, 4, &sp) && sp.buffer, "the pool has no buffer"); \ + copy_wait(sb, 0, sp.buffer, sp.offset, 4); \ + } while (0) +#define TS_SHADOW(off, into) \ + do { \ + TS_CHECK(vk_mem_lookup((uintptr_t)pool.mem + (off), 4, &sp) && sp.buffer, "the pool has no buffer"); \ + copy_wait(sp.buffer, sp.offset, sb, 16, 4); \ + vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); \ + (into) = s[4]; \ + } while (0) + uint32_t sx, sy; + *y = 2; + TS_CHECK(table_part(), "fence timed out"); // the page's first whole copy-in: its mark + TS_STORE(0xAAAA0001u); + *y = 3; + TS_CHECK(table_part(), "fence timed out"); // finds the store, overwrites it, the page keeps a base copy + TS_SHADOW(0x200, sx); + TS_CHECK(sx == 0, "the copy-in that found the store left X %#010x in the shadow, expected guest memory's 0", sx); + TS_STORE(0xAAAA0002u); + *y = 4; + TS_CHECK(table_part(), "fence timed out"); // brings Y, keeps X; its fence writes X back + TS_SHADOW(0x200, sx); + TS_SHADOW(0x100, sy); + TS_CHECK(sx == 0xAAAA0002u && sy == 4, "after a CPU write of the page the shadow holds X %#010x Y %u, expected " + "0xaaaa0002 and 4", sx, sy); + TS_CHECK(*x == 0xAAAA0002u && *y == 4, "guest memory holds X %#010x Y %u after the fence, expected 0xaaaa0002 and 4", + (uint32_t)*x, (uint32_t)*y); + *x = 0; // the CPU resets the stored word + TS_CHECK(table_part(), "fence timed out"); + TS_SHADOW(0x200, sx); + TS_CHECK(sx == 0 && *x == 0, "after the CPU reset X the shadow holds %#010x and guest memory %#010x", sx, (uint32_t)*x); + TS_STORE(0xAAAA0003u); // and a store after the reset comes back as well + TS_CHECK(table_part(), "fence timed out"); + TS_CHECK(*x == 0xAAAA0003u && *y == 4, "guest memory holds X %#010x Y %u after the second fence", (uint32_t)*x, + (uint32_t)*y); + g_gpu_checks += 5; + printf("probe-vkmem: a store into a shadow page that nothing notes survives the CPU's writes of that page, reaches " + "guest memory at the fence, and gives way to a CPU write of the same word\n"); + bad = 0; +done: + vk_sync_finish(); + vmaDestroyBuffer(c->vma, sb, sa); + vk_pool_fini(&pool); +#undef TS_CHECK +#undef TS_STORE +#undef TS_SHADOW + return bad; +} + // ReadFile into a watched page, buffered and unbuffered: timed, and whether write watch saw the kernel's write. static int test_readfile(uint8_t *dst) { char dir[PLAT_PATH_MAX], path[PLAT_PATH_MAX + 64]; @@ -917,7 +1011,7 @@ static int test_gpu(void) { vk_mem_stats(&st); GCHECK(!st.imported_pools && !st.physical_pools && !st.virtual_pools, "pools left after fini"); g_gpu_checks += 2; - return test_gpa_read(h); + return test_gpa_read(h) || test_table_store(h); } int probe_vkmem(void) { diff --git a/runtime/vk.h b/runtime/vk.h index 7153b63f..cff0622a 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -348,10 +348,12 @@ void vk_mem_fatal_note(char *out, size_t n); // the memory mode and the lazy w void vk_mem_shadow_sync(void); // A shader of the part being recorded reads through the device address table (vk_gpa.c), also when its descriptors // are reused: with shadow storage the part's prologue (vk_mem_gpa_sync) then brings every pool the table names up to -// date with the CPU's writes, since the pools such a shader reaches are never resolved. +// date with the CPU's writes, since the pools such a shader reaches are never resolved. fence: the submission's value, +// at which what the shaders stored through the table goes back to guest memory. void vk_mem_gpa_use(void); -void vk_mem_gpa_sync(void); +void vk_mem_gpa_sync(uint64_t fence); int vk_mem_shadow_storage(void); // CPU-visible pools are shadow copies of guest memory, not imports (NX2NAVI_SHADOW) +int vk_mem_shadow_table_stores(void); // shadow storage keeps what shaders store through the table (NX2NAVI_SHADOW_TABLE_STORES) void vk_mem_gpu_pending(uint64_t addr, uint64_t size); void vk_gpu_written(uint64_t addr, uint64_t size); // The same for a GPU write into a VRAM mirror, which went back to guest memory with the submission: the texture checks diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index 6bd4732a..c377646e 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -37,8 +37,11 @@ // NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis). // Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). With shadow storage the // pools are copies of guest memory: a part whose shaders use the table is marked (vk_exec_draw.c bind_resources) and -// its prologue copies the pages the CPU wrote into the copies the table names (vk_memory.c vk_mem_gpa_sync). What a -// shader stores through the table into a shadow copy is not noted as a GPU write and does not come back to guest memory. +// its prologue copies the pages the CPU wrote into the copies the table names (vk_memory.c vk_mem_gpa_sync). There +// the twin of a mirrored pool is its shadow copy, not guest memory, and what a shader stores through the table is not +// noted as a GPU write: it does not come back to guest memory, and the next whole copy-in of its page overwrites it. +// vk_memory.c "Pages a shader stores into through the device address table" has the sequence, what was measured, and +// the switch that keeps the stores (NX2NAVI_SHADOW_TABLE_STORES). #include #include diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index fb1f6e3b..68aa1b7d 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -87,6 +87,16 @@ typedef struct PoolRec { // csb a power of two, page 0 an empty slot struct SBase { uint64_t page; uint8_t *base; } *sb; uint32_t nsb, csb; + // shadow_lock ("Pages a shader stores into through the device address table" below): a mark per page of + // [import_lo, import_hi), the hash of the shadow page as its last whole copy-in left it, 0 not known; the pages + // found stored into (guest addresses), and whether the pool is on the list of pools that have such pages + uint32_t *smark; + uint64_t *tpg; + uint32_t ntpg, ctpg; + uint8_t tlisted; + uint64_t t_over; // whole copy-ins that found such a store in the page they overwrote, so far + uint64_t t_over_log; // t_over at the last log line + uint64_t s_in; // NX2NAVI_SHADOW_CHECK: bytes copied in whole since the last log line // NX2NAVI_TRACE=perf per-pool mirror traffic since the last mirror_pool_log, replay thread (the log reads and clears // them without a lock: a count lost to the race only blurs a diagnostic) uint64_t st_pages, st_flush, st_wb, st_merge; // bytes: written pages and flush ranges copied up, GPU writes back @@ -592,6 +602,7 @@ static void dirty_frame(uint64_t frame); static void mirror_log(uint64_t frame); static void mirror_pool_log(uint64_t frame); static void shadow_log(uint64_t frame); +static int shadow_check_on(void); static int dead_pending(void); static void vram_log(uint64_t frame); @@ -607,6 +618,8 @@ void vk_mem_census(uint64_t frame) { if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame), shadow_log(frame); // mirror_log also logs the write-watch line + else if (shadow_check_on() && frame % 60 == 0) + shadow_log(frame); // NX2NAVI_SHADOW_CHECK: the shadow lines at that rate without the perf trace if (frame == 1200) { // loading is over: the first window starts here for (int c = 0; c < VK_MCLS_N; c++) for (int u = 0; u < VK_MUSE_N; u++) plat_xchg64(&traffic[c][u], 0); @@ -1030,6 +1043,46 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { // GPU's when the fence does. Whole-range copies both ways lost the CPU's bytes: Ocarina writes a lighting parameter // into a buffer whose binding its compute passes write every frame, the GPU only ever read zero there, the lighting // coefficients and then the volumetric fog came out NaN, and the fog pass drew the 3D scene black. +// - The game must not get ahead of the write-backs. With an import the GPU's bytes are in guest memory when the GPU +// has stored them; here they get there in a completion callback, after the fence. A CPU write that should follow +// them and lands before that callback is overtaken: the write-back puts the GPU's older bytes over it and moves the +// base copy, and a rewrite with the base copy's value (a reset to zero) is no difference at any time. So the wait +// after a frame's first submission covers the write-backs with shadow storage (vk_sync.c "the executed timeline", +// which tells what the game rewrites there and what a frame looked like without it). +// - Pages a shader stores into through the device address table (vk_gpa.c): the store goes to the pool's VRAM mirror +// and to its twin, which here is the shadow, not guest memory. No binding names the range, so nothing notes it +// (vk_mem_gpu_pending): the page has no base copy and guest memory never gets the bytes. The game's culling passes +// store counts and records this way into pages whose other words the CPU rewrites. For one such page P, a word X the +// shaders store and a word Y the CPU writes: +// part A (the pass that stores): the CPU wrote Y, so A's prologue copies P in whole (shadow_copy) and queues it +// for the mirror (mirror_sync). X in the shadow and in the mirror is now guest memory's X, which no store ever +// reached. A runs and stores X = v into both. +// part B (a later part that reads X through the table), the CPU has not written P since A's write-watch snapshot: +// B's prologue leaves P alone and its draws read v. +// part B, the CPU has written Y again between the two snapshots: B's prologue copies P in whole and uploads it, X +// is guest memory's stale word again in both copies, and the draws read that. +// An import does not have the fault: its twin is guest memory, and a page copied up from there holds the store. +// Measured (NX2NAVI_SHADOW_CHECK, a standing run outdoors, RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10): the stores +// go into one page each of ten pools of 20 and 44 KiB, and whole copy-ins overwrite one exactly four times a frame +// (240 in every 60 frames): part A's case, the copy that precedes the frame's stores, which does no harm. Part B's +// second case would add to the four and was not seen in 1,800 frames; nothing proves it cannot happen where the +// frame's timing differs. +// NX2NAVI_SHADOW_TABLE_STORES=1 keeps the stores. The pages are found by their content: each whole copy-in leaves a +// hash of the page it wrote (smark), and the next one hashes the shadow page first. A different hash means something +// stored into the shadow in between, and with no base copy on the page that was a store through the table. That +// copy-in still overwrites the store (nothing tells its bytes from the CPU's), and from then on the page keeps a +// base copy: copy-ins take the CPU's bytes only, and the fence of every submission with a part that uses the table +// writes the stored bytes of all such pages back to guest memory (table_wb_cb), as the fence of a noted write does. +// Guest memory needs them: a word the CPU resets each frame to the value it had before the stores would otherwise +// never differ from its base, and the reset would never reach the GPU. +// It is off by default, for three reasons. The fault it removes was not seen. Its hashes cost 1.8 ms a frame of +// the replay there (copy_marked). And it puts these pages under the rule of the bullet above: a count the CPU +// resets before that write-back has landed keeps counting from its old value, where the whole copy-in of today +// resets it every time; whether the game resets any of these words from the CPU is not known. +// NX2NAVI_SHADOW_CHECK=1 counts, with the shadow line and at its rate every 60 frames (shadow_table_log): the whole +// copy-ins that overwrote a store through the table, per pool; the write-backs that moved bytes into a page the CPU +// had written since its copy-in (cpu_wrote_since); the bytes both sides changed since their base copy (taken from the +// CPU at a copy-in, from the GPU at a fence); what the copy-ins cost by kind. // NX2NAVI_SHADOW=1/0 forces the mode on or off; the default is on for Linux and Wine. #define SB_PAGE 4096ull static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock @@ -1041,6 +1094,22 @@ static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks; static uint64_t s_wb_seen, s_wb_ticks; // shadow_lock: bytes the fence write-backs compared, their time static uint64_t s_q_ticks; // shadow_lock: the copy-ins' write-watch queries static int wb_no_reup(void); +// Pages a shader stores into through the device address table. The two switches are read in mirror_mode. +static int s_learn; // NX2NAVI_SHADOW_TABLE_STORES: such pages get a base copy and a write-back +static int s_check; // NX2NAVI_SHADOW_CHECK: the counts below marked "check" +static PoolRec **tpools; // shadow_lock: the pools that have such pages +static int ntpools, ctpools; +static uint64_t s_tab_pages; // shadow_lock: such pages, all pools +// shadow_lock, since the last log: +static uint64_t s_tab_over; // whole copy-ins that overwrote a store through the table +static uint64_t s_tab_wb, s_tab_wb_seen, s_tab_wb_ticks, s_tab_wb_jobs; // table_wb_cb: bytes moved and compared, time, runs +static uint64_t s_marked, s_mark_ticks; // pages copied in whole with a mark; check: the time of those copies +static uint64_t s_plain_ticks; // check: the time of the whole copies made without marks +static uint64_t s_both_cpu, s_both_gpu; // check: bytes both sides changed, taken from the CPU / from the GPU +static uint64_t s_in_merge, s_in_nomirror; // check: bytes copied in by merge_page; whole, into pools without a mirror +static uint64_t s_gpa_ticks, s_gpa_syncs; // vk_mem_gpa_sync: its time (part of s_ticks) and its runs +static uint64_t s_t_over_all; // shadow_lock: s_tab_over since the start +static uint64_t s_late_pages, s_late_bytes; // shadow_lock, check: write-backs into pages the CPU wrote since their copy-in static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) { r->import_lo = addr & ~0xfffull; @@ -1184,12 +1253,91 @@ static uint64_t merge_page(uint8_t *dst, const uint8_t *src, uint8_t *b, uint64_ return moved; } +// ---- pages a shader stores into through the device address table ("shadow storage" above). shadow_lock held. + +// The mark of a 4 KiB page: a hash of its bytes, never 0. Four sums that do not wait for each other. +static uint32_t page_mark(const uint8_t *p) { + uint64_t h[4] = {0x9E3779B97F4A7C15ull, 0xC2B2AE3D27D4EB4Full, 0x165667B19E3779F9ull, 0x27D4EB2F165667C5ull}; + for (uint64_t i = 0; i < SB_PAGE; i += 32) { + uint64_t w[4]; + memcpy(w, p + i, 32); + for (int k = 0; k < 4; k++) { + h[k] = (h[k] ^ w[k]) * 0xFF51AFD7ED558CCDull; + h[k] = h[k] << 29 | h[k] >> 35; + } + } + uint64_t x = (h[0] ^ h[1] << 13 ^ h[2] << 27 ^ h[3] << 41 ^ h[1] >> 51 ^ h[2] >> 37 ^ h[3] >> 23) * 0xC4CEB9FE1A85EC53ull; + uint32_t m = (uint32_t)(x >> 32); + return m ? m : 1; +} + +// Guest page pg of r was stored into through the table: it keeps a base copy from here on, taken from the shadow as +// the whole copy-in just left it (guest memory's bytes), and table_wb_cb writes its stores back. +static void table_page_add(PoolRec *r, uint64_t pg) { + sbase_make(r, pg); + if (r->ntpg == r->ctpg) { + uint32_t cap = r->ctpg ? r->ctpg * 2 : 16; + uint64_t *t = realloc(r->tpg, cap * sizeof *t); + if (!t) rt_fatal("vk: out of memory for the shadow's table pages"); + r->tpg = t, r->ctpg = cap; + } + r->tpg[r->ntpg++] = pg; + s_tab_pages++; + if (r->tlisted) return; + if (ntpools == ctpools) grow((void **)&tpools, &ctpools, sizeof *tpools); + tpools[ntpools++] = r; + r->tlisted = 1; +} + +// NX2NAVI_SHADOW_CHECK: the bytes of [0, n) that both sides changed since the base copy b and that now differ, which a +// merge of src into dst is about to settle in src's favour. +static uint64_t both_changed(const uint8_t *dst, const uint8_t *src, const uint8_t *b, uint64_t n) { + uint64_t k = 0; + for (uint64_t i = 0; i < n; i++) k += (src[i] != b[i]) & (dst[i] != b[i]) & (dst[i] != src[i]); + return k; +} + +// Copies the whole guest pages [a, e) of r into its shadow, none of which has a base copy. Each page's mark tells +// whether the shadow page still holds what the last such copy left: if not, a shader stored into it through the table +// meanwhile, and this copy overwrites that store ("Pages a shader stores into" above says why it cannot be kept, and +// what the page gets from here on). +// Only with NX2NAVI_SHADOW_TABLE_STORES or NX2NAVI_SHADOW_CHECK (shadow_copy). A mark is two hashes of the page, one +// of them of bytes the copy would not have read: outdoors, where 2,400 pages a frame are copied in whole, the marks +// cost 1.8 ms a frame of the replay (3.09 against 1.32 ms of copy-ins, 67 against 75 fps, RTX 5090 with +// NX2NAVI_SHADOW=1, 2026-11-10). +static void copy_marked(PoolRec *r, uint64_t a, uint64_t e) { + if (!r->smark) { + r->smark = calloc((r->import_hi - r->import_lo) / SB_PAGE, sizeof *r->smark); + if (!r->smark) rt_fatal("vk: out of memory for the shadow's page marks"); + } + for (uint64_t pg = a; pg < e; pg += SB_PAGE) { + uint8_t *s = r->smap + (pg - r->import_lo); + uint32_t *m = &r->smark[(pg - r->import_lo) / SB_PAGE]; + int stored = *m && page_mark(s) != *m; + memcpy(s, (const void *)(uintptr_t)pg, SB_PAGE); + if (stored) { + s_tab_over++, s_t_over_all++, r->t_over++; + if (s_learn) { // merged from now on: its mark is not read again + table_page_add(r, pg); + continue; + } + } + *m = page_mark(s); // of the shadow, not of guest memory, which the CPU may be writing + } + s_marked += (e - a) / SB_PAGE; +} + // vk_pool_fini, after the pool left the index: off the touched list before its record goes. static void shadow_forget(PoolRec *r) { plat_mutex_lock(&shadow_lock); for (int i = 0; i < nstouch; i++) if (stouch[i] == r) { stouch[i] = stouch[--nstouch]; break; } + for (int i = 0; r->tlisted && i < ntpools; i++) + if (tpools[i] == r) { tpools[i] = tpools[--ntpools]; break; } s_live_n--, s_live_bytes -= (int64_t)(r->import_hi - r->import_lo); + s_tab_pages -= r->ntpg; + free(r->tpg), free(r->smark); + r->tpg = NULL, r->smark = NULL, r->ntpg = r->ctpg = 0, r->tlisted = 0; sbase_free(r); plat_mutex_unlock(&shadow_lock); } @@ -1205,12 +1353,25 @@ static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b) { if (e > b) e = b; if (e == a) { // a starts on such a page: its part of [a, b) uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < b ? pg + SB_PAGE : b; - s_copied += merge_page(r->smap + (a - r->import_lo), (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a); + uint8_t *s = r->smap + (a - r->import_lo); + if (s_check) s_both_cpu += both_changed(s, (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a); + uint64_t n = merge_page(s, (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a); + s_copied += n, s_in_merge += n; a = pe; continue; } } - memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, e - a); + // The copies are of whole pages (shadow_sync_pool); the marks need the device address table, whose stores + // they find, and one of the two switches. + uint64_t t0 = s_check ? qpc() : 0; + int marks = (s_learn || s_check) && r->gpa_rec && !((a | e) & (SB_PAGE - 1)); + if (marks) copy_marked(r, a, e); + else memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, e - a); + if (s_check) { + *(marks ? &s_mark_ticks : &s_plain_ticks) += qpc() - t0; + if (r->mstate != MS_ON) s_in_nomirror += e - a; + r->s_in += e - a; + } s_copied += e - a; a = e; } @@ -1334,6 +1495,18 @@ static int shadow_bulk_write(void) { return on; } +// NX2NAVI_SHADOW_CHECK: the CPU wrote guest page pg after the copy-in that sgen stands for: a write-watch snapshot +// stamped it later, or its write-watch bit is set and no snapshot has read it yet. A write-back that moves bytes into +// such a page may have come after a CPU write it should have come before ("The game must not get ahead of the +// write-backs" above); a page of which the CPU wrote other bytes counts too. +// No gen_lock, as lazy_clean: a stamp that lands meanwhile only blurs a count. +static int cpu_wrote_since(uint64_t pg, uint32_t sgen) { + const uint32_t *ch = __atomic_load_n(&cgen_chunk[pg >> GEN_CHUNK_SHIFT], __ATOMIC_ACQUIRE); + if (ch && __atomic_load_n(&ch[(pg >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)], __ATOMIC_RELAXED) > sgen) return 1; + uint64_t p[1], n = mem_written_peek(pg, SB_PAGE, p, 1); + return n && n != UINT64_MAX; +} + // Completion thread: the GPU's bytes of [addr, addr+size) from the shadow into guest memory (merge_page against the // base copies; a page without one whole). Returns the bytes moved. static uint64_t shadow_write_back(uint64_t addr, uint64_t size) { @@ -1350,8 +1523,12 @@ static uint64_t shadow_write_back(uint64_t addr, uint64_t size) { for (uint64_t a = addr, end = addr + size; a < end;) { uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < end ? pg + SB_PAGE : end; uint8_t *base = sbase_find(r, pg), *s = r->smap + (a - r->import_lo); - if (base) moved += merge_page((uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a); + if (base && s_check) s_both_gpu += both_changed((const uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a); + int late = base && s_check && cpu_wrote_since(pg, r->sgen); // before the merge's own stores into the page + uint64_t n = 0; + if (base) moved += n = merge_page((uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a); else memcpy((void *)(uintptr_t)a, s, pe - a), moved += pe - a; // a write never noted as pending + if (late && n) s_late_pages++, s_late_bytes += n; a = pe; } s_wb_bytes += moved, s_wb_seen += size, s_wb_ticks += qpc() - t0; @@ -1361,6 +1538,47 @@ static uint64_t shadow_write_back(uint64_t addr, uint64_t size) { return moved; } +// Completion thread, at the fence of a submission with a part that uses the device address table (vk_mem_gpa_sync): +// what the shaders stored into the pages found so far ("Pages a shader stores into" above) goes from the shadow into +// guest memory, the bytes that differ from the page's base copy. A page whose shadow equals its base is passed over +// before guest memory is touched, so a frozen region stays frozen. The pages that changed are stamped as a mirror's +// write-back is (swb_run): the stores are in the mirror already. +static void table_wb_cb(void *arg) { + (void)arg; + static uint64_t *wrote; // this thread only: the pages that changed + static int cwrote; + int nwrote = 0, bulk = shadow_bulk_write(), back = wb_no_reup(); // before the locks: their first calls log + uint64_t t0 = qpc(), moved = 0, seen = 0; + plat_mutex_lock(&shadow_lock); + for (int i = 0; i < ntpools; i++) { + PoolRec *r = tpools[i]; + for (uint32_t k = 0; k < r->ntpg; k++) { + uint64_t pg = r->tpg[k]; + uint8_t *base = sbase_find(r, pg), *s = r->smap + (pg - r->import_lo); + seen += SB_PAGE; + if (!base || !memcmp(s, base, SB_PAGE)) continue; + if (mem_frozen(pg)) mem_thaw(pg, SB_PAGE); + if (bulk) plat_mem_prepare_write(pg, SB_PAGE); + if (s_check) s_both_gpu += both_changed((const uint8_t *)(uintptr_t)pg, s, base, SB_PAGE); + uint64_t n = merge_page((uint8_t *)(uintptr_t)pg, s, base, SB_PAGE); + if (!n) continue; // the store came after the compare above: the next fence takes it + moved += n; + if (nvn_gpu_only(pg)) continue; + if (nwrote == cwrote) grow((void **)&wrote, &cwrote, sizeof *wrote); + wrote[nwrote++] = pg; + } + } + s_tab_wb += moved, s_tab_wb_seen += seen, s_tab_wb_jobs++; + if (nwrote) { + plat_mutex_lock(&gen_lock); + uint32_t g = ++gen_now; + for (int i = 0; i < nwrote; i++) stamp_page_as(wrote[i], g, back); + plat_mutex_unlock(&gen_lock); + } + s_tab_wb_ticks += qpc() - t0; + plat_mutex_unlock(&shadow_lock); +} + static void gpu_written(uint64_t addr, uint64_t size, int back) { // back: the bytes are in a VRAM mirror, and its write-back brings them into the shadow; that submission's // shadow_wb_cb, after the replay's own callbacks, then copies them into guest memory. @@ -1398,8 +1616,92 @@ int vk_mem_mirrored(uint64_t addr, uint64_t size) { return r->mstate == MS_ON && addr >= r->import_lo && addr + size <= r->import_hi; } +// shadow_lock held, with the shadow line: the pages a shader stores into through the device address table ("shadow +// storage" above), and with NX2NAVI_SHADOW_CHECK (pool_lock held shared too, for the index) what the copy-ins did and +// cost and the pools whose stores were overwritten most since the last line. +#define TABLE_LOG_TOP 6 +static void shadow_table_log(uint64_t frame) { + rt_log("mem: shadow table stores at frame %llu (%s): %llu pages in %d pools keep a base copy; since the last: %llu whole " + "copy-ins overwrote a store (%llu so far), %llu pages copied in with a mark; written back %.3f MiB of %.1f " + "compared in %llu fence jobs, %.2f ms", + (unsigned long long)frame, s_learn ? "NX2NAVI_SHADOW_TABLE_STORES=1: kept from the second copy-in on" : "overwritten", + (unsigned long long)s_tab_pages, ntpools, (unsigned long long)s_tab_over, (unsigned long long)s_t_over_all, + (unsigned long long)s_marked, s_tab_wb / 1048576.0, s_tab_wb_seen / 1048576.0, (unsigned long long)s_tab_wb_jobs, + qpc_ms(s_tab_wb_ticks)); + if (s_check) { + rt_log("mem: shadow check at frame %llu: copied in whole %.2f ms with marks, %.2f ms without; %.1f MiB by merge, " + "%.1f MiB whole into pools without a mirror; %llu table syncs %.2f ms; bytes both sides changed: %llu taken " + "from the CPU at a copy-in, %llu from the GPU at a fence; write-backs into %llu pages the CPU had written " + "since their copy-in, %llu bytes", + (unsigned long long)frame, qpc_ms(s_mark_ticks), qpc_ms(s_plain_ticks), s_in_merge / 1048576.0, + s_in_nomirror / 1048576.0, (unsigned long long)s_gpa_syncs, qpc_ms(s_gpa_ticks), + (unsigned long long)s_both_cpu, (unsigned long long)s_both_gpu, (unsigned long long)s_late_pages, + (unsigned long long)s_late_bytes); + PoolRec *top[TABLE_LOG_TOP]; + int nt = 0; + for (int i = 0; i < pool_index.n; i++) { + PoolRec *r = pool_index.e[i].buf; + uint64_t d = r->kind == POOL_IMPORT ? r->t_over - r->t_over_log : 0; + int k = nt; + while (k > 0 && top[k - 1]->t_over - top[k - 1]->t_over_log < d) k--; + if (!d || k >= TABLE_LOG_TOP) continue; + int last = nt < TABLE_LOG_TOP ? nt : TABLE_LOG_TOP - 1; + memmove(top + k + 1, top + k, (size_t)(last - k) * sizeof *top); + top[k] = r; + if (nt < TABLE_LOG_TOP) nt++; + } + for (int i = 0; i < nt; i++) { + char f[160]; + flag_names(top[i]->flags, f, sizeof f); + rt_log("mem: shadow check: pool %#llx+%#llx (%s, %s): %llu whole copy-ins overwrote a store (%llu so far), " + "%u of its pages keep a base copy for it", + (unsigned long long)top[i]->lo, (unsigned long long)top[i]->size, f, + top[i]->mstate == MS_ON ? "mirrored" : "no mirror", + (unsigned long long)(top[i]->t_over - top[i]->t_over_log), (unsigned long long)top[i]->t_over, top[i]->ntpg); + } + // the pools that took the most whole pages, and what all pools of each flag set took + struct { uint32_t flags; uint64_t bytes; int n; } cls[16]; + int ncls = 0; + nt = 0; + for (int i = 0; i < pool_index.n; i++) { + PoolRec *r = pool_index.e[i].buf; + if (r->kind != POOL_IMPORT || !r->s_in) continue; + int c = 0, k = nt; + while (c < ncls && cls[c].flags != r->flags) c++; + if (c == ncls && ncls < 16) cls[ncls++] = (__typeof__(cls[0])){r->flags, 0, 0}; + if (c < ncls) cls[c].bytes += r->s_in, cls[c].n++; + while (k > 0 && top[k - 1]->s_in < r->s_in) k--; + if (k >= TABLE_LOG_TOP) continue; + int last = nt < TABLE_LOG_TOP ? nt : TABLE_LOG_TOP - 1; + memmove(top + k + 1, top + k, (size_t)(last - k) * sizeof *top); + top[k] = r; + if (nt < TABLE_LOG_TOP) nt++; + } + for (int i = 0; i < ncls; i++) { + char f[160]; + flag_names(cls[i].flags, f, sizeof f); + rt_log("mem: shadow check: copied in whole into %d pools %s: %.2f MiB", cls[i].n, f, cls[i].bytes / 1048576.0); + } + for (int i = 0; i < nt; i++) { + char f[160]; + flag_names(top[i]->flags, f, sizeof f); + rt_log("mem: shadow check: pool %#llx+%#llx (%s, %s, %s): %.2f MiB copied in whole", + (unsigned long long)top[i]->lo, (unsigned long long)top[i]->size, f, + top[i]->mstate == MS_ON ? "mirrored" : "no mirror", top[i]->used ? "resolved by the GPU" : "never resolved", + top[i]->s_in / 1048576.0); + } + for (int i = 0; i < pool_index.n; i++) { + PoolRec *r = pool_index.e[i].buf; + if (r->kind == POOL_IMPORT) r->t_over_log = r->t_over, r->s_in = 0; + } + } + s_tab_over = s_tab_wb = s_tab_wb_seen = s_tab_wb_ticks = s_tab_wb_jobs = s_marked = s_mark_ticks = s_plain_ticks = 0; + s_both_cpu = s_both_gpu = s_in_merge = s_in_nomirror = s_gpa_ticks = s_gpa_syncs = s_late_pages = s_late_bytes = 0; +} + static void shadow_log(uint64_t frame) { if (!shadow_on) return; + if (s_check) plat_mutex_lock_shared(&pool_lock); // before shadow_lock: shadow_table_log walks the index plat_mutex_lock(&shadow_lock); rt_log("mem: shadow at frame %llu: %lld pools %.1f MiB; since the last: %llu pool syncs, %llu queries, %.1f MiB " "copied in (%llu untracked), %.2f ms (queries %.2f); %.1f MiB written back of %.1f MiB the GPU may have written, %.2f ms; %llu " @@ -1408,7 +1710,9 @@ static void shadow_log(uint64_t frame) { (unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, qpc_ms(s_ticks), qpc_ms(s_q_ticks), s_wb_bytes / 1048576.0, s_wb_seen / 1048576.0, qpc_ms(s_wb_ticks), (unsigned long long)s_sb_pages); s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = s_wb_seen = s_wb_ticks = s_q_ticks = 0; + if (s_learn || s_check) shadow_table_log(frame); plat_mutex_unlock(&shadow_lock); + if (s_check) plat_mutex_unlock_shared(&pool_lock); } void vk_mem_dirty_submit(void) { @@ -1657,6 +1961,14 @@ static void mirror_mode(int discrete) { #endif if (shadow_on) rt_log("mem: shadow storage: CPU-visible pools are host-visible copies of guest memory, not imports%s", sh && *sh ? " (NX2NAVI_SHADOW)" : ""); + s_learn = shadow_on && rt_knob("SHADOW_TABLE_STORES", "0") == 1; + s_check = shadow_on && cfg_bool("NX2NAVI_SHADOW_CHECK"); + if (shadow_on) + rt_log("mem: what shaders store through the device address table into a shadow copy %s " + "(NX2NAVI_SHADOW_TABLE_STORES)%s", + s_learn ? "is kept through the copy-ins and written back to guest memory, from the second copy-in of its page on" + : "is overwritten by the next whole copy-in of its page", + s_check ? "; NX2NAVI_SHADOW_CHECK counts the overwrites" : ""); rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why, mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory"); // Import is the integrated-GPU mode. On a discrete card every draw reads the pools over PCIe (the comment above @@ -2071,6 +2383,8 @@ static void mirror_sync(PoolRec *r) { static int gpa_part_used; // replay thread: the part being recorded uses the table void vk_mem_gpa_use(void) { gpa_part_used = 1; } int vk_mem_shadow_storage(void) { return shadow_on; } +static int shadow_check_on(void) { return s_check; } +int vk_mem_shadow_table_stores(void) { return s_learn; } static int gpa_shadow_pool(const PoolRec *r) { return r->kind == POOL_IMPORT && r->gpa_rec && r->smap; } @@ -2090,10 +2404,18 @@ static int shadow_pool_clean(const PoolRec *r, uint32_t epoch, int cpu) { return 1; } -void vk_mem_gpa_sync(void) { +void vk_mem_gpa_sync(uint64_t fence) { if (!gpa_part_used) return; gpa_part_used = 0; if (!shadow_on) return; + // What this part's shaders store through the table goes back to guest memory at the submission's fence, once a + // submission however many of its parts use the table (table_wb_cb). Asked before the copy-ins below: the part + // whose copy-ins find the first such page leaves it to the next part that uses the table. + static uint64_t wb_fence; // replay thread: the last fence that got the job + if (s_learn && fence && fence != wb_fence && __atomic_load_n(&ntpools, __ATOMIC_RELAXED)) { + wb_fence = fence; + vk_completion_add(fence, table_wb_cb, NULL); + } uint64_t t0 = qpc(), run0 = 0, run1 = 0; uint32_t epoch = vk_texture_epoch(); int cpu = vk_mirror_on && wb_no_reup(), have = 0; // before the locks: its first call logs @@ -2126,6 +2448,7 @@ void vk_mem_gpa_sync(void) { PoolRec *r = pool_index.e[i].buf; if (gpa_shadow_pool(r) && r->mstate == MS_ON && r->acc_sub != msub) mirror_sync(r); } + s_gpa_ticks += qpc() - t0, s_gpa_syncs++; s_ticks += qpc() - t0; plat_mutex_unlock(&shadow_lock); plat_mutex_unlock_shared(&pool_lock); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index d28dadf3..1fa32e43 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -531,7 +531,7 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { // First brings the shadow storage up to date with the CPU's writes, which needs no command buffer. static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted - vk_mem_gpa_sync(); // a part whose shaders read through the device address table never resolved the pools they reach + vk_mem_gpa_sync(fence); // a part whose shaders read through the device address table never resolved the pools they reach vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE; int i = claim_slot(c, fence); // before reading slots: the claim may move it