From afe44e0e5488e27d17743ff7db785656d0a1d096 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:09:42 -0500 Subject: [PATCH] Let the device address table read a new VRAM mirror only once the mirror's first copy has run. A mirror is made while a submission is recorded, and its first copy of the pool is in that submission's prologue. The table's pages moved to the mirror at once, and the table is memory the shaders read as they run: every part already handed to the queue read the mirror through it before the copy had run, memory nothing had written yet and, on a card over its budget, a block the driver had not made resident for those parts. A GTX 1650 4 GB lost its device that way at area loads in every session of 0.7.1 (#507): six reports of a GPU read fault at byte 0 of a mirror made in the faulting submission, each the first byte of a new 2 MiB-aligned block, with mirrors being made over the budget, and none with the table kept empty. The mirror code is the same in 0.9.2. Now the pages keep reading the imported buffer and take the mirror as their twin, so what shaders write through the table goes to both and what they read is guest memory, as before the mirror existed. Once the GPU has finished the submission that holds the copy, the pages move to the mirror with the imported buffer as the twin. gpa_late = 0 moves them at once, as before. tests/gpa_check.py covers the new table step; the mirror side was compiled and run through --probe-vkmem, not played. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 5 +++ runtime/cfg.c | 2 +- runtime/vk_gpa.c | 48 +++++++++++++++++++++------- runtime/vk_gpa.h | 4 +++ runtime/vk_memory.c | 77 +++++++++++++++++++++++++++++++++++++++++---- tests/gpa_check.c | 23 ++++++++++++++ 6 files changed, 140 insertions(+), 19 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 579f560b..076febae 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -976,3 +976,8 @@ # those second writes cost about 3 ms of GPU time a frame. 0 (default): every mirror sends them. A pool that needs them # and is left without shows as foliage stretched into columns or flickering. # gpa_twin_max_kb = 0 +# 1 (default): shaders that follow the game's pointers read a memory pool's new copy in video memory only once that +# copy has been filled; until then they read the game's own memory. 0: they read the new copy at once, as up to 0.9.2, +# where a card over its video memory budget could lose the device at an area load (a GPU read fault at the first byte +# of the new copy). +# gpa_late = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index 8075fbf4..0d843815 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -47,7 +47,7 @@ static const char *const known[] = { "DESC_TEMPLATE", "DEVICE_PROFILE", "DMABUF", "DMABUF_MB", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DRAW_CRUMBS", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "GPU_TIMELINE", "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", - "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", + "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_LATE", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_LIVE", "HEAP_LAZY", "HDR_PEAK_NITS", "HDR_UI_NITS", // retired: the HDR look's settings of earlier versions, now fixed (vk_hdr.c, shaders/hdr_tonemap.frag); read by // nothing, listed so that a file which still sets them logs no unknown key diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index 32cec945..9daf0a14 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -319,20 +319,20 @@ void vk_gpa_clear(uint64_t lo, uint64_t hi) { blocks_release(p0, p1); } -static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { - if (!vk_gpa_on() || !buf) return old_rec; - uint64_t dev = vk_gpa_device_address(buf); - uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; - plat_mutex_lock(&gpa_lock); - // the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too. - // NX2NAVI_GPA_TWIN=0 (diagnosis): no twin, the writes go to the mirror alone, as before patch 0034. +// NX2NAVI_GPA_TWIN=0 (diagnosis): no twin, the writes go to the mirror alone, as before patch 0034. +static int twins_on(void) { static int twins = -1; if (twins < 0) { const char *e = cfg_get("NX2NAVI_GPA_TWIN"); twins = !(e && !strcmp(e, "0")); } - uint64_t twin = old_rec && twins && keep_twin ? (uint64_t)G.cpu[REC_WORD + old_rec * REC_WORDS] | - (uint64_t)G.cpu[REC_WORD + old_rec * REC_WORDS + 1] << 32 - : dev - buf_lo; - uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0; - if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } + return twins; +} + +// gpa_lock held: the first address of record rec (the buffer its pages read). +static uint64_t rec_first(uint32_t rec) { + return (uint64_t)G.cpu[REC_WORD + rec * REC_WORDS] | (uint64_t)G.cpu[REC_WORD + rec * REC_WORDS + 1] << 32; +} + +// gpa_lock held: the pages of [p0, p1) that name old_rec (or none) name r from here on. +static void pages_move(uint64_t p0, uint64_t p1, uint32_t old_rec, uint32_t r) { for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 1); if (!w) { @@ -342,11 +342,35 @@ static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, } if (!G.cpu[w] || G.cpu[w] == old_rec) page_put(w, r); } +} + +static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { + if (!vk_gpa_on() || !buf) return old_rec; + uint64_t dev = vk_gpa_device_address(buf); + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + // the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too. + uint64_t twin = old_rec && twins_on() && keep_twin ? rec_first(old_rec) : dev - buf_lo; + uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0; + if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } + pages_move(p0, p1, old_rec, r); plat_mutex_unlock(&gpa_lock); if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); return r; } +uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { + if (!old_rec || !buf || !vk_gpa_on() || !twins_on()) return 0; + uint64_t dev = vk_gpa_device_address(buf); + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + uint32_t r = dev ? rec_alloc(rec_first(old_rec), dev - buf_lo) : 0; + if (r) pages_move(p0, p1, old_rec, r); + plat_mutex_unlock(&gpa_lock); + if (r) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); + return r; +} + uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { return remap(lo, hi, old_rec, buf, buf_lo, 1); } diff --git a/runtime/vk_gpa.h b/runtime/vk_gpa.h index f5e1d4e1..29df805e 100644 --- a/runtime/vk_gpa.h +++ b/runtime/vk_gpa.h @@ -24,5 +24,9 @@ uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, // The same without the twin: the pages name buf alone (a pool whose VRAM mirror is evicted goes back to its imported // buffer). uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); +// The pages of [lo, hi) that name old_rec keep reading old_rec's buffer and get buf as their twin: shaders read the +// pool where they did and write buf too (a VRAM mirror until its first copy has run, vk_memory.c mirror_create). +// Returns the new record, or 0 when there is none to give or twins are off (old_rec then stays as it is). +uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); void vk_gpa_report(void); void vk_gpa_shutdown(void); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 33308bc8..c1f1e447 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -61,6 +61,8 @@ typedef struct PoolRec { // VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only. uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED uint32_t gpa_rec; // POOL_IMPORT: its record in the device address table (vk_gpa.c), 0 none + uint64_t gpa_at; // pool_lock: the submission after which the table reads the mirror (gpa_switch); + // 0 when it does already, or there is no mirror struct GpaMap { uint64_t lo, hi; uint32_t rec; } *vmaps; // POOL_VIRT: its mapped ranges in the table int nvmaps, cvmaps; VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf @@ -2756,6 +2758,52 @@ static int twin_wanted(const PoolRec *r) { return !twin_max || (int64_t)(r->import_hi - r->import_lo) <= twin_max; } +// ---- when the device address table starts reading a new mirror (NX2NAVI_GPA_LATE, default 1; 0 = at once, as before) +// A mirror is made while the replay records a submission, and its first copy of the pool is in that submission's +// prologue. The table is memory the shaders read as they run, so with its pages moved to the mirror at once, every +// part already handed to the queue (the submissions before, and the earlier parts of this one) read the mirror through +// the table before that copy had run: memory nothing had written yet, and on a card over its budget a block the +// driver had not made resident for those parts. A GTX 1650 4 GB lost its device that way at area loads in every +// session: six reports of a GPU read fault at byte 0 of a mirror made in the faulting submission, each the first byte +// of a new 2 MiB-aligned block, with mirrors being made over the budget; with the table kept empty the same routes ran +// without a loss. +// Now the pages keep reading the imported buffer and name the mirror as their twin (vk_gpa_remap_writes): what shaders +// write through the table goes to both, so the mirror misses nothing, and what they read is guest memory, as before +// the mirror existed. Once the GPU has finished the submission whose prologue holds the copy, the pages move to the +// mirror with the imported buffer as the twin (gpa_switch, at the start of a later submission). In between, a shader +// that reads through the table what another wrote through a buffer binding sees guest memory, which gets such a write +// at the write-back; data written through the table itself, as the game's foliage, instance and light records are, +// is read where it was written. +static int n_gpa_wait; // pool_lock: mirrors whose pages still read the imported buffer (PoolRec.gpa_at) +static uint64_t n_gpa_late; // replay: pages moved to their mirror after its first copy, so far + +static int gpa_late(void) { + static int on = -1; + if (on < 0) { + on = rt_knob("GPA_LATE", "1") == 1; + rt_log("mem: the device address table reads a new VRAM mirror %s (NX2NAVI_GPA_LATE)", + on ? "once the mirror's first copy has run" : "at once, as before"); + } + return on; +} + +// Replay thread, at the start of a submission: the mirrors whose first copy the GPU has finished take the table's reads. +static void gpa_switch(void) { + if (!__atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) return; + uint64_t done = vk_sync_fence_done(); + plat_mutex_lock(&pool_lock); + for (int i = 0; i < nm && n_gpa_wait; i++) { + PoolRec *r = mlist[i]; + if (!r->gpa_at || done < r->gpa_at) continue; + r->gpa_at = 0, n_gpa_wait--; + if (!r->gpa_rec || !r->mbuf) continue; + r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) + : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + n_gpa_late++; + } + plat_mutex_unlock(&pool_lock); +} + // Replay thread: the pool's mirror on its first buffer use. static void mirror_create(PoolRec *r) { VkCtx *c = vk_ctx(); @@ -2848,16 +2896,28 @@ static void mirror_create(PoolRec *r) { r->acc_sub = msub; r->mstate = MS_ON; r->asked_room = 0; - // The device address table names the mirror from now on: the GPU's writes go there, and the sync copies the CPU's. - // With a twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. + // The device address table names the mirror: the GPU's writes go there, and the sync copies the CPU's. With a + // twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. Reads + // move to the mirror once its first copy has run (gpa_late, below). r->twin = (uint8_t)twin_wanted(r); if (!r->twin) { plat_inc64(&m_untwinned_n); plat_add64(&m_untwinned_bytes, (int64_t)n); } - if (r->gpa_rec) - r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) - : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + if (r->gpa_rec) { + plat_mutex_lock(&pool_lock); + uint32_t late = gpa_late() ? vk_gpa_remap_writes(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) : 0; + if (late) { + uint64_t f = vk_exec_recording_fence(); + r->gpa_rec = late; + r->gpa_at = f ? f : vk_sync_issued() + 1; // the submission being recorded: its prologue holds the copy + n_gpa_wait++; + } else { + r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) + : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + } + plat_mutex_unlock(&pool_lock); + } range_add(&pend, &npend, &cpend, r, 0, n); mrep.first_n++, mrep.first_bytes += n; plat_inc64(&m_live_n); @@ -2866,6 +2926,7 @@ static void mirror_create(PoolRec *r) { static void mirror_forget(PoolRec *r) { __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); + if (r->gpa_at) r->gpa_at = 0, n_gpa_wait--; // the caller takes the table's pages off the mirror for (int i = 0; i < nm; i++) if (mlist[i] == r) { memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist); @@ -3932,7 +3993,7 @@ static void hot_check(void) { } void vk_mem_mirror_begin(void) { - if (vk_mirror_on) hot_check(); + if (vk_mirror_on) gpa_switch(), hot_check(); if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions if (!freeze_idle) return; plat_mutex_lock(&gen_lock); @@ -4387,6 +4448,10 @@ static void mirror_log(uint64_t frame) { "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; flushed and not copied (write-watched) %.3f MiB%s", (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M, a.flush_skipped / f / M, capl); + if (n_gpa_late || __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) + rt_log("mem: mirror frame %llu: the device address table took up %llu mirrors after their first copy so far, %d " + "wait for theirs (NX2NAVI_GPA_LATE)", (unsigned long long)frame, (unsigned long long)n_gpa_late, + __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)); } // After vram_log: every heap's usage as the driver reports it (VK_EXT_memory_budget) beside VMA's share of it, VMA by diff --git a/tests/gpa_check.c b/tests/gpa_check.c index 2d194db0..7bda3d39 100644 --- a/tests/gpa_check.c +++ b/tests/gpa_check.c @@ -125,6 +125,29 @@ int main(void) { CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "a plain page is its own twin"); gpu_done(); + // a new mirror takes the writes first and the reads once its first copy has run (vk_memory.c mirror_create, + // gpa_switch): until then the pages read the imported buffer and the mirror is their twin + VkBuffer m1 = new_buf(); + uint32_t w0 = vk_gpa_remap_writes(a0, a1, r0, m1, a0); + CHECK(w0 && w0 != r0, "a new mirror gets a record for the writes"); + expect(a0, a1, b0, a0, "a new mirror before its first copy: reads stay on the imported buffer"); + CHECK(read_through(a0 + 4096, 1) == dev_of(m1) + 4096 && read_through(a1 - 4096, 1) == dev_of(m1) + (a1 - 4096 - a0), + "a new mirror before its first copy is the twin, which takes the writes"); + gpu_done(); + r0 = vk_gpa_remap(a0, a1, w0, m1, a0); + expect(a0, a1, m1, a0, "a new mirror after its first copy: reads move to it"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "and the imported buffer is the twin again"); + // evicted while it still waits for its first copy: the pages go back to the imported buffer alone + r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0); + gpu_done(); + VkBuffer m2 = new_buf(); + w0 = vk_gpa_remap_writes(a0, a1, r0, m2, a0); + r0 = vk_gpa_remap_plain(a0, a1, w0, b0, a0); + expect(a0, a1, b0, a0, "a mirror gone before its first copy"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "leaves the imported buffer as its own twin"); + CHECK(!vk_gpa_remap_writes(a0, a1, 0, m2, a0), "a pool without a record gets none for a mirror's writes"); + gpu_done(); + // an unmapped range reads the zero buffer at once, and its blocks serve no other range while the GPU may still read // the old L1 entries uint32_t old_blk[3] = {block_of(a0), block_of(a0 + 16 * MiB), block_of(a0 + 32 * MiB)};