From bd9b48ff0404ca4b51466d663369a103c537102a Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 05:23:43 -0500 Subject: [PATCH] Keep a fault from releasing a lazily written range unseen while a part is handed over, merge what a part skipped and must write back after all, and leave the lazy write-back off on Linux unless set. A review found a gap: a part checked what it had skipped at its end, and a fault between that check and the part's hand-over to the queue fetched without the part's writes while nothing wrote them back afterwards. The replay now opens a window from the check to the hand-over, per part, and a fault on an armed page waits for it to close (at most 20 ms; the replay thread's own faults and thaws do not wait and are counted). What a part skipped for a range that is no longer armed goes back through the usual snapshot and merge, taken in that part's prologue, and not as a plain copy, which would have put the mirror's words over what the CPU had just written there. A part split off earlier makes the same check before its prologue. wb_lazy_touch is a test that reads armed ranges on purpose. Dev build, Hyrule Field: 71.1 fps as before; standing, a fight, pause and unpause, and the touch test made 146 to 338 faults each, none inside a window, no fault of its own. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 2 +- runtime/host.h | 1 + runtime/memory.c | 63 ++++++++++++++++++++++++++++++++++++--------- runtime/vk.h | 2 ++ runtime/vk_memory.c | 47 ++++++++++++++++++++++++--------- runtime/vk_sync.c | 3 +++ 6 files changed, 93 insertions(+), 25 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 22e32eff..d21bd609 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -642,7 +642,7 @@ # wb_split = 1 # 1: what the game's shaders write into its memory is copied back from the video card once, and after that only when # the game's CPU code touches it (most of it never is: 66 MiB a frame in Hyrule Field). 0: copied back at the end of -# every GPU submission, as before. +# every GPU submission, as before. On Linux it is off unless set here. # wb_lazy = 1 # 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every # page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in diff --git a/runtime/host.h b/runtime/host.h index 24365288..c8ee1418 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -190,6 +190,7 @@ int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64 uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off); int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max); int mem_lazy_armed(uint64_t a, uint64_t n); +void mem_lazy_window(int open); void mem_lazy_release(uint64_t a, uint64_t n); void mem_lazy_report(uint64_t frame, uint64_t skipped); int mem_lazy_pick(uint64_t *addr, int max); // test: addresses of armed pages diff --git a/runtime/memory.c b/runtime/memory.c index 7016e1c7..e87439d4 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -119,6 +119,13 @@ static void unprobe_locked(uint64_t r) { // being touched), written back every submission as before. Every thaw (before the kernel writes or reads guest // memory, and before mapping changes) releases the armed ranges of its regions the same way. // A region with armed pages is not frozen (freeze_locked wants every page read-write), and a frozen one is not armed. +// A fetch brings what the GPU work handed over so far wrote. The part of a submission that is being recorded has left +// its writes out as well, and makes up for it when it finds a range it skipped no longer armed (vk_memory.c +// lazy_part_check). Between that check and the part's hand-over a release would be seen by neither side, so the +// replay opens a window there (mem_lazy_window) and a fault on an armed page waits for it to close: the fetch then +// has the part's writes. Only a fault of the replay thread itself, a wait of 20 ms, or a release by a thaw goes ahead +// inside the window (lz_gaps counts the faults): that part's writes to the range then reach guest memory with the +// range's next write-back. typedef struct { uint64_t lo, hi, h0, h1; int64_t delta; uint8_t stale; } LzSeg; // pages [lo, hi); buffer offset = address + delta static LzSeg *lzs; // frz_lock static int lz_n, lz_cap; // frz_lock @@ -129,7 +136,8 @@ static volatile int32_t lz_on; static int lz_want; // the setting, read in mem_init static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size); // frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time -static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns; +static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps; +static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else static void lazy_flags_locked(uint64_t lo, uint64_t hi) { for (uint64_t r = lo >> FRZ_SHIFT; r <= (hi - 1) >> FRZ_SHIFT; r++) { @@ -290,14 +298,40 @@ int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64 return 1; } +// Opens (under frz_lock, so no release is half done) or closes the window of the replay's part. +void mem_lazy_window(int open) { + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; + if (!open) { + __atomic_store_n(&lz_win_tid, 0, __ATOMIC_RELEASE); + return; + } + plat_mutex_lock(&frz_lock); + __atomic_store_n(&lz_win_tid, plat_thread_id(), __ATOMIC_RELEASE); + plat_mutex_unlock(&frz_lock); +} + static int lazy_fault(uint64_t a, int write) { if (a >= RT_ADDRESS_LIMIT || !__atomic_load_n(&lzr[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE)) return 0; - int hit = 0; - plat_mutex_lock(&frz_lock); - for (int i = 0; i < lz_n && !hit; i++) - if (a >= lzs[i].lo && a < lzs[i].hi) lazy_drop_locked(i, write != 0), hit = 1; - plat_mutex_unlock(&frz_lock); - return hit; + uint32_t me = plat_thread_id(); + for (int tries = 0;; tries++) { + plat_mutex_lock(&frz_lock); + int i = 0; + while (i < lz_n && !(a >= lzs[i].lo && a < lzs[i].hi)) i++; + if (i == lz_n) { + plat_mutex_unlock(&frz_lock); + return 0; + } + uint32_t w = __atomic_load_n(&lz_win_tid, __ATOMIC_ACQUIRE); + if (w && w != me && tries < 400) { // up to 20 ms: after the hand-over the fetch has that part's writes + plat_mutex_unlock(&frz_lock); + plat_wait_ns(50000); + continue; + } + if (w) lz_gaps++; + lazy_drop_locked(i, write != 0); + plat_mutex_unlock(&frz_lock); + return 1; + } } // Test (NX2NAVI_WB_LAZY_TOUCH): the first addresses of up to max armed segments, another set each call, for a caller @@ -318,15 +352,16 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; plat_mutex_lock(&frz_lock); uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes, - fns = lz_fetch_ns; + fns = lz_fetch_ns, gaps = lz_gaps; int n = lz_n; for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo; - lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = 0; + lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = lz_gaps = 0; plat_mutex_unlock(&frz_lock); rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the " - "mirror, %llu read and %llu write faults, %llu other releases, %llu fetches of %.1f MiB in %.2f ms", - (unsigned long long)frame, n, bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr, - (unsigned long long)fw, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6); + "mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of " + "%.1f MiB in %.2f ms", (unsigned long long)frame, n, bytes / 1048576.0, skipped / 1048576.0, + (unsigned long long)fr, (unsigned long long)fw, (unsigned long long)gaps, (unsigned long long)other, + (unsigned long long)nf, fb / 1048576.0, fns / 1e6); } // frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. @@ -583,7 +618,11 @@ void mem_init(void) { if (cfg_bool("NX2NAVI_WB_PROBE")) { // a diagnosis: off, no access fault reaches probe_fault and nothing is probed plat_mem_access_hook(access_fault); __atomic_store_n(&prb_on, 1, __ATOMIC_RELEASE); +#ifdef _WIN32 } else if (cfg_bool("NX2NAVI_WB_LAZY")) { // armed only once the back end can fetch (mem_lazy_enable) +#else // the fetch runs inside the access-fault handler, which has not run on this platform yet: only when set + } else if (cfg_bool("NX2NAVI_WB_LAZY") && cfg_is_set("NX2NAVI_WB_LAZY")) { +#endif plat_mem_access_hook(access_fault); lz_want = 1; } diff --git a/runtime/vk.h b/runtime/vk.h index 0c330abb..0363f160 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -323,6 +323,8 @@ void vk_mem_mirror_begin(void); int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); int vk_mem_mirror_pending(void); void vk_mem_mirror_prologue(VkCommandBuffer cb); +void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over +void vk_mem_lazy_handed(void); // Shadow storage (vk_memory.c): vk_sync.c calls vk_mem_shadow_sync right before each submit; the replay notes every GPU // write into CPU-visible memory at record time (vk_mem_gpu_pending) and reports it at the fence with vk_gpu_written, // which copies the bytes back into guest memory before telling the texture cache (vk_guest_written). diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index d2436cfa..ccc4c44e 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2115,17 +2115,20 @@ static void snap_free(void *arg) { // completion thread: the submission is don // A range a shader wrote goes back to guest memory once and is then armed (vk_gpu_written_back, mem_lazy_arm). While it // stays armed, gpu_wrote leaves the later writes of its pages in the mirror: no snapshot, no merge, no copy. When the // CPU touches a page, memory.c calls lazy_fetch with the mirror and the imported buffer of the range: a copy of its -// own, sent after every submission handed over so far (the held ones first), and waited for. A submission that is -// still being recorded when that happens has already left its write out, so vk_mem_mirror_end copies the ranges it -// skipped that are no longer armed (lz_skip). +// own, sent after every part handed over so far (the held ones first), and waited for. The part that is still being +// recorded when that happens has already left its writes out: at its end (lazy_part_check, from vk_mem_mirror_end or +// from the prologue of a part split off earlier) it writes back, as ever, what it skipped and finds no longer armed; +// the snapshot for that merge is taken in the part's own prologue, before its GPU work. From that check to the part's +// hand-over (vk_mem_lazy_handed) memory.c's window keeps a fault from releasing a range unseen. static VkCommandPool lz_pool; static VkCommandBuffer lz_cb; static VkFence lz_fence; static volatile int32_t lz_state; // 0 not asked, 1 on, 2 off static PlatMutex lz_init_lock = PLAT_MUTEX_INIT; -typedef struct { PoolRec *r; uint64_t off, size; } LzSkip; -static LzSkip *lz_skip; // replay: what this submission left in the mirror +typedef struct { VkMemSpan sp; uint64_t size; int shader; } LzSkip; +static LzSkip *lz_skip; // replay: what the part being recorded left in the mirror static int lz_nskip, lz_cskip; +static int lz_win; // replay: the part's window is open (lazy_part_check) static uint64_t lz_skipped; // replay: bytes left in the mirror since the last log line static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, @@ -2206,13 +2209,38 @@ static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { } if (i < k) { if (lz_nskip == lz_cskip) grow((void **)&lz_skip, &lz_cskip, sizeof *lz_skip); - lz_skip[lz_nskip++] = (LzSkip){r, sp->offset + (lo - g0), hi - lo}; + LzSkip k = {*sp, hi - lo, shader}; + k.sp.offset = sp->offset + (lo - g0); + lz_skip[lz_nskip++] = k; lz_skipped += hi - lo; } run = hi; } } +// The end of the part being recorded: what it left in the mirror for a range that is no longer armed (the CPU touched +// it meanwhile) is written back after all. Opens memory.c's window, which vk_mem_lazy_handed closes. +static void lazy_part_check(void) { + if (lz_win || !lz_nskip) return; + mem_lazy_window(1); + lz_win = 1; + for (int i = 0; i < lz_nskip; i++) { + const PoolRec *r = lz_skip[i].sp.pool; + if (!mem_lazy_armed(r->import_lo + lz_skip[i].sp.offset, lz_skip[i].size)) + gpu_wrote_range(&lz_skip[i].sp, lz_skip[i].size, lz_skip[i].shader); + } +} + +// vk_sync.c, before a part's prologue is recorded: the check for a part that was split off (the last part's ran in +// vk_mem_mirror_end already). +void vk_mem_lazy_part_end(void) { lazy_part_check(); } + +// vk_sync.c, once the part is held or handed to the queue: a fetch from here on has its writes. +void vk_mem_lazy_handed(void) { + lz_nskip = 0; + if (lz_win) mem_lazy_window(0), lz_win = 0; +} + static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; @@ -2482,12 +2510,7 @@ int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); } int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { if (!vk_mirror_on) return 0; int rec = 0; - // What this submission left in the mirror for a range that was armed then and is not now (the CPU touched it while - // the submission was recorded): its bytes go down after all, as a copy. - for (int i = 0; i < lz_nskip; i++) - if (!mem_lazy_armed(lz_skip[i].r->import_lo + lz_skip[i].off, lz_skip[i].size)) - range_add(&wbl, &nwbl, &cwbl, lz_skip[i].r, lz_skip[i].off, lz_skip[i].size); - lz_nskip = 0; + lazy_part_check(); // before the merges are recorded: what the last part skipped and has to write back after all if (nsnl) { // the merge reads the mirror the submission's shaders wrote and writes the imported buffers mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 6d9e1176..aba2794f 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -500,6 +500,7 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { // First brings the shadow storage up to date with the CPU's writes, which needs no command buffer. static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted + vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE; int i = claim_slot(c, fence); // before reading slots: the claim may move it VkCommandBuffer cb = slots[i].cb; @@ -536,6 +537,7 @@ static VkCommandBuffer split_part(int exec) { si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence; ts_submitted(); queue_submit(c, &si, 0); // held or handed to the queue (submit batching) + vk_mem_lazy_handed(); vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, @@ -607,6 +609,7 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { .pSignalSemaphoreInfos = sig}; ts_submitted(); queue_submit(c, &si, fence); // held or handed to the queue (submit batching) + vk_mem_lazy_handed(); s->fence = recorded ? fence : 0; vk_mem_note_submit(fence); plat_mutex_lock(&st_lock);