From e473dd8826ed5c7b0d956440ead5efc1d4d6af3f Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 05:37:14 -0500 Subject: [PATCH] Arm a lazily written range only when its mirror holds every CPU write to it, so a fetch no longer copies older mirror bytes over what the CPU wrote. A fetch copies the mirror over the whole range, not the merge of the bytes the GPU changed that a write-back does. The completion thread arms a range after its write-back reached guest memory; a CPU write into those pages in between, not yet carried up by a sync of the mirror, was then overwritten by the mirror's older bytes at the first fetch. In play that was single frames with another frame's constants: the night lighting missing from the whole scene in Hyrule Field every 0.1 to 0.4 s, and a black scene in Kakariko, gone with wb_lazy = 0. Now the pages are made inaccessible first and the back end then checks the CPU-write page generations against the mirror's last sync and the write-watch bits nobody has read yet; when either shows a write the mirror lacks, the pages are given back and the range arms at a later write-back. The lazy line counts the arms refused. Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 5 +++-- runtime/memory.c | 38 +++++++++++++++++++++++++++----------- runtime/vk_memory.c | 24 +++++++++++++++++++++--- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index c8ee1418..81effc41 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -186,8 +186,9 @@ void mem_thaw(uint64_t a, uint64_t n); // makes the frozen r void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws); // NX2NAVI_WB_PROBE=1 (memory.c, "Probed regions"): who reads what the GPU wrote back into guest memory. // NX2NAVI_WB_LAZY (memory.c, "Lazy write-back"): pages the GPU wrote that come down only when the CPU touches them. -int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size)); -uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off); +int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size), + int (*clean)(uint64_t lo, uint64_t hi, void *ctx)); +uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off, void *ctx); int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max); int mem_lazy_armed(uint64_t a, uint64_t n); void mem_lazy_window(int open); diff --git a/runtime/memory.c b/runtime/memory.c index e87439d4..2906b1b5 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -135,6 +135,10 @@ static uint8_t lz_back[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: secon static volatile int32_t lz_on; static int lz_want; // the setting, read in mem_init static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size); +// The back end's test that the mirror holds every CPU write to [lo, hi) (vk_memory.c lazy_clean): a fetch copies the +// mirror's bytes over the whole range, so a CPU write the mirror does not have yet would be lost to it. +static int (*lz_clean)(uint64_t lo, uint64_t hi, void *ctx); +static uint64_t lz_unclean; // frz_lock: arms refused since the last report because the mirror lacked a CPU write // frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps; static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else @@ -189,8 +193,13 @@ static int lazy_covered_locked(uint64_t lo, uint64_t hi) { // frz_lock held exclusively: [x, e), inside one region, becomes inaccessible when every committed page of it is // read-write, or inaccessible as part of a segment already; anything else (a read-only page, a hole, a page the game -// made inaccessible itself) leaves it as it is. -static int lazy_protect_locked(uint64_t x, uint64_t e) { +// made inaccessible itself) leaves it as it is. Then the back end checks that the mirror ctx belongs to holds every CPU +// write to [x, e) (lz_clean): after the protection no write can come in, so a write that came before it is in the +// write-watch or the page generations. When the mirror lacks one, the pages are given back and nothing is armed: the +// CPU wrote into the range after its write-back reached guest memory and before this arm, and the next sync of the +// mirror carries that write up; a fetch before it would have copied the mirror's older bytes over it (single frames +// with another frame's lighting constants, or a black scene). +static int lazy_protect_locked(uint64_t x, uint64_t e, void *ctx) { uint64_t run[16][2]; int k = 0; for (uint64_t a = x, b; a < e; a = b) { @@ -205,12 +214,17 @@ static int lazy_protect_locked(uint64_t x, uint64_t e) { while (i--) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW); return 0; } + if (lz_clean && !lz_clean(x, e, ctx)) { + for (int i = 0; i < k; i++) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW); + lz_unclean++; + return 0; + } return 1; } // Arms the whole pages of [a, a + n), which the back end has just written back from the buffer h0 into h1 at offset // off: they fault on their next access. Returns the bytes armed now. Not in a frozen, probed, held or thaw-held region. -uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off) { +uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off, void *ctx) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || a >= RT_ADDRESS_LIMIT || !watched(a, n)) return 0; uint64_t lo = (a + PAGE - 1) & ~(PAGE - 1), hi = (a + n) & ~(PAGE - 1), armed = 0; if (hi <= lo) return 0; @@ -226,7 +240,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t e = (r + 1) << FRZ_SHIFT; if (e > hi) e = hi; if (frz[r] || prb[r] || (lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0) || - (thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) || !lazy_protect_locked(x, e)) + (thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) || !lazy_protect_locked(x, e, ctx)) continue; uint64_t slo = x, shi = e; uint8_t stale = 0; @@ -291,9 +305,10 @@ void mem_lazy_release(uint64_t a, uint64_t n) { // The back end is ready to fetch: arming starts. 0 when the setting is off (or the probe is on, which watches the // same pages). -int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size)) { +int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size), + int (*clean)(uint64_t lo, uint64_t hi, void *ctx)) { if (!lz_want) return 0; - lz_fetch = fetch; + lz_fetch = fetch, lz_clean = clean; __atomic_store_n(&lz_on, 1, __ATOMIC_RELEASE); return 1; } @@ -352,16 +367,17 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; plat_mutex_lock(&frz_lock); uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes, - fns = lz_fetch_ns, gaps = lz_gaps; + fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean; int n = lz_n; for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo; - lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = lz_gaps = 0; + lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = lz_gaps = lz_unclean = 0; plat_mutex_unlock(&frz_lock); rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the " "mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of " - "%.1f MiB in %.2f ms", (unsigned long long)frame, n, bytes / 1048576.0, skipped / 1048576.0, - (unsigned long long)fr, (unsigned long long)fw, (unsigned long long)gaps, (unsigned long long)other, - (unsigned long long)nf, fb / 1048576.0, fns / 1e6); + "%.1f MiB in %.2f ms, %llu arms refused (a CPU write not in the mirror yet)", (unsigned long long)frame, n, + bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr, (unsigned long long)fw, + (unsigned long long)gaps, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6, + (unsigned long long)unclean); } // frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index cad8d492..6ae3d250 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1150,15 +1150,16 @@ static void gpu_written(uint64_t addr, uint64_t size, int back) { void vk_gpu_written(uint64_t addr, uint64_t size) { gpu_written(addr, size, 0); } static int wb_lazy(void); +static int lazy_clean(uint64_t lo, uint64_t hi, void *ctx); void vk_gpu_written_back(uint64_t addr, uint64_t size) { gpu_written(addr, size, 1); mem_probe(addr, size); // NX2NAVI_WB_PROBE: the bytes are in guest memory now; note who reads them next VkMemSpan sp; if (size && wb_lazy() && vk_mem_resolve(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on - const PoolRec *r = sp.pool; + PoolRec *r = sp.pool; if (r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo && addr + size <= r->import_hi) - mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo); + mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo, r); } } @@ -2163,6 +2164,23 @@ static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) { vkResetFences(c->device, 1, &lz_fence); } +// memory.c's test before it arms [lo, hi) of the pool ctx (frz_lock held there, the pages already inaccessible): the +// mirror holds every CPU write to them. A fetch copies the mirror over the whole range, not the merge of the bytes the +// GPU changed that a write-back does, so a CPU write that came after the range's write-back reached guest memory and +// before this arm, and that no sync of the mirror has carried up yet, would be overwritten by the mirror's older bytes: +// a page stamped after the mirror's last sync (the CPU-write table, mgen), or a write-watch bit nobody has read yet. +// No lock: the pages are inaccessible, so nothing stamps them meanwhile; the mirror's mgen only grows. +static int lazy_clean(uint64_t lo, uint64_t hi, void *ctx) { + const PoolRec *r = ctx; + uint32_t mgen = __atomic_load_n(&r->mgen, __ATOMIC_ACQUIRE); + for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) { + const uint32_t *ch = __atomic_load_n(&cgen_chunk[a >> GEN_CHUNK_SHIFT], __ATOMIC_ACQUIRE); + if (ch && __atomic_load_n(&ch[(a >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)], __ATOMIC_RELAXED) > mgen) return 0; + } + uint64_t p[1]; + return mem_written_peek(lo, hi - lo, p, 1) == 0; // UINT64_MAX (not watched) counts as written +} + static int wb_lazy(void) { int s = __atomic_load_n(&lz_state, __ATOMIC_ACQUIRE); if (s) return s == 1; @@ -2179,7 +2197,7 @@ static int wb_lazy(void) { .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; on = on && vkAllocateCommandBuffers(c->device, &ai, &lz_cb) == VK_SUCCESS && vkCreateFence(c->device, &fi, NULL, &lz_fence) == VK_SUCCESS; - on = on && mem_lazy_enable(lazy_fetch); + on = on && mem_lazy_enable(lazy_fetch, lazy_clean); } rt_log("mem: what shaders write into mirrored pools %s (NX2NAVI_WB_LAZY)", on ? "goes back to guest memory once and then only when the CPU touches it"