From ebb37689cf9d6d1f1ec1ed995d4ed1acdda090c4 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Wed, 11 Nov 2026 15:35:25 -0500 Subject: [PATCH] Thaw a frozen region's pages outside the freeze lock, from the runs its freeze kept, so a thaw slowed by the driver's pinning no longer holds the replay and the game's other threads. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 +- runtime/memory.c | 167 ++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 153 insertions(+), 16 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index cf26c264..01d7f708 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -46,7 +46,7 @@ static const char *const known[] = { "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLSS_IDLE_FREE", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", - "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_BUDGET_US", "FREE_CORES", + "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_LIVE", "HEAP_LAZY", "HDR_PEAK_NITS", "HDR_UI_NITS", // retired: the HDR look's settings of earlier versions, now fixed (vk_hdr.c, shaders/hdr_tonemap.frag); read by diff --git a/runtime/memory.c b/runtime/memory.c index 8fd095d7..6f0030f5 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -93,6 +93,21 @@ static volatile int64_t frz_hot_holds; // quick thaws tha static volatile int32_t nfrz; static PlatMutex frz_lock = PLAT_MUTEX_INIT; static volatile int64_t frz_count[4]; // freezes; thaws by a write fault, by mem_thaw, by a mapping change +// Thaws outside frz_lock (NX2NAVI_FREEZE_ASYNC, default 1; 0: every thaw under the lock, as before). While the game +// streams an area it creates and finalizes up to 1,500 memory pools a second, the driver pins and unpins the imported +// ones, and the VirtualQuery and VirtualProtect calls of a thaw then take 10-20 ms each. Under frz_lock that held the +// replay (mem_lazy_skip, the freeze pass) and every other thread that faulted or thawed, a 25-45 ms frame. So a thaw by +// a write fault or by mem_thaw marks its regions FRZ_THAWING under the lock, makes their pages read-write without it and +// ends them under it again. A region being thawed is no longer frozen for anyone who asks (a freeze, an arm, a probe +// leave it alone); a fault in it waits for the thaw to end and runs again; a mapping change waits for it first +// (thaw_wait_locked), so a late VirtualProtect never undoes the change; mem_thaw makes its pages read-write itself. +// freeze_locked keeps the runs it made read-only (frz_runs), so a thaw needs no VirtualQuery. +#define FRZ_THAWING 2 +static int frz_async = 1; // the setting, read in mem_init +typedef struct { uint8_t n; uint16_t run[16][2]; } FrzRuns; // pages of the region, [first, end) +static FrzRuns *frz_runs[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock (written at a freeze): NULL or n 0 = not kept +static volatile int64_t frz_async_n; // thaws done outside the lock, since the start +static volatile int64_t frz_protect_fails; // VirtualProtect failures of those thaws, since the start // Probed regions (NX2NAVI_WB_PROBE=1, a diagnosis). The back end copies what the GPU wrote into a mirrored pool back // into guest memory after every submission, in case the CPU or a host path reads it; whether anything does is what @@ -118,7 +133,7 @@ static int watched(uint64_t addr, uint64_t size) { // frz_lock held exclusively: the pages of a probed region get back what they had, read-only in a frozen region. static void unprobe_locked(uint64_t r) { if (!prb[r]) return; - int prot = frz[r] ? PLAT_PROT_READ : PLAT_PROT_RW; + int prot = frz[r] == 1 ? PLAT_PROT_READ : PLAT_PROT_RW; uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, b; for (; a < end; a = b) // every inaccessible run is the probe's: mem_probe took only regions without one if (plat_mem_state(a, end, &b) == PLAT_PROT_NONE && !plat_mem_protect(a, b - a, prot)) @@ -578,6 +593,13 @@ void mem_lazy_note(char *out, size_t n) { // One log line since the last one; skipped: the bytes the back end left in the mirror since then. void mem_lazy_report(uint64_t frame, uint64_t skipped) { + static int64_t async_said = -1; + int64_t an = __atomic_load_n(&frz_async_n, __ATOMIC_RELAXED), af = __atomic_load_n(&frz_protect_fails, __ATOMIC_RELAXED); + if ((rt_trace & RT_TRACE_PERF) && an != async_said) { + async_said = an; + rt_log("mem: thaws outside the freeze lock so far %lld (NX2NAVI_FREEZE_ASYNC), page protections that failed there " + "%lld", (long long)an, (long long)af); + } if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; plat_mutex_lock(&frz_lock); uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes, @@ -628,19 +650,56 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { (unsigned long long)calls, (unsigned long long)small); } -// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. -static void thaw_locked(uint64_t r, int why) { +// Makes the pages region r's freeze made read-only read-write again: the runs freeze_locked kept, else every read-only +// run (freeze_locked left reserved holes, heap pages given back, as they were; every committed page of a frozen region +// was read-write at the freeze, and mapping changes thaw or wait before they run, so these are all ours). strict: under +// frz_lock, a failure ends the process; outside it (a thaw in flight, FRZ_THAWING) the pages are what the freeze left, +// and a failure is only counted. +static void thaw_pages(uint64_t r, int strict) { + uint64_t base = r << FRZ_SHIFT; + const FrzRuns *k = frz_runs[r]; + if (k && k->n) { + for (int i = 0; i < k->n; i++) { + uint64_t a = base + ((uint64_t)k->run[i][0] << 12), n = (uint64_t)(k->run[i][1] - k->run[i][0]) << 12; + if (!plat_mem_protect(a, n, PLAT_PROT_RW)) { + if (strict) + rt_fatal("memory: thawing %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)n); + plat_inc64(&frz_protect_fails); + } + } + return; + } + uint64_t a = base, end = a + FRZ_SIZE, b; + for (; a < end; a = b) + if (plat_mem_state(a, end, &b) == PLAT_PROT_READ && !plat_mem_protect(a, b - a, PLAT_PROT_RW)) { + if (strict) rt_fatal("memory: thawing %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)(b - a)); + plat_inc64(&frz_protect_fails); + } +} + +// frz_lock held exclusively: what a thaw does before the pages (the hold, the lazy write-back's ranges, a probe). +// Returns 1 when the region is frozen. +static int thaw_begin_locked(uint64_t r, int why) { if (why == 2) thaw_ms[r] = (uint32_t)plat_now_ms() | 1; if (lzr[r]) lazy_drop_region_locked(r, why + 1); // not a fault on an armed page: no hold if (prb[r]) unprobe_locked(r); - if (!frz[r]) return; - // The read-only runs only: freeze_locked left reserved holes (heap pages given back) as they were. Every committed - // page of a frozen region was read-write at the freeze, and mapping changes thaw before they run, so these are all - // ours. - uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, b; - for (; a < end; a = b) - if (plat_mem_state(a, end, &b) == PLAT_PROT_READ && !plat_mem_protect(a, b - a, PLAT_PROT_RW)) - rt_fatal("memory: thawing %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)(b - a)); + return frz[r] == 1; +} + +static void thaw_end_locked(uint64_t r, int why); + +// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. +// A region another thread is thawing (FRZ_THAWING) gets its pages made read-write here too, and is ended here. +static void thaw_locked(uint64_t r, int why) { + if (!thaw_begin_locked(r, why) && frz[r] != FRZ_THAWING) return; + thaw_pages(r, 1); + __atomic_store_n(&frz[r], FRZ_THAWING, __ATOMIC_RELEASE); + thaw_end_locked(r, why); +} + +// frz_lock held exclusively: the end of a thaw. Nothing when another thread ended it meanwhile. +static void thaw_end_locked(uint64_t r, int why) { + if (frz[r] != FRZ_THAWING) return; __atomic_store_n(&frz[r], 0, __ATOMIC_RELEASE); plat_dec32(&nfrz); plat_inc64(&frz_count[why]); @@ -664,6 +723,20 @@ static void thaw_range_locked(uint64_t a, uint64_t n, int why) { for (uint64_t r = r0; r <= r1; r++) thaw_locked(r, why); } +// frz_lock held exclusively: returns once no region of [a, a + n) is being thawed outside the lock (FRZ_THAWING), +// releasing the lock while it waits. A mapping change runs after this: the thaw's VirtualProtect then cannot land on +// the pages after the change and undo it. +static void thaw_wait_locked(uint64_t a, uint64_t n) { + if (!n || a >= RT_ADDRESS_LIMIT) return; + uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT; + for (uint64_t r = r0; r <= r1; r++) + while (__atomic_load_n(&frz[r], __ATOMIC_ACQUIRE) == FRZ_THAWING) { + plat_mutex_unlock(&frz_lock); + plat_wait_ns(20000); + plat_mutex_lock(&frz_lock); + } +} + // Lazy heap (NX2NAVI_HEAP_LAZY, Windows only; off by default). Windows charges every committed page to the system's // commit at once, and SetHeapSize asks for the whole application heap up front: in Castle Town after Hyrule Field the // heap held 6.8 GiB of commit of which 3.4 GiB had ever been resident, with 0.45 GB in the page file. With it on, @@ -738,8 +811,39 @@ static int heap_lazy_fault(uint64_t a) { void mem_thaw(uint64_t a, uint64_t n) { plat_mutex_lock(&frz_lock); // also when nothing is frozen: the hold keeps a read's buffer writable - if (heap_lazy) heap_lazy_commit_locked(a, n); // the kernel's write would fail on a reserved page as well - thaw_range_locked(a, n, 2); + if (heap_lazy || !frz_async || !n || a >= RT_ADDRESS_LIMIT) { + if (heap_lazy) { + thaw_wait_locked(a, n); // it commits pages: a mapping change + heap_lazy_commit_locked(a, n); // the kernel's write would fail on a reserved page as well + } + thaw_range_locked(a, n, 2); + plat_mutex_unlock(&frz_lock); + return; + } + // The regions frozen now are marked FRZ_THAWING and made read-write without the lock (see FRZ_THAWING), up to + // MANY at a time; those another thread is thawing are made read-write here too, as the caller's kernel access needs + // every page of them now, and that thread ends them. + enum { MANY = 64 }; + uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT; + for (uint64_t r = r0; r <= r1;) { + uint64_t mine[MANY], theirs[MANY]; + int nm = 0, nt = 0; + for (; r <= r1 && nm < MANY && nt < MANY; r++) { + if (thaw_begin_locked(r, 2)) { + __atomic_store_n(&frz[r], FRZ_THAWING, __ATOMIC_RELEASE); + mine[nm++] = r; + } else if (frz[r] == FRZ_THAWING) { + theirs[nt++] = r; + } + } + if (!nm && !nt) continue; + plat_mutex_unlock(&frz_lock); + for (int i = 0; i < nm; i++) thaw_pages(mine[i], 0); + for (int i = 0; i < nt; i++) thaw_pages(theirs[i], 0); + plat_add64(&frz_async_n, nm); + plat_mutex_lock(&frz_lock); + for (int i = 0; i < nm; i++) thaw_end_locked(mine[i], 2); + } plat_mutex_unlock(&frz_lock); } @@ -760,6 +864,13 @@ static int freeze_locked(uint64_t r) { while (i--) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW); return 0; } + // the runs, for a thaw without VirtualQuery (thaw_pages); kept per region from its first freeze on + if (n && !frz_runs[r]) frz_runs[r] = calloc(1, sizeof *frz_runs[r]); + if (frz_runs[r]) { + frz_runs[r]->n = (uint8_t)n; + for (int i = 0; i < n; i++) + frz_runs[r]->run[i][0] = (uint16_t)((run[i][0] - a) >> 12), frz_runs[r]->run[i][1] = (uint16_t)((run[i][1] - a) >> 12); + } return n > 0; } @@ -898,9 +1009,30 @@ void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws // The platform's write-fault hook (plat_mem_fault_hook): 1 retries the write. static int thaw_fault(uint64_t a) { if (a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return 0; + uint64_t r = a >> FRZ_SHIFT; plat_mutex_lock(&frz_lock); - int was = frz[a >> FRZ_SHIFT]; - thaw_locked(a >> FRZ_SHIFT, 1); + int was = frz[r]; + if (was == FRZ_THAWING) { // another thread's thaw: the write runs again once it ended + while (frz[r] == FRZ_THAWING) { + plat_mutex_unlock(&frz_lock); + plat_wait_ns(20000); + plat_mutex_lock(&frz_lock); + } + plat_mutex_unlock(&frz_lock); + return 1; + } + if (frz_async && was == 1) { // its pages outside the lock (FRZ_THAWING) + thaw_begin_locked(r, 1); + __atomic_store_n(&frz[r], FRZ_THAWING, __ATOMIC_RELEASE); + plat_mutex_unlock(&frz_lock); + thaw_pages(r, 0); + plat_inc64(&frz_async_n); + plat_mutex_lock(&frz_lock); + thaw_end_locked(r, 1); + plat_mutex_unlock(&frz_lock); + return 1; + } + thaw_locked(r, 1); plat_mutex_unlock(&frz_lock); if (was) return 1; uint64_t end; @@ -910,6 +1042,7 @@ static int thaw_fault(uint64_t a) { // Mapping changes thaw the regions they touch and run under frz_lock, so no freeze sees them half done. static void commit(uint64_t a, uint64_t n, const char *what) { plat_mutex_lock(&frz_lock); + thaw_wait_locked(a, n); thaw_range_locked(a, n, 3); int ok = !n || plat_mem_commit(a, n); plat_mutex_unlock(&frz_lock); @@ -918,6 +1051,7 @@ static void commit(uint64_t a, uint64_t n, const char *what) { static void decommit(uint64_t a, uint64_t n, const char *what) { plat_mutex_lock(&frz_lock); + thaw_wait_locked(a, n); thaw_range_locked(a, n, 3); int ok = !n || plat_mem_decommit(a, n); plat_mutex_unlock(&frz_lock); @@ -926,6 +1060,7 @@ static void decommit(uint64_t a, uint64_t n, const char *what) { static void protect(uint64_t a, uint64_t n, int prot, const char *what) { plat_mutex_lock(&frz_lock); + thaw_wait_locked(a, n); thaw_range_locked(a, n, 3); int ok = !n || plat_mem_protect(a, n, prot); plat_mutex_unlock(&frz_lock); @@ -978,6 +1113,8 @@ void mem_init(void) { lz_want = 1; lz_chunk = cfg_bool("NX2NAVI_WB_LAZY_CHUNK"); } + const char *fa = cfg_get("NX2NAVI_FREEZE_ASYNC"); // thaws outside frz_lock (FRZ_THAWING); unset: on + frz_async = !fa || !*fa || atoi(fa) != 0; const char *pool = cfg_get("NX2NAVI_APP_POOL"); L = (RtGuestLayout){ .aslr = {low, RT_ADDRESS_LIMIT - low},