Thaw a frozen region's pages outside the freeze lock, from the runs its freeze kept, so a thaw slowed by the driver's pinning no longer holds the replay and the game's other threads.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 15:35:25 -05:00
parent fa9dcbca5c
commit ebb37689cf
2 changed files with 153 additions and 16 deletions

View File

@@ -46,7 +46,7 @@ static const char *const known[] = {
"COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT",
"DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLSS_IDLE_FREE", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
"DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES",
"FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_BUDGET_US", "FREE_CORES",
"FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES",
"FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR",
"HDR_GAME_NITS", "HDR_LIVE", "HEAP_LAZY", "HDR_PEAK_NITS", "HDR_UI_NITS",
// retired: the HDR look's settings of earlier versions, now fixed (vk_hdr.c, shaders/hdr_tonemap.frag); read by

View File

@@ -93,6 +93,21 @@ static volatile int64_t frz_hot_holds; // quick thaws tha
static volatile int32_t nfrz;
static PlatMutex frz_lock = PLAT_MUTEX_INIT;
static volatile int64_t frz_count[4]; // freezes; thaws by a write fault, by mem_thaw, by a mapping change
// Thaws outside frz_lock (NX2NAVI_FREEZE_ASYNC, default 1; 0: every thaw under the lock, as before). While the game
// streams an area it creates and finalizes up to 1,500 memory pools a second, the driver pins and unpins the imported
// ones, and the VirtualQuery and VirtualProtect calls of a thaw then take 10-20 ms each. Under frz_lock that held the
// replay (mem_lazy_skip, the freeze pass) and every other thread that faulted or thawed, a 25-45 ms frame. So a thaw by
// a write fault or by mem_thaw marks its regions FRZ_THAWING under the lock, makes their pages read-write without it and
// ends them under it again. A region being thawed is no longer frozen for anyone who asks (a freeze, an arm, a probe
// leave it alone); a fault in it waits for the thaw to end and runs again; a mapping change waits for it first
// (thaw_wait_locked), so a late VirtualProtect never undoes the change; mem_thaw makes its pages read-write itself.
// freeze_locked keeps the runs it made read-only (frz_runs), so a thaw needs no VirtualQuery.
#define FRZ_THAWING 2
static int frz_async = 1; // the setting, read in mem_init
typedef struct { uint8_t n; uint16_t run[16][2]; } FrzRuns; // pages of the region, [first, end)
static FrzRuns *frz_runs[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock (written at a freeze): NULL or n 0 = not kept
static volatile int64_t frz_async_n; // thaws done outside the lock, since the start
static volatile int64_t frz_protect_fails; // VirtualProtect failures of those thaws, since the start
// Probed regions (NX2NAVI_WB_PROBE=1, a diagnosis). The back end copies what the GPU wrote into a mirrored pool back
// into guest memory after every submission, in case the CPU or a host path reads it; whether anything does is what
@@ -118,7 +133,7 @@ static int watched(uint64_t addr, uint64_t size) {
// frz_lock held exclusively: the pages of a probed region get back what they had, read-only in a frozen region.
static void unprobe_locked(uint64_t r) {
if (!prb[r]) return;
int prot = frz[r] ? PLAT_PROT_READ : PLAT_PROT_RW;
int prot = frz[r] == 1 ? PLAT_PROT_READ : PLAT_PROT_RW;
uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, b;
for (; a < end; a = b) // every inaccessible run is the probe's: mem_probe took only regions without one
if (plat_mem_state(a, end, &b) == PLAT_PROT_NONE && !plat_mem_protect(a, b - a, prot))
@@ -578,6 +593,13 @@ void mem_lazy_note(char *out, size_t n) {
// One log line since the last one; skipped: the bytes the back end left in the mirror since then.
void mem_lazy_report(uint64_t frame, uint64_t skipped) {
static int64_t async_said = -1;
int64_t an = __atomic_load_n(&frz_async_n, __ATOMIC_RELAXED), af = __atomic_load_n(&frz_protect_fails, __ATOMIC_RELAXED);
if ((rt_trace & RT_TRACE_PERF) && an != async_said) {
async_said = an;
rt_log("mem: thaws outside the freeze lock so far %lld (NX2NAVI_FREEZE_ASYNC), page protections that failed there "
"%lld", (long long)an, (long long)af);
}
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return;
plat_mutex_lock(&frz_lock);
uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes,
@@ -628,19 +650,56 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) {
(unsigned long long)calls, (unsigned long long)small);
}
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
static void thaw_locked(uint64_t r, int why) {
// Makes the pages region r's freeze made read-only read-write again: the runs freeze_locked kept, else every read-only
// run (freeze_locked left reserved holes, heap pages given back, as they were; every committed page of a frozen region
// was read-write at the freeze, and mapping changes thaw or wait before they run, so these are all ours). strict: under
// frz_lock, a failure ends the process; outside it (a thaw in flight, FRZ_THAWING) the pages are what the freeze left,
// and a failure is only counted.
static void thaw_pages(uint64_t r, int strict) {
uint64_t base = r << FRZ_SHIFT;
const FrzRuns *k = frz_runs[r];
if (k && k->n) {
for (int i = 0; i < k->n; i++) {
uint64_t a = base + ((uint64_t)k->run[i][0] << 12), n = (uint64_t)(k->run[i][1] - k->run[i][0]) << 12;
if (!plat_mem_protect(a, n, PLAT_PROT_RW)) {
if (strict)
rt_fatal("memory: thawing %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)n);
plat_inc64(&frz_protect_fails);
}
}
return;
}
uint64_t a = base, end = a + FRZ_SIZE, b;
for (; a < end; a = b)
if (plat_mem_state(a, end, &b) == PLAT_PROT_READ && !plat_mem_protect(a, b - a, PLAT_PROT_RW)) {
if (strict) rt_fatal("memory: thawing %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)(b - a));
plat_inc64(&frz_protect_fails);
}
}
// frz_lock held exclusively: what a thaw does before the pages (the hold, the lazy write-back's ranges, a probe).
// Returns 1 when the region is frozen.
static int thaw_begin_locked(uint64_t r, int why) {
if (why == 2) thaw_ms[r] = (uint32_t)plat_now_ms() | 1;
if (lzr[r]) lazy_drop_region_locked(r, why + 1); // not a fault on an armed page: no hold
if (prb[r]) unprobe_locked(r);
if (!frz[r]) return;
// The read-only runs only: freeze_locked left reserved holes (heap pages given back) as they were. Every committed
// page of a frozen region was read-write at the freeze, and mapping changes thaw before they run, so these are all
// ours.
uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, b;
for (; a < end; a = b)
if (plat_mem_state(a, end, &b) == PLAT_PROT_READ && !plat_mem_protect(a, b - a, PLAT_PROT_RW))
rt_fatal("memory: thawing %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)(b - a));
return frz[r] == 1;
}
static void thaw_end_locked(uint64_t r, int why);
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
// A region another thread is thawing (FRZ_THAWING) gets its pages made read-write here too, and is ended here.
static void thaw_locked(uint64_t r, int why) {
if (!thaw_begin_locked(r, why) && frz[r] != FRZ_THAWING) return;
thaw_pages(r, 1);
__atomic_store_n(&frz[r], FRZ_THAWING, __ATOMIC_RELEASE);
thaw_end_locked(r, why);
}
// frz_lock held exclusively: the end of a thaw. Nothing when another thread ended it meanwhile.
static void thaw_end_locked(uint64_t r, int why) {
if (frz[r] != FRZ_THAWING) return;
__atomic_store_n(&frz[r], 0, __ATOMIC_RELEASE);
plat_dec32(&nfrz);
plat_inc64(&frz_count[why]);
@@ -664,6 +723,20 @@ static void thaw_range_locked(uint64_t a, uint64_t n, int why) {
for (uint64_t r = r0; r <= r1; r++) thaw_locked(r, why);
}
// frz_lock held exclusively: returns once no region of [a, a + n) is being thawed outside the lock (FRZ_THAWING),
// releasing the lock while it waits. A mapping change runs after this: the thaw's VirtualProtect then cannot land on
// the pages after the change and undo it.
static void thaw_wait_locked(uint64_t a, uint64_t n) {
if (!n || a >= RT_ADDRESS_LIMIT) return;
uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT;
for (uint64_t r = r0; r <= r1; r++)
while (__atomic_load_n(&frz[r], __ATOMIC_ACQUIRE) == FRZ_THAWING) {
plat_mutex_unlock(&frz_lock);
plat_wait_ns(20000);
plat_mutex_lock(&frz_lock);
}
}
// Lazy heap (NX2NAVI_HEAP_LAZY, Windows only; off by default). Windows charges every committed page to the system's
// commit at once, and SetHeapSize asks for the whole application heap up front: in Castle Town after Hyrule Field the
// heap held 6.8 GiB of commit of which 3.4 GiB had ever been resident, with 0.45 GB in the page file. With it on,
@@ -738,8 +811,39 @@ static int heap_lazy_fault(uint64_t a) {
void mem_thaw(uint64_t a, uint64_t n) {
plat_mutex_lock(&frz_lock); // also when nothing is frozen: the hold keeps a read's buffer writable
if (heap_lazy) heap_lazy_commit_locked(a, n); // the kernel's write would fail on a reserved page as well
thaw_range_locked(a, n, 2);
if (heap_lazy || !frz_async || !n || a >= RT_ADDRESS_LIMIT) {
if (heap_lazy) {
thaw_wait_locked(a, n); // it commits pages: a mapping change
heap_lazy_commit_locked(a, n); // the kernel's write would fail on a reserved page as well
}
thaw_range_locked(a, n, 2);
plat_mutex_unlock(&frz_lock);
return;
}
// The regions frozen now are marked FRZ_THAWING and made read-write without the lock (see FRZ_THAWING), up to
// MANY at a time; those another thread is thawing are made read-write here too, as the caller's kernel access needs
// every page of them now, and that thread ends them.
enum { MANY = 64 };
uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT;
for (uint64_t r = r0; r <= r1;) {
uint64_t mine[MANY], theirs[MANY];
int nm = 0, nt = 0;
for (; r <= r1 && nm < MANY && nt < MANY; r++) {
if (thaw_begin_locked(r, 2)) {
__atomic_store_n(&frz[r], FRZ_THAWING, __ATOMIC_RELEASE);
mine[nm++] = r;
} else if (frz[r] == FRZ_THAWING) {
theirs[nt++] = r;
}
}
if (!nm && !nt) continue;
plat_mutex_unlock(&frz_lock);
for (int i = 0; i < nm; i++) thaw_pages(mine[i], 0);
for (int i = 0; i < nt; i++) thaw_pages(theirs[i], 0);
plat_add64(&frz_async_n, nm);
plat_mutex_lock(&frz_lock);
for (int i = 0; i < nm; i++) thaw_end_locked(mine[i], 2);
}
plat_mutex_unlock(&frz_lock);
}
@@ -760,6 +864,13 @@ static int freeze_locked(uint64_t r) {
while (i--) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW);
return 0;
}
// the runs, for a thaw without VirtualQuery (thaw_pages); kept per region from its first freeze on
if (n && !frz_runs[r]) frz_runs[r] = calloc(1, sizeof *frz_runs[r]);
if (frz_runs[r]) {
frz_runs[r]->n = (uint8_t)n;
for (int i = 0; i < n; i++)
frz_runs[r]->run[i][0] = (uint16_t)((run[i][0] - a) >> 12), frz_runs[r]->run[i][1] = (uint16_t)((run[i][1] - a) >> 12);
}
return n > 0;
}
@@ -898,9 +1009,30 @@ void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws
// The platform's write-fault hook (plat_mem_fault_hook): 1 retries the write.
static int thaw_fault(uint64_t a) {
if (a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return 0;
uint64_t r = a >> FRZ_SHIFT;
plat_mutex_lock(&frz_lock);
int was = frz[a >> FRZ_SHIFT];
thaw_locked(a >> FRZ_SHIFT, 1);
int was = frz[r];
if (was == FRZ_THAWING) { // another thread's thaw: the write runs again once it ended
while (frz[r] == FRZ_THAWING) {
plat_mutex_unlock(&frz_lock);
plat_wait_ns(20000);
plat_mutex_lock(&frz_lock);
}
plat_mutex_unlock(&frz_lock);
return 1;
}
if (frz_async && was == 1) { // its pages outside the lock (FRZ_THAWING)
thaw_begin_locked(r, 1);
__atomic_store_n(&frz[r], FRZ_THAWING, __ATOMIC_RELEASE);
plat_mutex_unlock(&frz_lock);
thaw_pages(r, 0);
plat_inc64(&frz_async_n);
plat_mutex_lock(&frz_lock);
thaw_end_locked(r, 1);
plat_mutex_unlock(&frz_lock);
return 1;
}
thaw_locked(r, 1);
plat_mutex_unlock(&frz_lock);
if (was) return 1;
uint64_t end;
@@ -910,6 +1042,7 @@ static int thaw_fault(uint64_t a) {
// Mapping changes thaw the regions they touch and run under frz_lock, so no freeze sees them half done.
static void commit(uint64_t a, uint64_t n, const char *what) {
plat_mutex_lock(&frz_lock);
thaw_wait_locked(a, n);
thaw_range_locked(a, n, 3);
int ok = !n || plat_mem_commit(a, n);
plat_mutex_unlock(&frz_lock);
@@ -918,6 +1051,7 @@ static void commit(uint64_t a, uint64_t n, const char *what) {
static void decommit(uint64_t a, uint64_t n, const char *what) {
plat_mutex_lock(&frz_lock);
thaw_wait_locked(a, n);
thaw_range_locked(a, n, 3);
int ok = !n || plat_mem_decommit(a, n);
plat_mutex_unlock(&frz_lock);
@@ -926,6 +1060,7 @@ static void decommit(uint64_t a, uint64_t n, const char *what) {
static void protect(uint64_t a, uint64_t n, int prot, const char *what) {
plat_mutex_lock(&frz_lock);
thaw_wait_locked(a, n);
thaw_range_locked(a, n, 3);
int ok = !n || plat_mem_protect(a, n, prot);
plat_mutex_unlock(&frz_lock);
@@ -978,6 +1113,8 @@ void mem_init(void) {
lz_want = 1;
lz_chunk = cfg_bool("NX2NAVI_WB_LAZY_CHUNK");
}
const char *fa = cfg_get("NX2NAVI_FREEZE_ASYNC"); // thaws outside frz_lock (FRZ_THAWING); unset: on
frz_async = !fa || !*fa || atoi(fa) != 0;
const char *pool = cfg_get("NX2NAVI_APP_POOL");
L = (RtGuestLayout){
.aslr = {low, RT_ADDRESS_LIMIT - low},