Mark a region for the lazy write-back before its pages become inaccessible, so that a read during an arm is not passed to the crash handler.

mem_lazy_arm made a range's pages inaccessible (lazy_protect_locked: the protect, then the back end's clean test) and set the region's lzr mark only after the segment was in the list. lazy_fault returns 0 at once for an unmarked region, without taking frz_lock, so a CPU read in that window found the page inaccessible in access_fault and went to the crash handler as an access violation. A write survived only because thaw_fault waits for frz_lock. Arms come from the completion thread (vk_memory.c vk_gpu_written_back) when a submission's fence completes, which is when the game's own fence wait returns and it reads the results; the window lasts as long as the protect and the clean test, which can take tens of milliseconds while the driver pins memory.

The mark is now set before the protect, after the cheap refusals, and put back when the protect or the clean test refuses the arm. A fault from the protect on finds the mark, waits for frz_lock and then finds the segment, or the pages given back. Every store to lzr is made under frz_lock, which the arm holds, so putting it back is exact.

Not reproduced in the game before the change. The new tests\lazy_fetch_check.py reproduces it with the real Windows fault handler: a thread reads a page while it is armed and given back 20,000 times (arm_race: an access violation in 5 of 6 runs before the change), and once with a clean test of 100 ms (arm_race_slow: an access violation in every run). Both pass in every run since. wb_lazy_check, heap_lazy_copy_check, freeze_hold_check and mem_growth_check passed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/building.md

@@ -688,6 +688,12 @@ A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice.
 `vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and
 its reason by platform, by GPU driver and by what the player set. No game files, no GPU.
 
+A change to the lazy write-back's arms, faults and releases (`runtime/memory.c` "Lazy write-back") passes
+`python tests\lazy_fetch_check.py`, which builds memory.c with the Windows fault handler and a fetch that only waits,
+and runs each scenario as its own process, so an access violation fails it: a thread that reads a range over and over
+while it is armed and given back again never reaches the crash handler, also when the arm's clean test takes 100 ms.
+Windows only; no game files, no GPU.
+
 A change to when the freezing of idle guest regions waits (`runtime/freeze_hold.h`, `runtime/vk_memory.c`
 `freeze_pass`) passes `python tests\freeze_hold_check.py`, which builds the header alone: the backoff after a slow
 freeze doubles while slow ones come within the quiet time, fast freezes between them included, stops at its cap and

runtime/memory.c

@@ -463,8 +463,21 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
                 }
         }
         int no = frz[r] ? LZ_NO_FROZEN : prb[r] ? LZ_NO_PAGES : lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0 ? LZ_NO_HOT :
-                 thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS ? LZ_NO_THAW :
-                 !lazy_protect_locked(x, e, ctx) ? LZ_NO_PAGES : -1;
+                 thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS ? LZ_NO_THAW : -1;
+        if (no < 0) {
+            // The region is marked before its pages become inaccessible. A fault on them from the protect on then finds
+            // the mark, waits in lazy_fault for frz_lock, held here, and finds the segment or its pages given back.
+            // Marked only after the push, a read in between had lazy_fault return 0 at once, and access_fault passed
+            // the read of an inaccessible page to the crash handler. had: a segment of the region, or an earlier turn
+            // of this loop, marked it already; when the arm is refused the mark goes back to what it was (every store
+            // to lzr is made under frz_lock).
+            int had = lzr[r];
+            if (!had) __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE);
+            if (!lazy_protect_locked(x, e, ctx)) {
+                no = LZ_NO_PAGES;
+                if (!had) __atomic_store_n(&lzr[r], 0, __ATOMIC_RELEASE);
+            }
+        }
         if (no >= 0) {
             lz_no[no] += e - x;
             continue;
@@ -482,7 +495,6 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t
             }
         lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale});
         plat_mutex_unlock(&lz_lock);
-        __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE);
         armed += e - x;
     }
     plat_mutex_unlock(&frz_lock);

tests/lazy_fetch_check.c

@@ -0,0 +1,126 @@
+// runtime/memory.c's lazy write-back (memory.c "Lazy write-back"), built with the Windows platform layer and its real
+// fault handler, the rest of the runtime stubbed. The back end's fetch is a stand-in that waits (the GPU copy and the
+// wait for it) and then writes a marker over the first 64 bytes of the copy, opening the page for it, as the GPU's
+// copy writes past the pages' protection; the back end's clean test may wait too. Built and run by
+// tests/lazy_fetch_check.py, one scenario per process (the argument), so an access violation passed to the crash
+// handler shows as the exit code. Prints each failed check and exits 1 on any.
+#include "memory.c"
+
+#include <stdarg.h>
+
+static int fails;
+#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
+
+// ---- the runtime memory.c and the platform layer reach
+unsigned rt_trace;
+uint64_t rt_guest_start_ns;
+const unsigned long long rt_app_pool_bytes = 0xCD500000ull;
+const RtLayoutEntry rt_layout[] = {{"main", MOD_MAIN_BASE, MOD_MAIN_SIZE}};
+const int rt_layout_count = 1;
+const char *cfg_get(const char *name) { (void)name; return NULL; }
+const char *cfg_value(const char *name) { (void)name; return "0"; }
+int cfg_bool(const char *name) {
+    if (!strcmp(name, "NX2NAVI_WB_LAZY")) return 1;
+    if (!strcmp(name, "NX2NAVI_WB_LAZY_CHUNK")) return getenv("CHECK_CHUNK") != NULL;
+    return 0;
+}
+void cfg_init(void) {}
+int cfg_setenv(const char *name, const char *value) { (void)name, (void)value; return 0; }
+int rt_knob(const char *name, const char *def) { (void)name; return strcmp(def, "0") != 0; }
+int ldr_count(void) { return 0; }
+const RtLoaded *ldr_module(int i) { (void)i; return NULL; }
+uint64_t perf_mark_begin(void) { return 0; }
+void perf_mark_end(int kind, uint64_t t0, uint64_t n) { (void)kind, (void)t0, (void)n; }
+void perf_size(PerfSizes *s, const char *name, uint64_t count, uint64_t limit, uint64_t bytes) {
+    (void)s, (void)name, (void)count, (void)limit, (void)bytes;
+}
+uint64_t res_heap_extra_bytes(void) { return 0; }
+static char last_log[4096];
+void rt_log(const char *fmt, ...) {
+    va_list ap;
+    va_start(ap, fmt);
+    vsnprintf(last_log, sizeof last_log, fmt, ap);
+    va_end(ap);
+    printf("  log: %s\n", last_log);
+}
+void rt_logf(const char *fmt, ...) { (void)fmt; }
+void rt_fatal(const char *fmt, ...) {
+    va_list ap;
+    va_start(ap, fmt);
+    printf("FATAL: ");
+    vprintf(fmt, ap);
+    printf("\n");
+    va_end(ap);
+    fflush(stdout);
+    _Exit(3);
+}
+
+// ---- the back end's stand-ins
+static uint64_t heap;
+static volatile uint64_t fetch_ns = 300000000ull, clean_ns;
+static volatile int fetch_calls;
+static volatile uint8_t marker = 0xA0;
+static void fake_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) {
+    (void)h0, (void)size;
+    __atomic_add_fetch(&fetch_calls, 1, __ATOMIC_SEQ_CST);
+    plat_wait_ns(fetch_ns);
+    uint64_t a = h1 + off;   // the GPU's copy writes past the protection; here the page is opened for it
+    plat_mem_protect(a & ~(PAGE - 1), PAGE, PLAT_PROT_RW);
+    for (int j = 0; j < 64; j++) ((volatile uint8_t *)(uintptr_t)a)[j] = marker;
+    plat_mem_protect(a & ~(PAGE - 1), PAGE, PLAT_PROT_NONE);
+}
+static int fake_clean(uint64_t lo, uint64_t hi, void *ctx) {
+    (void)lo, (void)hi, (void)ctx;
+    if (clean_ns) plat_wait_ns(clean_ns);   // GetWriteWatch while the driver pins memory
+    return 1;
+}
+
+// the range [a, a + n) of the heap, as written back from buffer h0 into the heap's own (h1 = heap, offset a - heap)
+static uint64_t arm_buf(uint64_t a, uint64_t n, uint64_t h0) { return mem_lazy_arm(a, n, h0, heap, a - heap, NULL); }
+
+static volatile int spin_stop;
+static volatile uint64_t spin_reads;
+static void spinner(void *arg) {
+    volatile uint8_t *p = arg;
+    while (!__atomic_load_n(&spin_stop, __ATOMIC_SEQ_CST)) (void)*p, spin_reads++;
+}
+
+int main(int argc, char **argv) {
+    setvbuf(stdout, NULL, _IONBF, 0);
+    const char *what = argc > 1 ? argv[1] : "";
+    mem_init();
+    if (mem_set_heap_size(256ull << 20, &heap) != R_OK || !heap) return printf("SetHeapSize failed\n"), 2;
+    if (!mem_lazy_enable(fake_fetch, fake_clean)) return printf("the lazy write-back is off\n"), 2;
+    memset((void *)(uintptr_t)heap, 0x11, 256ull << 20);
+    const uint64_t K64 = 0x10000;
+
+    if (!strcmp(what, "arm_race") || !strcmp(what, "arm_race_slow")) {
+        // A thread reads a page over and over while the completion thread's arm (vk_memory.c vk_gpu_written_back)
+        // makes its range inaccessible and a release gives it back: no read may reach the crash handler. The slow one
+        // has the clean test take 100 ms, as GetWriteWatch can while the driver pins memory, with the pages
+        // inaccessible all along.
+        uint64_t t = heap + (4ull << 20), r = t >> FRZ_SHIFT;
+        void *th = plat_thread_new(spinner, (void *)(uintptr_t)(t + 0x100), "spinner");
+        plat_wait_ns(20000000);
+        int slow = !strcmp(what, "arm_race_slow");
+        if (slow) clean_ns = 100000000;
+        uint64_t armed = 0;
+        for (int i = 0; i < (slow ? 1 : 20000); i++) {
+            plat_mutex_lock(&frz_lock);   // the hold a touch leaves, so that every turn arms again
+            lz_hot[r] = 0;
+            plat_mutex_unlock(&frz_lock);
+            armed += arm_buf(t, K64, 1) != 0;
+            mem_lazy_release(t, K64);
+        }
+        spin_stop = 1;
+        plat_thread_join(th);
+        CHECK(armed, "nothing was armed");
+        printf("  %llu arms, %llu reads\n", (unsigned long long)armed, (unsigned long long)spin_reads);
+    } else {
+        printf("unknown scenario %s\n", what);
+        return 2;
+    }
+    if (fails) return printf("%s: %d checks failed\n", what, fails), 1;
+    printf("%s: all checks passed\n", what);
+    return 0;
+}

tests/lazy_fetch_check.py

@@ -0,0 +1,90 @@
+"""Build and run tests/lazy_fetch_check.c: runtime/memory.c's lazy write-back (memory.c "Lazy write-back") with the
+Windows platform layer and its real fault handler, the rest of the runtime stubbed and the back end's fetch a stand-in
+that only waits. Each scenario runs as its own process, so an access violation passed to the crash handler shows as
+its exit code; any exit code other than 0, or a scenario that does not end within 120 s, fails.
+
+Windows only (the fault handler is): no game files, no GPU. The module layout the runtime includes is a stand-in
+written to the test's build folder.
+
+Usage: lazy_fetch_check.py [scenario ...]
+"""
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
+# name: (scenario argument, extra environment)
+SCENARIOS = {
+    'arm_race': ('arm_race', {}),
+    'arm_race_slow': ('arm_race_slow', {}),
+}
+
+
+def compiler():
+    if os.environ.get('CC'):
+        return os.environ['CC']
+    if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
+        return str(MSYS_BIN / 'clang.exe')
+    return shutil.which('clang') or 'clang'
+
+
+LAYOUT = '''// A stand-in for the generated layout.h (tests/lazy_fetch_check.py).
+#pragma once
+#define MOD_MAIN_BASE 0x7000000000ull
+#define MOD_MAIN_SIZE 0x4ea0000ull
+#define MOD_COUNT 3
+'''
+
+
+def main():
+    if os.name != 'nt':
+        print('lazy_fetch_check: the fault handler it runs is Windows only; skipped')
+        return 0
+    names = sys.argv[1:] or list(SCENARIOS)
+    unknown = [n for n in names if n not in SCENARIOS]
+    if unknown:
+        print('lazy_fetch_check: unknown scenario %s; known: %s' % (', '.join(unknown), ', '.join(SCENARIOS)))
+        return 2
+    work = ROOT / 'build' / 'test' / 'lazy_fetch'
+    work.mkdir(parents=True, exist_ok=True)
+    (work / 'layout.h').write_text(LAYOUT)
+    exe = work / 'lazy_fetch_check.exe'
+    env = dict(os.environ)
+    if MSYS_BIN.exists():
+        env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
+    rt = ROOT / 'runtime'
+    plat = [rt / 'platform' / 'win32.c', rt / 'platform' / 'common.c']
+    libs = ['-lbcrypt', '-lws2_32', '-lpsapi', '-lwinmm', '-ldbghelp', '-lshell32', '-lole32', '-luser32', '-ladvapi32']
+    subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-fno-strict-aliasing',
+                           '-march=x86-64-v3', '-I', str(rt), '-I', str(work),
+                           str(ROOT / 'tests' / 'lazy_fetch_check.c'), *map(str, plat), '-o', str(exe), '-lm',
+                           *libs], env=env)
+    bad = 0
+    for name in names:
+        arg, extra = SCENARIOS[name]
+        try:
+            p = subprocess.run([str(exe), arg], env={**env, **extra}, capture_output=True, text=True, timeout=120)
+        except subprocess.TimeoutExpired:
+            bad += 1
+            print('lazy_fetch_check: %s: FAIL: did not end within 120 s' % name)
+            continue
+        code = p.returncode & 0xFFFFFFFF
+        if code:
+            bad += 1
+            print('lazy_fetch_check: %s: FAIL: exit code %#x%s' % (
+                name, code, ' (an access violation)' if code == 0xC0000005 else ''))
+            sys.stdout.write(p.stdout)
+        else:
+            print('lazy_fetch_check: %s: passed' % name)
+    if bad:
+        print('lazy_fetch_check: %d of %d scenarios failed' % (bad, len(names)))
+        return 1
+    print('lazy_fetch_check: all %d scenarios passed' % len(names))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())