diff --git a/docs/building.md b/docs/building.md index 82174da6..f9795515 100644 --- a/docs/building.md +++ b/docs/building.md @@ -688,6 +688,12 @@ A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice. `vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and its reason by platform, by GPU driver and by what the player set. No game files, no GPU. +A change to the lazy write-back's arms, faults and releases (`runtime/memory.c` "Lazy write-back") passes +`python tests\lazy_fetch_check.py`, which builds memory.c with the Windows fault handler and a fetch that only waits, +and runs each scenario as its own process, so an access violation fails it: a thread that reads a range over and over +while it is armed and given back again never reaches the crash handler, also when the arm's clean test takes 100 ms. +Windows only; no game files, no GPU. + A change to when the freezing of idle guest regions waits (`runtime/freeze_hold.h`, `runtime/vk_memory.c` `freeze_pass`) passes `python tests\freeze_hold_check.py`, which builds the header alone: the backoff after a slow freeze doubles while slow ones come within the quiet time, fast freezes between them included, stops at its cap and diff --git a/runtime/memory.c b/runtime/memory.c index 2be0e113..ac4f73a3 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -463,8 +463,21 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t } } int no = frz[r] ? LZ_NO_FROZEN : prb[r] ? LZ_NO_PAGES : lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0 ? LZ_NO_HOT : - thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS ? LZ_NO_THAW : - !lazy_protect_locked(x, e, ctx) ? LZ_NO_PAGES : -1; + thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS ? LZ_NO_THAW : -1; + if (no < 0) { + // The region is marked before its pages become inaccessible. A fault on them from the protect on then finds + // the mark, waits in lazy_fault for frz_lock, held here, and finds the segment or its pages given back. + // Marked only after the push, a read in between had lazy_fault return 0 at once, and access_fault passed + // the read of an inaccessible page to the crash handler. had: a segment of the region, or an earlier turn + // of this loop, marked it already; when the arm is refused the mark goes back to what it was (every store + // to lzr is made under frz_lock). + int had = lzr[r]; + if (!had) __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE); + if (!lazy_protect_locked(x, e, ctx)) { + no = LZ_NO_PAGES; + if (!had) __atomic_store_n(&lzr[r], 0, __ATOMIC_RELEASE); + } + } if (no >= 0) { lz_no[no] += e - x; continue; @@ -482,7 +495,6 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t } lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale}); plat_mutex_unlock(&lz_lock); - __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE); armed += e - x; } plat_mutex_unlock(&frz_lock); diff --git a/tests/lazy_fetch_check.c b/tests/lazy_fetch_check.c new file mode 100644 index 00000000..4e2f7b7e --- /dev/null +++ b/tests/lazy_fetch_check.c @@ -0,0 +1,126 @@ +// runtime/memory.c's lazy write-back (memory.c "Lazy write-back"), built with the Windows platform layer and its real +// fault handler, the rest of the runtime stubbed. The back end's fetch is a stand-in that waits (the GPU copy and the +// wait for it) and then writes a marker over the first 64 bytes of the copy, opening the page for it, as the GPU's +// copy writes past the pages' protection; the back end's clean test may wait too. Built and run by +// tests/lazy_fetch_check.py, one scenario per process (the argument), so an access violation passed to the crash +// handler shows as the exit code. Prints each failed check and exits 1 on any. +#include "memory.c" + +#include + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +// ---- the runtime memory.c and the platform layer reach +unsigned rt_trace; +uint64_t rt_guest_start_ns; +const unsigned long long rt_app_pool_bytes = 0xCD500000ull; +const RtLayoutEntry rt_layout[] = {{"main", MOD_MAIN_BASE, MOD_MAIN_SIZE}}; +const int rt_layout_count = 1; +const char *cfg_get(const char *name) { (void)name; return NULL; } +const char *cfg_value(const char *name) { (void)name; return "0"; } +int cfg_bool(const char *name) { + if (!strcmp(name, "NX2NAVI_WB_LAZY")) return 1; + if (!strcmp(name, "NX2NAVI_WB_LAZY_CHUNK")) return getenv("CHECK_CHUNK") != NULL; + return 0; +} +void cfg_init(void) {} +int cfg_setenv(const char *name, const char *value) { (void)name, (void)value; return 0; } +int rt_knob(const char *name, const char *def) { (void)name; return strcmp(def, "0") != 0; } +int ldr_count(void) { return 0; } +const RtLoaded *ldr_module(int i) { (void)i; return NULL; } +uint64_t perf_mark_begin(void) { return 0; } +void perf_mark_end(int kind, uint64_t t0, uint64_t n) { (void)kind, (void)t0, (void)n; } +void perf_size(PerfSizes *s, const char *name, uint64_t count, uint64_t limit, uint64_t bytes) { + (void)s, (void)name, (void)count, (void)limit, (void)bytes; +} +uint64_t res_heap_extra_bytes(void) { return 0; } +static char last_log[4096]; +void rt_log(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + vsnprintf(last_log, sizeof last_log, fmt, ap); + va_end(ap); + printf(" log: %s\n", last_log); +} +void rt_logf(const char *fmt, ...) { (void)fmt; } +void rt_fatal(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + printf("FATAL: "); + vprintf(fmt, ap); + printf("\n"); + va_end(ap); + fflush(stdout); + _Exit(3); +} + +// ---- the back end's stand-ins +static uint64_t heap; +static volatile uint64_t fetch_ns = 300000000ull, clean_ns; +static volatile int fetch_calls; +static volatile uint8_t marker = 0xA0; +static void fake_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) { + (void)h0, (void)size; + __atomic_add_fetch(&fetch_calls, 1, __ATOMIC_SEQ_CST); + plat_wait_ns(fetch_ns); + uint64_t a = h1 + off; // the GPU's copy writes past the protection; here the page is opened for it + plat_mem_protect(a & ~(PAGE - 1), PAGE, PLAT_PROT_RW); + for (int j = 0; j < 64; j++) ((volatile uint8_t *)(uintptr_t)a)[j] = marker; + plat_mem_protect(a & ~(PAGE - 1), PAGE, PLAT_PROT_NONE); +} +static int fake_clean(uint64_t lo, uint64_t hi, void *ctx) { + (void)lo, (void)hi, (void)ctx; + if (clean_ns) plat_wait_ns(clean_ns); // GetWriteWatch while the driver pins memory + return 1; +} + +// the range [a, a + n) of the heap, as written back from buffer h0 into the heap's own (h1 = heap, offset a - heap) +static uint64_t arm_buf(uint64_t a, uint64_t n, uint64_t h0) { return mem_lazy_arm(a, n, h0, heap, a - heap, NULL); } + +static volatile int spin_stop; +static volatile uint64_t spin_reads; +static void spinner(void *arg) { + volatile uint8_t *p = arg; + while (!__atomic_load_n(&spin_stop, __ATOMIC_SEQ_CST)) (void)*p, spin_reads++; +} + +int main(int argc, char **argv) { + setvbuf(stdout, NULL, _IONBF, 0); + const char *what = argc > 1 ? argv[1] : ""; + mem_init(); + if (mem_set_heap_size(256ull << 20, &heap) != R_OK || !heap) return printf("SetHeapSize failed\n"), 2; + if (!mem_lazy_enable(fake_fetch, fake_clean)) return printf("the lazy write-back is off\n"), 2; + memset((void *)(uintptr_t)heap, 0x11, 256ull << 20); + const uint64_t K64 = 0x10000; + + if (!strcmp(what, "arm_race") || !strcmp(what, "arm_race_slow")) { + // A thread reads a page over and over while the completion thread's arm (vk_memory.c vk_gpu_written_back) + // makes its range inaccessible and a release gives it back: no read may reach the crash handler. The slow one + // has the clean test take 100 ms, as GetWriteWatch can while the driver pins memory, with the pages + // inaccessible all along. + uint64_t t = heap + (4ull << 20), r = t >> FRZ_SHIFT; + void *th = plat_thread_new(spinner, (void *)(uintptr_t)(t + 0x100), "spinner"); + plat_wait_ns(20000000); + int slow = !strcmp(what, "arm_race_slow"); + if (slow) clean_ns = 100000000; + uint64_t armed = 0; + for (int i = 0; i < (slow ? 1 : 20000); i++) { + plat_mutex_lock(&frz_lock); // the hold a touch leaves, so that every turn arms again + lz_hot[r] = 0; + plat_mutex_unlock(&frz_lock); + armed += arm_buf(t, K64, 1) != 0; + mem_lazy_release(t, K64); + } + spin_stop = 1; + plat_thread_join(th); + CHECK(armed, "nothing was armed"); + printf(" %llu arms, %llu reads\n", (unsigned long long)armed, (unsigned long long)spin_reads); + } else { + printf("unknown scenario %s\n", what); + return 2; + } + if (fails) return printf("%s: %d checks failed\n", what, fails), 1; + printf("%s: all checks passed\n", what); + return 0; +} diff --git a/tests/lazy_fetch_check.py b/tests/lazy_fetch_check.py new file mode 100644 index 00000000..cab71912 --- /dev/null +++ b/tests/lazy_fetch_check.py @@ -0,0 +1,90 @@ +"""Build and run tests/lazy_fetch_check.c: runtime/memory.c's lazy write-back (memory.c "Lazy write-back") with the +Windows platform layer and its real fault handler, the rest of the runtime stubbed and the back end's fetch a stand-in +that only waits. Each scenario runs as its own process, so an access violation passed to the crash handler shows as +its exit code; any exit code other than 0, or a scenario that does not end within 120 s, fails. + +Windows only (the fault handler is): no game files, no GPU. The module layout the runtime includes is a stand-in +written to the test's build folder. + +Usage: lazy_fetch_check.py [scenario ...] +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') +# name: (scenario argument, extra environment) +SCENARIOS = { + 'arm_race': ('arm_race', {}), + 'arm_race_slow': ('arm_race_slow', {}), +} + + +def compiler(): + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +LAYOUT = '''// A stand-in for the generated layout.h (tests/lazy_fetch_check.py). +#pragma once +#define MOD_MAIN_BASE 0x7000000000ull +#define MOD_MAIN_SIZE 0x4ea0000ull +#define MOD_COUNT 3 +''' + + +def main(): + if os.name != 'nt': + print('lazy_fetch_check: the fault handler it runs is Windows only; skipped') + return 0 + names = sys.argv[1:] or list(SCENARIOS) + unknown = [n for n in names if n not in SCENARIOS] + if unknown: + print('lazy_fetch_check: unknown scenario %s; known: %s' % (', '.join(unknown), ', '.join(SCENARIOS))) + return 2 + work = ROOT / 'build' / 'test' / 'lazy_fetch' + work.mkdir(parents=True, exist_ok=True) + (work / 'layout.h').write_text(LAYOUT) + exe = work / 'lazy_fetch_check.exe' + env = dict(os.environ) + if MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + rt = ROOT / 'runtime' + plat = [rt / 'platform' / 'win32.c', rt / 'platform' / 'common.c'] + libs = ['-lbcrypt', '-lws2_32', '-lpsapi', '-lwinmm', '-ldbghelp', '-lshell32', '-lole32', '-luser32', '-ladvapi32'] + subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-fno-strict-aliasing', + '-march=x86-64-v3', '-I', str(rt), '-I', str(work), + str(ROOT / 'tests' / 'lazy_fetch_check.c'), *map(str, plat), '-o', str(exe), '-lm', + *libs], env=env) + bad = 0 + for name in names: + arg, extra = SCENARIOS[name] + try: + p = subprocess.run([str(exe), arg], env={**env, **extra}, capture_output=True, text=True, timeout=120) + except subprocess.TimeoutExpired: + bad += 1 + print('lazy_fetch_check: %s: FAIL: did not end within 120 s' % name) + continue + code = p.returncode & 0xFFFFFFFF + if code: + bad += 1 + print('lazy_fetch_check: %s: FAIL: exit code %#x%s' % ( + name, code, ' (an access violation)' if code == 0xC0000005 else '')) + sys.stdout.write(p.stdout) + else: + print('lazy_fetch_check: %s: passed' % name) + if bad: + print('lazy_fetch_check: %d of %d scenarios failed' % (bad, len(names))) + return 1 + print('lazy_fetch_check: all %d scenarios passed' % len(names)) + return 0 + + +if __name__ == '__main__': + sys.exit(main())