diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 731c7f9c..cebc8851 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1168,6 +1168,31 @@ static uint32_t *gen_chunk[GEN_CHUNKS]; static uint32_t gen_now; // last generation handed out static PlatMutex gen_lock = PLAT_MUTEX_INIT; static uint64_t gen_pages[1024]; + +// Generations are 32 bits, and nothing compares them across a wrap: past it a stamp from before reads as newer than +// every later check, and a stamp of 0 as never stamped, so textures and pool mirrors the CPU writes can miss a write +// and every check reads as a write. At about 181 a frame that comes after about 110 h of play at 60 fps, 27 h at 240. +// Only trace=perf's "mem: sizes" counted them, so gen_next, which hands each one out, logs where the session stands at +// 0xC0000000, 0xD0000000, 0xE0000000 and 0xF0000000, and the wrap itself (gen_mark): five lines at most. +static void gen_mark(uint32_t g) { // gen_lock held exclusively; the log takes no lock of this file + uint64_t frames = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + double secs = rt_guest_start_ns ? (double)(kern_now_ns() - rt_guest_start_ns) / 1e9 : 0, all = 4294967296.0; + if (!g) { + rt_log("mem: page generations ran out at frame %llu after %.1f h: from here on textures and pool mirrors the " + "CPU writes can show old content and every check uploads again; restart the game", + (unsigned long long)frames, secs / 3600); + return; + } + rt_log("mem: page generations: %u of 4294967295 handed out at frame %llu (%.0f a frame); at this rate they run out " + "in about %.1f h of play, after which textures and pool mirrors the CPU writes can miss a write and every " + "check reads as a write: restart the game before then", g, (unsigned long long)frames, + frames ? g / (double)frames : 0.0, secs > 0 ? (all - g) * secs / g / 3600 : 0.0); +} +static inline uint32_t gen_next(void) { // gen_lock held exclusively + uint32_t g = ++gen_now; + if (__builtin_expect(!(g & 0x0fffffffu) && (g >= 0xc0000000u || !g), 0)) gen_mark(g); + return g; +} // The newest generation stamped on any page of each 2 MiB region (the write-watch snapshot regions below): the mirror // sync skips regions with nothing newer than its pool's mirror. #define REG_SHIFT 21 @@ -1280,7 +1305,7 @@ void vk_guest_written(uint64_t addr, uint64_t size) { if (!size || nvn_gpu_only(addr)) return; uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; plat_mutex_lock(&gen_lock); - stamp_range(lo, hi, ++gen_now); + stamp_range(lo, hi, gen_next()); plat_mutex_unlock(&gen_lock); } @@ -1288,7 +1313,7 @@ int vk_mem_changed(uint64_t addr, uint64_t size, uint32_t since, uint32_t *now) if (!size) { *now = since; return 0; } uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; plat_mutex_lock(&gen_lock); - uint32_t g = ++gen_now; + uint32_t g = gen_next(); int tracked = 1, frozen = 1; for (uint64_t a = lo & ~((1ull << REG_SHIFT) - 1); a < hi && frozen; a += 1ull << REG_SHIFT) frozen = mem_frozen(a); while (!frozen) { @@ -1334,7 +1359,7 @@ static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) { while (k + 1 <= r1 && snap_at[k + 1] != epoch && !mem_frozen((k + 1) << SNAP_SHIFT)) k++; uint64_t a = r << SNAP_SHIFT, b = (k + 1) << SNAP_SHIFT; int tracked = 1; - if (!g) g = ++gen_now; + if (!g) g = gen_next(); for (;;) { uint64_t n = ww_query(a, b - a); if (n == UINT64_MAX) { tracked = 0; break; } @@ -1973,7 +1998,7 @@ static void shadow_sync_pool(PoolRec *r, uint32_t epoch, int cpu) { s_queries += snap_regions(r0, r1, epoch); for (uint64_t k = r0; k <= r1 && tracked; k++) tracked = !snap_untracked[k]; } else { // outside the snapshot regions, or before the first guest submission: one query per unfrozen run - uint32_t g = ++gen_now; + uint32_t g = gen_next(); for (uint64_t a = lo; a < hi;) { uint64_t b = (a | ((1ull << REG_SHIFT) - 1)) + 1; if (b > hi) b = hi; @@ -2143,7 +2168,7 @@ static void table_wb_cb(void *arg) { s_tab_wb += moved, s_tab_wb_seen += seen, s_tab_wb_jobs++; if (nwrote) { plat_mutex_lock(&gen_lock); - uint32_t g = ++gen_now; + uint32_t g = gen_next(); for (int i = 0; i < nwrote; i++) stamp_page_as(wrote[i], g, back); plat_mutex_unlock(&gen_lock); } @@ -2159,7 +2184,7 @@ static void gpu_written(uint64_t addr, uint64_t size, int back) { uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; back = back && wb_no_reup(); // read before gen_lock: its first call logs plat_mutex_lock(&gen_lock); - uint32_t g = ++gen_now; + uint32_t g = gen_next(); for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page_as(a, g, back); plat_mutex_unlock(&gen_lock); } @@ -2315,7 +2340,7 @@ void vk_mem_dirty_submit(void) { if (r->kind == POOL_IMPORT && r->bufbound) dlist[n++] = r; } plat_mutex_lock(&gen_lock); - uint32_t g = ++gen_now, since = dirty_gen; + uint32_t g = gen_next(), since = dirty_gen; dirty_gen = g; for (int i = 0; i < n;) { // one query per run of pools (sorted by address) uint64_t lo = page_lo(dlist[i]), hi = page_hi(dlist[i]); @@ -3503,7 +3528,7 @@ static void swb_run(uint64_t a, uint64_t b) { if (b <= a || !shadow_write_back(a, b - a) || nvn_gpu_only(a)) return; int back = wb_no_reup(); plat_mutex_lock(&gen_lock); - uint32_t g = ++gen_now; + uint32_t g = gen_next(); for (uint64_t x = a & ~0xfffull; x < b; x += 1ull << GEN_PAGE_SHIFT) stamp_page_as(x, g, back); plat_mutex_unlock(&gen_lock); } @@ -4005,7 +4030,7 @@ static void freeze_pass(void) { int ok = mem_freeze(a); uint64_t t1 = qpc(); slow_mem_call("freeze", a, 1ull << REG_SHIFT, 0, t1 - t0); - if (ok) query_stamp(a, a + (1ull << REG_SHIFT), ++gen_now); + if (ok) query_stamp(a, a + (1ull << REG_SHIFT), gen_next()); else n_frz_refused++; if (!freeze_budget_ns) continue; frz_freeze_done(&frz, t1, t1 - t0, frz_quiet_ns); diff --git a/tests/gen_wrap_check.c b/tests/gen_wrap_check.c new file mode 100644 index 00000000..41f46472 --- /dev/null +++ b/tests/gen_wrap_check.c @@ -0,0 +1,145 @@ +// runtime/vk_memory.c's page write generations (the section from its "page write generations" banner to its "dirty +// census" banner, extracted by tests/gen_wrap_check.py into gen_section.inc): the lines gen_next logs as the 32-bit +// generations near their wrap, and what the comparisons answer across it. Prints each failed check and exits 1 on any. +// Stubs: no write-watch (mem_written_pages reports no page), nothing frozen, no pools. +// +// The comparisons across the wrap are wrong today (AUDIT F-8-01: the generations have no wrap handling; its fix waits +// for approval). The check expects the four wrong answers and says so: the change that handles the wrap turns them +// into failures here, and flips the expectations with it. +#include +#include +#include +#include +#include + +#define RT_ADDRESS_LIMIT (1ull << 39) +typedef struct { int x; } PlatMutex; +#define PLAT_MUTEX_INIT {0} +static void plat_mutex_lock(PlatMutex *m) { (void)m; } +static void plat_mutex_unlock(PlatMutex *m) { (void)m; } +static void plat_mutex_lock_shared(PlatMutex *m) { (void)m; } +static void plat_mutex_unlock_shared(PlatMutex *m) { (void)m; } +static uint64_t qpc(void) { return 0; } +static int rt_trace; +#define RT_TRACE_PERF 1 +static void plat_add64(volatile int64_t *p, int64_t v) { *p += v; } +static int64_t plat_xchg64(volatile int64_t *p, int64_t v) { int64_t o = *p; *p = v; return o; } +static struct { uint64_t frames; } gfx_counters; +static uint64_t fake_now, rt_guest_start_ns; +static uint64_t kern_now_ns(void) { return fake_now; } +static void grow(void **arr, int *cap, size_t elem) { + int n = *cap ? *cap * 2 : 64; + *arr = realloc(*arr, (size_t)n * elem); + *cap = n; +} +uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { + (void)addr, (void)size, (void)pages, (void)max; + return 0; +} +int mem_frozen(uint64_t a) { (void)a; return 0; } +int mem_watched(uint64_t addr, uint64_t size) { (void)addr, (void)size; return 1; } +int nvn_gpu_only(uint64_t a) { (void)a; return 0; } +static const char *cfg_get(const char *k) { (void)k; return NULL; } +typedef struct { uintptr_t lo, hi, base; void *buf; } pidx_ent; // snap_widen's pool lookup: no pools +typedef struct { int n; } pidx; +static pidx pool_index; +static PlatMutex pool_lock; +static const pidx_ent *pidx_find(const pidx *x, uintptr_t a, uint64_t l) { (void)x, (void)a, (void)l; return NULL; } +enum { POOL_IMPORT = 1 }; +typedef struct { int kind; void *buf; uint64_t import_lo, import_hi; } PoolRec; + +// The log: the generation lines are kept, the others dropped. +static char gen_lines[16][512]; +static int n_gen_lines; +static void rt_log(const char *fmt, ...) { + char line[512]; + va_list ap; + va_start(ap, fmt); + vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + if (strncmp(line, "mem: page generations", 21)) return; + if (n_gen_lines < 16) memcpy(gen_lines[n_gen_lines], line, sizeof line); + n_gen_lines++; +} +static void rt_fatal(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + printf("FATAL: "); + vprintf(fmt, ap); + printf("\n"); + va_end(ap); + exit(70); +} + +#include "gen_section.inc" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +#define TEX 0x100000000ull // a texture of 4 pages in one 2 MiB region +#define OTHER 0x100200000ull // a second one in the next region + +// Hands out generations with vk_guest_written from just below `to` until gen_now reaches `to` + 3. +static void walk_to(uint32_t to) { + gen_now = to - 4; + for (int i = 0; i < 7; i++) vk_guest_written(0x300000000ull, 0x1000); +} + +int main(void) { + // ---- the lines: one at each of 0xC0000000, 0xD0000000, 0xE0000000, 0xF0000000 and at the wrap, no other + rt_guest_start_ns = 1000000000ull; + fake_now = rt_guest_start_ns + 3600ull * 1000000000ull; // an hour in + gfx_counters.frames = 216000; // at 60 fps + walk_to(0x10000000u); // below the first mark: nothing + walk_to(0xb0000000u); + CHECK(n_gen_lines == 0, "%d lines before 0xC0000000", n_gen_lines); + static const uint32_t marks[] = {0xc0000000u, 0xd0000000u, 0xe0000000u, 0xf0000000u}; + for (int k = 0; k < 4; k++) { + int was = n_gen_lines; + walk_to(marks[k]); + char want[32]; + snprintf(want, sizeof want, ": %u of 4294967295", marks[k]); + CHECK(n_gen_lines == was + 1 && strstr(gen_lines[was], want), "the mark at %#x gave %d lines (%s)", marks[k], + n_gen_lines - was, n_gen_lines > was ? gen_lines[was] : "none"); + } + // 0xC0000000 in an hour of 216,000 frames: about 14,913 a frame, and a quarter of the range left: 20 min + CHECK(n_gen_lines >= 1 && strstr(gen_lines[0], "(14913 a frame)") && strstr(gen_lines[0], "in about 0.3 h"), + "the first line's rate or hours: %s", n_gen_lines ? gen_lines[0] : "none"); + int was = n_gen_lines; + walk_to(0); // across the wrap + CHECK(n_gen_lines == was + 1 && strstr(gen_lines[was], "ran out at frame 216000 after 1.0 h"), + "the wrap gave %d lines (%s)", n_gen_lines - was, n_gen_lines > was ? gen_lines[was] : "none"); + walk_to(0x20000000u); + CHECK(n_gen_lines == 5, "%d lines in all, not 5", n_gen_lines); + for (int i = 0; i < n_gen_lines && i < 16; i++) printf(" %s\n", gen_lines[i]); + + // ---- the comparisons across the wrap: the four answers known to be wrong (expected, see the top) + int known = 0; + gen_now = 999; + vk_guest_written(TEX, 0x4000); + vk_guest_written(OTHER, 0x4000); + uint32_t w1, w2, now; + vk_mem_changed(TEX, 0x4000, 0, &w1); + vk_mem_changed(OTHER, 0x4000, 0, &w2); + gen_now = UINT32_MAX - 6; + CHECK(vk_mem_changed(TEX, 0x4000, w1, &w1) == 0, "an unwritten texture read as written before the wrap"); + vk_mem_changed(OTHER, 0x4000, w2, &w2); + for (int i = 0; i < 10; i++) vk_mem_changed(0x200000000ull, 0x1000, 0, &now); // other checks use generations + vk_guest_written(TEX, 0x1000); // the CPU writes the texture after the wrap + known += vk_mem_changed(TEX, 0x4000, w1, &now) != 1; // 1. the write is missed + vk_mem_changed(OTHER, 0x4000, w2, &w2); + known += vk_mem_changed(OTHER, 0x4000, w2, &now) == 1; // 2. an unwritten texture reads as written at every check + w2 = now; + known += vk_mem_changed_snap(OTHER, 0x4000, w2, 7, &now) == 1; // 3. the same through the snapshot path + gen_now = UINT32_MAX; + vk_guest_written(TEX + 0x2000, 0x1000); // 4. a write stamped at the wrap gets 0, which reads as never stamped + known += vk_mem_changed(TEX + 0x2000, 0x1000, 0, &now) != 1; + CHECK(known == 4, "%d of the 4 known wrong answers across the wrap remain: if the wrap is handled now, flip these " + "expectations", known); + printf("gen_wrap_check: across the wrap %d of 4 comparisons still answer wrong (known, expected until the wrap " + "is handled)\n", known); + + if (fails) return printf("gen_wrap_check: %d checks failed\n", fails), 1; + printf("gen_wrap_check: all checks passed\n"); + return 0; +} diff --git a/tests/gen_wrap_check.py b/tests/gen_wrap_check.py new file mode 100644 index 00000000..c8789c43 --- /dev/null +++ b/tests/gen_wrap_check.py @@ -0,0 +1,57 @@ +"""Build and run tests/gen_wrap_check.c: runtime/vk_memory.c's page write generations, extracted verbatim from its +"page write generations" banner to its "dirty census" banner, with the lines gen_next logs as the 32-bit generations +near their wrap (one at each of 0xC0000000, 0xD0000000, 0xE0000000, 0xF0000000 and one at the wrap), and the four +comparisons known to answer wrong across the wrap. + +No GPU, no game files, any Python. + +Usage: gen_wrap_check.py [--cc ] +""" +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') +BEGIN = '// ---------------------------------------------------------------- page write generations' +END = '// ---------------------------------------------------------------- dirty census' + + +def compiler(cc): + if cc: + return cc + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--cc', help='the C compiler (default: $CC, MSYS2 UCRT64 clang, clang on PATH)') + args = ap.parse_args() + src = (ROOT / 'runtime' / 'vk_memory.c').read_text(encoding='utf-8-sig') + a, b = src.find(BEGIN), src.find(END) + if a < 0 or b < a: + print('gen_wrap_check: FAIL: the section banners are not in runtime/vk_memory.c') + return 1 + work = ROOT / 'build' / 'test' / 'gen_wrap' + work.mkdir(parents=True, exist_ok=True) + (work / 'gen_section.inc').write_text('// Extracted from runtime/vk_memory.c by tests/gen_wrap_check.py.\n' + + src[a:b], encoding='utf-8') + exe = work / ('gen_wrap_check.exe' if os.name == 'nt' else 'gen_wrap_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(args.cc), '-std=gnu11', '-O1', '-Wall', '-Wno-unused-function', + '-Wno-unused-variable', '-I', str(work), str(ROOT / 'tests' / 'gen_wrap_check.c'), '-o', + str(exe)], env=env) + return subprocess.run([str(exe)], env=env, timeout=120).returncode + + +if __name__ == '__main__': + sys.exit(main())