From 27f93a314ad07807abafc51213992e72af3fe732 Mon Sep 17 00:00:00 2001 From: "automata." Date: Wed, 11 Nov 2026 17:43:01 -0500 Subject: [PATCH] Save completed pipeline cache data during NVIDIA background builds --- runtime/boot.c | 2 +- runtime/gfx_present.c | 2 +- runtime/svc.c | 2 +- runtime/vk.h | 1 + runtime/vk_pipeline.c | 83 ++++++++++---- tests/cache_exit_snapshot_check.py | 173 +++++++++++++++++++++++++++++ 6 files changed, 241 insertions(+), 22 deletions(-) create mode 100644 tests/cache_exit_snapshot_check.py diff --git a/runtime/boot.c b/runtime/boot.c index bbb4a21f..76f7b984 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -327,7 +327,7 @@ void rt_frame_presented(const tex_t *t, uint64_t n) { coop_shutdown(); // LEAVE and BYE to the room, before the process ends fs_save_flush(); vk_pipeline_prewarm_marker_clear(); // an orderly end, not a driver crash in the boot's builds (vk_pipeline.c pm_skipped) - vk_pipeline_save(); // no-op without the Vulkan back end + vk_pipeline_save_exit(); // no-op without the Vulkan back end prof_write(); rtss_limit_restore(); fflush(stderr); diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index 0b034c2a..b0d87802 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -86,7 +86,7 @@ void gfx_window_closed(void) { // The window closed while the boot still built its pipelines: not the driver crash the marker is there to catch. // Left in place, the next start took it for one and every later start skipped the prebuild (vk_pipeline.c pm_skipped). vk_pipeline_prewarm_marker_clear(); - vk_pipeline_save(); + vk_pipeline_save_exit(); prof_write(); // plat_exit runs no atexit handlers rtss_limit_restore(); // RivaTuner Statistics Server's profile as it was coop_shutdown(); // LEAVE and BYE to the room, before the process ends diff --git a/runtime/svc.c b/runtime/svc.c index c94c9061..e4cd9bba 100644 --- a/runtime/svc.c +++ b/runtime/svc.c @@ -247,7 +247,7 @@ static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) { coop_shutdown(); // LEAVE and BYE to the room, before the process ends fs_save_flush(); vk_pipeline_prewarm_marker_clear(); // as the window's close (gfx_present.c gfx_window_closed) - vk_pipeline_save(); + vk_pipeline_save_exit(); prof_write(); rtss_limit_restore(); fflush(NULL); diff --git a/runtime/vk.h b/runtime/vk.h index e2940910..f6683ff3 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -872,6 +872,7 @@ int vk_pipeline_set_indep(int on); typedef struct { uint64_t graphics, compute, failures, hits, layouts, push_layouts, async_built, pending_skips; } VkPipeStats; void vk_pipeline_stats(VkPipeStats *s); void vk_pipeline_save(void); // writes the VkPipelineCache to pipeline-cache.bin next to nx2navi.exe +void vk_pipeline_save_exit(void); // final snapshot: briefly stop new cache builds, with the same bounded wait void vk_pipeline_autosave(void); // vk_pipeline_save at most every 20 s, when pipelines were built since void vk_pipeline_shutdown(void); // waits for background builds, saves, destroys pipelines, layouts and the // cache; before vk_shutdown diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 3b8f9edd..ff4a31a8 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -522,11 +522,25 @@ static void pb_log(const char *when) { typedef struct { VkStructureType sType; const void *pNext; VkPipelineCreateFlags flags; } PipeCiHead; // Builds into P.cache against reads of its data. Every create with P.cache holds this shared (pipe_create_in); the -// copy of the data in cache_save holds it exclusively and only takes it when no build is running. The Vulkan rules +// normal autosave holds it exclusively and only takes it when no build is running. The NVIDIA exit snapshot uses +// shared access too, so an unfinished compile cannot prevent saving completed data; merges still take exclusive +// access. The Vulkan rules // allow reading a cache created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT while pipelines are // built into it, but Intel's Windows driver (101.8991 on an Arc 140V) crashed inside vkGetPipelineCacheData during the // copy, reading through a null entry, with compute and library builds running beside the autosave. static PlatMutex cache_rw = PLAT_MUTEX_INIT; +static volatile int32_t cache_saving_at_exit; + +// Normal autosaves never hold new builds back. At process exit, let existing +// driver calls finish without admitting more until the cache has been copied. +static void cache_build_enter(void) { + for (;;) { + while (__atomic_load_n(&cache_saving_at_exit, __ATOMIC_ACQUIRE)) plat_sleep_ms(1); + plat_mutex_lock_shared(&cache_rw); + if (!__atomic_load_n(&cache_saving_at_exit, __ATOMIC_ACQUIRE)) return; + plat_mutex_unlock_shared(&cache_rw); + } +} // A pipeline cache of this thread's own that its creates build into in place of P.cache; VK_NULL_HANDLE (every thread // but the background builder of large compute shaders) for P.cache itself. A create into it does not hold cache_rw, so @@ -571,7 +585,7 @@ static VkResult pipe_create_in(int kind, void *ci, VkPipeline *pipe, VkPipelineC int shared = cache && !own_cache; if (cache && own_cache) cache = own_cache; uint64_t t0 = perf_mark_begin(); // NX2NAVI_TRACE=perf: the create on any thread, its wait for cache_rw included - if (shared) plat_mutex_lock_shared(&cache_rw); + if (shared) cache_build_enter(); VkResult r; if (shared && P.play) { PipeCiHead *hd = ci; @@ -626,7 +640,7 @@ VkResult vk_pipeline_create_foreign(VkDevice dev, int compute, uint32_t n, const return r; } VkResult r = VK_SUCCESS; - plat_mutex_lock_shared(&cache_rw); + cache_build_enter(); if (!P.play) { // one cache: whether it held them is not known here; its save looks at the data's size r = foreign_one(dev, compute, cache, n, infos, alloc, out); if (r == VK_SUCCESS) __atomic_fetch_add(&FP.made, n, __ATOMIC_RELAXED), __atomic_fetch_add(&cache_merges, n, __ATOMIC_RELAXED); @@ -6310,7 +6324,7 @@ static void lb_warm_prepared(const char *dir) { // A driver whose compiler crashes on one of these would crash at every preparation: seed-pipelines-running.txt marks // the step, and a run that finds the marker of a run that died skips the step for good on this folder (the pipelines // are then built at their first draw, as without a list) until the marker is deleted. -static int cache_save(uint32_t wait_ms); +static int cache_save(uint32_t wait_ms, int exiting); static int seed_pipelines(const char *dir) { const char *mode = cfg_value("NX2NAVI_PREPARE_PIPELINES"); @@ -6424,7 +6438,7 @@ static int seed_pipelines(const char *dir) { while (K.left) plat_cond_wait(&done_cv, &P.lock); // those a pool worker took uint32_t built = K.built - built0, failed = K.failed - failed0; plat_mutex_unlock(&P.lock); - int saved = cache_save(5000) > 0; // as a save at exit: it waits that long for a moment with no build running + int saved = cache_save(5000, 0) > 0; // explicit preparation waits for a moment with no build running plat_file_delete(mark); char msg[640]; snprintf(msg, sizeof msg, "prepare-pipelines: %zu listed pipelines: %u built (%u mesh), %u failed, %u the same as one " @@ -6739,26 +6753,53 @@ static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shu // every save copied and wrote the whole cache whenever a pipeline had been created since the last one, also when the // cache had held it: three saves of 358.1 MB in one run that added 0.3 MB (2026-11-11). // The lock is only ever tried, never waited for: a waiting exclusive request would hold back every new build -// (a shared request queues behind it), so a pending save could stall the replay. wait_ms: how long to keep trying +// (a shared request queues behind it), so a pending autosave could stall the replay. Only the final process-exit +// snapshot uses shared access on NVIDIA; other drivers retain the admission gate and exclusive snapshot, including +// the reported Intel concurrent-read failure above. wait_ms: how long to keep trying // while builds run (0: one try). Returns 0 when no build-free moment came, -1 when the data could not be copied or // its file not written (the log says which), 1 otherwise (saved, or nothing to save). -static int cache_save_one(int which, uint32_t wait_ms) { +static int cache_save_one(int which, uint32_t wait_ms, int exiting) { VkPipelineCache cache = which ? P.play : P.cache; const char *path = which ? P.play_path : P.cache_path; if (!cache) return 1; + VkCtx *ctx = vk_ctx(); + int concurrent = exiting && ctx && ctx->props.vendorID == 0x10de; + int gated = exiting && !concurrent; uint64_t until = plat_now_ms() + wait_ms; - while (!plat_mutex_trylock(&cache_rw)) { - if (plat_now_ms() >= until) return 0; + if (gated) __atomic_store_n(&cache_saving_at_exit, 1, __ATOMIC_RELEASE); + while (!(concurrent ? plat_mutex_trylock_shared(&cache_rw) : plat_mutex_trylock(&cache_rw))) { + if (plat_now_ms() >= until) { + if (gated) __atomic_store_n(&cache_saving_at_exit, 0, __ATOMIC_RELEASE); + return 0; + } plat_sleep_ms(2); } size_t size = 0; void *data = NULL; int sized = vkGetPipelineCacheData(P.device, cache, &size, NULL) == VK_SUCCESS && size; int same = sized && size == P.saved[which]; - int got = sized && !same && (data = malloc(size)) && vkGetPipelineCacheData(P.device, cache, &size, data) == VK_SUCCESS; - plat_mutex_unlock(&cache_rw); + int got = 0; + for (int attempt = 0; sized && !same && attempt < (concurrent ? 3 : 1); attempt++) { + // The cache may grow between the size query and read. A little spare capacity absorbs small additions; an + // incomplete read is retried from a fresh size query. Never replace the last saved cache with a partial one. + size_t spare = concurrent ? 65536 : 0; + if (size > SIZE_MAX - spare) break; + size_t capacity = size + spare; + if (!(data = malloc(capacity))) break; + size = capacity; + VkResult r = vkGetPipelineCacheData(P.device, cache, &size, data); + if (r == VK_SUCCESS && size && size <= capacity) { got = 1; break; } + free(data), data = NULL; + if (!concurrent || r != VK_INCOMPLETE) break; + sized = vkGetPipelineCacheData(P.device, cache, &size, NULL) == VK_SUCCESS && size; + } + if (concurrent) plat_mutex_unlock_shared(&cache_rw); + else plat_mutex_unlock(&cache_rw); + // Release admission before logging, file IO or taking any other lock. A + // paused builder may still own its caller's pipeline or library lock. + if (gated) __atomic_store_n(&cache_saving_at_exit, 0, __ATOMIC_RELEASE); static uint64_t skipped[2]; - int failed = sized && !same && !got; + int failed = !sized || (!same && !got); if (same) skipped[which]++; if (failed) rt_log("vk_pipeline: %s not saved: no copy of its %.1f MB", path, size / 1048576.0); if (got) { @@ -6790,7 +6831,7 @@ static int cache_save_one(int which, uint32_t wait_ms) { } // Both caches' saves (cache_save_one): 1 when each is done, 0 when one found no build-free moment, -1 when one failed. -static int cache_save(uint32_t wait_ms) { +static int cache_save(uint32_t wait_ms, int exiting) { if (!P.ready || !P.cache || !P.cache_path[0]) return 1; // Not under P.lock: holding P.lock through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped // every pipeline lookup of the replay, a freeze at each 20 s autosave. @@ -6800,8 +6841,8 @@ static int cache_save(uint32_t wait_ms) { return 1; } int ok = 1, r; - if (!P.play || P.fold) ok = cache_save_one(0, wait_ms); - if (P.play && (r = cache_save_one(1, wait_ms)) < ok) ok = r; // the worse of the two + if (!P.play || P.fold) ok = cache_save_one(0, wait_ms, exiting); + if (P.play && (r = cache_save_one(1, wait_ms, exiting)) < ok) ok = r; // the worse of the two plat_mutex_unlock(&save_lock); return ok; } @@ -6810,7 +6851,7 @@ static int cache_save(uint32_t wait_ms) { #define SAVE_WAIT_MS 5000 // Returns cache_save's answer. -static int save_with(uint32_t wait_ms) { +static int save_with(uint32_t wait_ms, int exiting) { if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log(); if (P.ready && P.gpl) { rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; " @@ -6865,7 +6906,7 @@ static int save_with(uint32_t wait_ms) { } gfx_shader_record_flush(1); if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below - int saved = cache_save(wait_ms); + int saved = cache_save(wait_ms, exiting); if (!saved && wait_ms) rt_log("vk_pipeline: %s not saved: pipeline builds kept running for %u ms", P.play ? P.play_path : P.cache_path, (unsigned)wait_ms); @@ -6874,7 +6915,11 @@ static int save_with(uint32_t wait_ms) { // The save at exit and after the shader preparation: it waits up to SAVE_WAIT_MS for a moment with no build running. void vk_pipeline_save(void) { - save_with(SAVE_WAIT_MS); + save_with(SAVE_WAIT_MS, 0); +} + +void vk_pipeline_save_exit(void) { + save_with(SAVE_WAIT_MS, 1); } // The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it @@ -6903,7 +6948,7 @@ static void autosave_once(void) { if (!plat_mutex_trylock(&cache_rw)) return; plat_mutex_unlock(&cache_rw); } - if (save_tried(&sr, n, now, save_with(0) > 0)) + if (save_tried(&sr, n, now, save_with(0, 0) > 0)) rt_log("vk_pipeline: the pipeline cache was not saved in %d tries in a row; the next try is made when a new " "pipeline is built, or at exit", SAVE_RETRIES + 1); } diff --git a/tests/cache_exit_snapshot_check.py b/tests/cache_exit_snapshot_check.py new file mode 100644 index 00000000..f31c47a8 --- /dev/null +++ b/tests/cache_exit_snapshot_check.py @@ -0,0 +1,173 @@ +"""Windows: exercise exit snapshots during builds, growing cache data, and the conservative other-driver path.""" +from pathlib import Path +import subprocess +import sys + +root = Path(__file__).resolve().parents[1] +source = (root / 'runtime/vk_pipeline.c').read_text() + + +def function(signature): + start = source.index(signature) + brace = source.index('{', start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +harness = r''' +#include +#include +#include +#include +#include +#include +typedef SRWLOCK PlatMutex; +typedef void *VkPipelineCache; +typedef int VkResult; +#define VK_SUCCESS 0 +#define VK_INCOMPLETE 5 +typedef struct { struct { unsigned vendorID; } props; } VkCtx; +static VkCtx context = {{0x8086}}; +static VkCtx *vk_ctx(void) { return &context; } +static size_t mock_size = 4; +static int grow_once, grow_always, query_fail; +static PlatMutex cache_rw = SRWLOCK_INIT; +static volatile int32_t cache_saving_at_exit; +static volatile LONG active, snapshots; +static HANDLE old_in, release_old, new_in, file_in, release_file; +static int block_file, fail_file, writes, save_result; +static struct { + VkPipelineCache cache, play; + int device, fold, feedback; + const char *cache_path, *play_path; + uint64_t saved[2], cache_loaded, play_loaded; +} P; +static void plat_mutex_lock_shared(PlatMutex *m) { AcquireSRWLockShared(m); } +static void plat_mutex_unlock_shared(PlatMutex *m) { ReleaseSRWLockShared(m); } +static int plat_mutex_trylock(PlatMutex *m) { return TryAcquireSRWLockExclusive(m); } +static int plat_mutex_trylock_shared(PlatMutex *m) { return TryAcquireSRWLockShared(m); } +static void plat_mutex_unlock(PlatMutex *m) { ReleaseSRWLockExclusive(m); } +static void plat_sleep_ms(uint32_t ms) { Sleep(ms); } +static uint64_t plat_now_ms(void) { return GetTickCount64(); } +static void rt_log(const char *fmt, ...) { (void)fmt; } +static void fb_log(const char *when) { (void)when; assert(!cache_saving_at_exit); } +static void plat_file_delete(const char *path) { (void)path; } +static int plat_file_write_atomic(const char *path, const void *data, size_t size) { + (void)path; + assert(!cache_saving_at_exit); // admission must reopen before IO and diagnostics + assert(size == mock_size && !memcmp(data, "test", 4)); + writes++; + if (block_file) { + SetEvent(file_in); + assert(WaitForSingleObject(release_file, 2000) == WAIT_OBJECT_0); + } + return !fail_file; +} +static int vkGetPipelineCacheData(int device, VkPipelineCache cache, size_t *size, void *data) { + (void)device; (void)cache; + if (context.props.vendorID != 0x10de) assert(!active); + InterlockedIncrement(&snapshots); + if (!data && query_fail) return -1; + if (data && (grow_once || grow_always)) { + grow_once = 0; mock_size = *size + 100; return VK_INCOMPLETE; + } + if (data) { assert(*size >= mock_size); memcpy(data, "test", 4); } + *size = mock_size; + return VK_SUCCESS; +} +''' +harness += function('static void cache_build_enter(void)') + '\n' +harness += function('static int cache_save_one(int which, uint32_t wait_ms, int exiting)') + '\n' +harness += r''' +static DWORD WINAPI builder(void *old) { + cache_build_enter(); + InterlockedIncrement(&active); + SetEvent(old ? old_in : new_in); + if (old) assert(WaitForSingleObject(release_old, 2000) == WAIT_OBJECT_0); + InterlockedDecrement(&active); + plat_mutex_unlock_shared(&cache_rw); + return 0; +} +static DWORD WINAPI saver(void *unused) { + (void)unused; + save_result = cache_save_one(0, 2000, 1); + return 0; +} +static void join(HANDLE t) { assert(WaitForSingleObject(t, 2000) == WAIT_OBJECT_0); CloseHandle(t); } +int main(void) { + P.cache = (void *)1; P.cache_path = "unused"; P.play_path = "unused-play"; P.feedback = 1; + old_in = CreateEventW(NULL, TRUE, FALSE, NULL); + release_old = CreateEventW(NULL, TRUE, FALSE, NULL); + new_in = CreateEventW(NULL, TRUE, FALSE, NULL); + file_in = CreateEventW(NULL, TRUE, FALSE, NULL); + release_file = CreateEventW(NULL, TRUE, FALSE, NULL); + HANDLE old = CreateThread(NULL, 0, builder, (void *)1, 0, NULL); + assert(WaitForSingleObject(old_in, 2000) == WAIT_OBJECT_0); + block_file = 1; + HANDLE saving = CreateThread(NULL, 0, saver, NULL, 0, NULL); + uint64_t until = GetTickCount64() + 2000; + while (!__atomic_load_n(&cache_saving_at_exit, __ATOMIC_ACQUIRE)) { + assert(GetTickCount64() < until); Sleep(1); + } + HANDLE next = CreateThread(NULL, 0, builder, NULL, 0, NULL); + assert(WaitForSingleObject(new_in, 20) == WAIT_TIMEOUT); + SetEvent(release_old); + assert(WaitForSingleObject(file_in, 2000) == WAIT_OBJECT_0); + assert(WaitForSingleObject(new_in, 2000) == WAIT_OBJECT_0); // IO does not hold admission closed + SetEvent(release_file); + join(old); join(next); join(saving); + assert(save_result == 1 && snapshots == 2 && writes == 1 && !cache_saving_at_exit); + + // Ordinary autosave does not wait behind an active build or close admission. + cache_build_enter(); + assert(cache_save_one(0, 0, 0) == 0 && !cache_saving_at_exit); + // A final save times out on a genuinely long driver call and restores admission. + assert(cache_save_one(0, 1, 1) == 0 && !cache_saving_at_exit); + plat_mutex_unlock_shared(&cache_rw); + cache_build_enter(); plat_mutex_unlock_shared(&cache_rw); + + block_file = 0; + assert(cache_save_one(0, 100, 1) == 1 && writes == 1); // unchanged data is not rewritten + P.saved[0] = 0; fail_file = 1; + assert(cache_save_one(0, 100, 1) == -1 && !cache_saving_at_exit && P.saved[0] == 0); + // NVIDIA exit reads can finish while a build remains active; normal autosaves retain the conservative path. + fail_file=0;context.props.vendorID=0x10de;P.saved[0]=0; + ResetEvent(old_in);ResetEvent(release_old); + old=CreateThread(NULL,0,builder,(void *)1,0,NULL); + assert(WaitForSingleObject(old_in,2000)==WAIT_OBJECT_0); + assert(active==1); + int before=writes; + assert(cache_save_one(0,100,1)==1 && writes==before+1 && active==1 && !cache_saving_at_exit); + assert(cache_save_one(0,0,0)==0 && active==1); // no gameplay autosave behavior change + SetEvent(release_old);join(old); + + // Cache growth retries a fresh size query, and a partial result never replaces an existing file. + P.saved[0]=0;grow_once=1; + assert(cache_save_one(0,100,1)==1 && P.saved[0]==mock_size); + before=writes; + assert(cache_save_one(0,100,1)==1 && writes==before); + uint64_t saved=P.saved[0];mock_size++;grow_always=1; + assert(cache_save_one(0,100,1)==-1 && writes==before && P.saved[0]==saved); + grow_always=0;query_fail=1; + assert(cache_save_one(0,100,1)==-1 && writes==before && P.saved[0]==saved); + query_fail=0; + + // A merge owns exclusive access: even the concurrent snapshot must not race it. + assert(plat_mutex_trylock(&cache_rw)); + assert(cache_save_one(0,0,1)==0 && writes==before); + plat_mutex_unlock(&cache_rw); + puts("Exit snapshot: active builds, merge exclusion, cache growth/retry, partial/error preservation and fallback passed"); + return 0; +} +''' +out = root / 'build/test/cache-exit-snapshot' +out.mkdir(parents=True, exist_ok=True) +(out / 'check.c').write_text(harness) +cc = sys.argv[1] if len(sys.argv) > 1 else 'clang' +subprocess.run([cc, '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', str(out/'check.c'), + '-o', str(out/'check.exe')], check=True) +subprocess.run([str(out/'check.exe')], check=True, timeout=15)