Save completed pipeline cache data during NVIDIA background builds

This commit is contained in:
automata.
2026-11-11 17:43:01 -05:00
committed by tangerineturtle1
parent 60ac5f5982
commit 27f93a314a
6 changed files with 241 additions and 22 deletions

View File

@@ -327,7 +327,7 @@ void rt_frame_presented(const tex_t *t, uint64_t n) {
coop_shutdown(); // LEAVE and BYE to the room, before the process ends
fs_save_flush();
vk_pipeline_prewarm_marker_clear(); // an orderly end, not a driver crash in the boot's builds (vk_pipeline.c pm_skipped)
vk_pipeline_save(); // no-op without the Vulkan back end
vk_pipeline_save_exit(); // no-op without the Vulkan back end
prof_write();
rtss_limit_restore();
fflush(stderr);

View File

@@ -86,7 +86,7 @@ void gfx_window_closed(void) {
// The window closed while the boot still built its pipelines: not the driver crash the marker is there to catch.
// Left in place, the next start took it for one and every later start skipped the prebuild (vk_pipeline.c pm_skipped).
vk_pipeline_prewarm_marker_clear();
vk_pipeline_save();
vk_pipeline_save_exit();
prof_write(); // plat_exit runs no atexit handlers
rtss_limit_restore(); // RivaTuner Statistics Server's profile as it was
coop_shutdown(); // LEAVE and BYE to the room, before the process ends

View File

@@ -247,7 +247,7 @@ static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) {
coop_shutdown(); // LEAVE and BYE to the room, before the process ends
fs_save_flush();
vk_pipeline_prewarm_marker_clear(); // as the window's close (gfx_present.c gfx_window_closed)
vk_pipeline_save();
vk_pipeline_save_exit();
prof_write();
rtss_limit_restore();
fflush(NULL);

View File

@@ -872,6 +872,7 @@ int vk_pipeline_set_indep(int on);
typedef struct { uint64_t graphics, compute, failures, hits, layouts, push_layouts, async_built, pending_skips; } VkPipeStats;
void vk_pipeline_stats(VkPipeStats *s);
void vk_pipeline_save(void); // writes the VkPipelineCache to pipeline-cache.bin next to nx2navi.exe
void vk_pipeline_save_exit(void); // final snapshot: briefly stop new cache builds, with the same bounded wait
void vk_pipeline_autosave(void); // vk_pipeline_save at most every 20 s, when pipelines were built since
void vk_pipeline_shutdown(void); // waits for background builds, saves, destroys pipelines, layouts and the
// cache; before vk_shutdown

View File

@@ -522,11 +522,25 @@ static void pb_log(const char *when) {
typedef struct { VkStructureType sType; const void *pNext; VkPipelineCreateFlags flags; } PipeCiHead;
// Builds into P.cache against reads of its data. Every create with P.cache holds this shared (pipe_create_in); the
// copy of the data in cache_save holds it exclusively and only takes it when no build is running. The Vulkan rules
// normal autosave holds it exclusively and only takes it when no build is running. The NVIDIA exit snapshot uses
// shared access too, so an unfinished compile cannot prevent saving completed data; merges still take exclusive
// access. The Vulkan rules
// allow reading a cache created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT while pipelines are
// built into it, but Intel's Windows driver (101.8991 on an Arc 140V) crashed inside vkGetPipelineCacheData during the
// copy, reading through a null entry, with compute and library builds running beside the autosave.
static PlatMutex cache_rw = PLAT_MUTEX_INIT;
static volatile int32_t cache_saving_at_exit;
// Normal autosaves never hold new builds back. At process exit, let existing
// driver calls finish without admitting more until the cache has been copied.
static void cache_build_enter(void) {
for (;;) {
while (__atomic_load_n(&cache_saving_at_exit, __ATOMIC_ACQUIRE)) plat_sleep_ms(1);
plat_mutex_lock_shared(&cache_rw);
if (!__atomic_load_n(&cache_saving_at_exit, __ATOMIC_ACQUIRE)) return;
plat_mutex_unlock_shared(&cache_rw);
}
}
// A pipeline cache of this thread's own that its creates build into in place of P.cache; VK_NULL_HANDLE (every thread
// but the background builder of large compute shaders) for P.cache itself. A create into it does not hold cache_rw, so
@@ -571,7 +585,7 @@ static VkResult pipe_create_in(int kind, void *ci, VkPipeline *pipe, VkPipelineC
int shared = cache && !own_cache;
if (cache && own_cache) cache = own_cache;
uint64_t t0 = perf_mark_begin(); // NX2NAVI_TRACE=perf: the create on any thread, its wait for cache_rw included
if (shared) plat_mutex_lock_shared(&cache_rw);
if (shared) cache_build_enter();
VkResult r;
if (shared && P.play) {
PipeCiHead *hd = ci;
@@ -626,7 +640,7 @@ VkResult vk_pipeline_create_foreign(VkDevice dev, int compute, uint32_t n, const
return r;
}
VkResult r = VK_SUCCESS;
plat_mutex_lock_shared(&cache_rw);
cache_build_enter();
if (!P.play) { // one cache: whether it held them is not known here; its save looks at the data's size
r = foreign_one(dev, compute, cache, n, infos, alloc, out);
if (r == VK_SUCCESS) __atomic_fetch_add(&FP.made, n, __ATOMIC_RELAXED), __atomic_fetch_add(&cache_merges, n, __ATOMIC_RELAXED);
@@ -6310,7 +6324,7 @@ static void lb_warm_prepared(const char *dir) {
// A driver whose compiler crashes on one of these would crash at every preparation: seed-pipelines-running.txt marks
// the step, and a run that finds the marker of a run that died skips the step for good on this folder (the pipelines
// are then built at their first draw, as without a list) until the marker is deleted.
static int cache_save(uint32_t wait_ms);
static int cache_save(uint32_t wait_ms, int exiting);
static int seed_pipelines(const char *dir) {
const char *mode = cfg_value("NX2NAVI_PREPARE_PIPELINES");
@@ -6424,7 +6438,7 @@ static int seed_pipelines(const char *dir) {
while (K.left) plat_cond_wait(&done_cv, &P.lock); // those a pool worker took
uint32_t built = K.built - built0, failed = K.failed - failed0;
plat_mutex_unlock(&P.lock);
int saved = cache_save(5000) > 0; // as a save at exit: it waits that long for a moment with no build running
int saved = cache_save(5000, 0) > 0; // explicit preparation waits for a moment with no build running
plat_file_delete(mark);
char msg[640];
snprintf(msg, sizeof msg, "prepare-pipelines: %zu listed pipelines: %u built (%u mesh), %u failed, %u the same as one "
@@ -6739,26 +6753,53 @@ static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shu
// every save copied and wrote the whole cache whenever a pipeline had been created since the last one, also when the
// cache had held it: three saves of 358.1 MB in one run that added 0.3 MB (2026-11-11).
// The lock is only ever tried, never waited for: a waiting exclusive request would hold back every new build
// (a shared request queues behind it), so a pending save could stall the replay. wait_ms: how long to keep trying
// (a shared request queues behind it), so a pending autosave could stall the replay. Only the final process-exit
// snapshot uses shared access on NVIDIA; other drivers retain the admission gate and exclusive snapshot, including
// the reported Intel concurrent-read failure above. wait_ms: how long to keep trying
// while builds run (0: one try). Returns 0 when no build-free moment came, -1 when the data could not be copied or
// its file not written (the log says which), 1 otherwise (saved, or nothing to save).
static int cache_save_one(int which, uint32_t wait_ms) {
static int cache_save_one(int which, uint32_t wait_ms, int exiting) {
VkPipelineCache cache = which ? P.play : P.cache;
const char *path = which ? P.play_path : P.cache_path;
if (!cache) return 1;
VkCtx *ctx = vk_ctx();
int concurrent = exiting && ctx && ctx->props.vendorID == 0x10de;
int gated = exiting && !concurrent;
uint64_t until = plat_now_ms() + wait_ms;
while (!plat_mutex_trylock(&cache_rw)) {
if (plat_now_ms() >= until) return 0;
if (gated) __atomic_store_n(&cache_saving_at_exit, 1, __ATOMIC_RELEASE);
while (!(concurrent ? plat_mutex_trylock_shared(&cache_rw) : plat_mutex_trylock(&cache_rw))) {
if (plat_now_ms() >= until) {
if (gated) __atomic_store_n(&cache_saving_at_exit, 0, __ATOMIC_RELEASE);
return 0;
}
plat_sleep_ms(2);
}
size_t size = 0;
void *data = NULL;
int sized = vkGetPipelineCacheData(P.device, cache, &size, NULL) == VK_SUCCESS && size;
int same = sized && size == P.saved[which];
int got = sized && !same && (data = malloc(size)) && vkGetPipelineCacheData(P.device, cache, &size, data) == VK_SUCCESS;
plat_mutex_unlock(&cache_rw);
int got = 0;
for (int attempt = 0; sized && !same && attempt < (concurrent ? 3 : 1); attempt++) {
// The cache may grow between the size query and read. A little spare capacity absorbs small additions; an
// incomplete read is retried from a fresh size query. Never replace the last saved cache with a partial one.
size_t spare = concurrent ? 65536 : 0;
if (size > SIZE_MAX - spare) break;
size_t capacity = size + spare;
if (!(data = malloc(capacity))) break;
size = capacity;
VkResult r = vkGetPipelineCacheData(P.device, cache, &size, data);
if (r == VK_SUCCESS && size && size <= capacity) { got = 1; break; }
free(data), data = NULL;
if (!concurrent || r != VK_INCOMPLETE) break;
sized = vkGetPipelineCacheData(P.device, cache, &size, NULL) == VK_SUCCESS && size;
}
if (concurrent) plat_mutex_unlock_shared(&cache_rw);
else plat_mutex_unlock(&cache_rw);
// Release admission before logging, file IO or taking any other lock. A
// paused builder may still own its caller's pipeline or library lock.
if (gated) __atomic_store_n(&cache_saving_at_exit, 0, __ATOMIC_RELEASE);
static uint64_t skipped[2];
int failed = sized && !same && !got;
int failed = !sized || (!same && !got);
if (same) skipped[which]++;
if (failed) rt_log("vk_pipeline: %s not saved: no copy of its %.1f MB", path, size / 1048576.0);
if (got) {
@@ -6790,7 +6831,7 @@ static int cache_save_one(int which, uint32_t wait_ms) {
}
// Both caches' saves (cache_save_one): 1 when each is done, 0 when one found no build-free moment, -1 when one failed.
static int cache_save(uint32_t wait_ms) {
static int cache_save(uint32_t wait_ms, int exiting) {
if (!P.ready || !P.cache || !P.cache_path[0]) return 1;
// Not under P.lock: holding P.lock through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped
// every pipeline lookup of the replay, a freeze at each 20 s autosave.
@@ -6800,8 +6841,8 @@ static int cache_save(uint32_t wait_ms) {
return 1;
}
int ok = 1, r;
if (!P.play || P.fold) ok = cache_save_one(0, wait_ms);
if (P.play && (r = cache_save_one(1, wait_ms)) < ok) ok = r; // the worse of the two
if (!P.play || P.fold) ok = cache_save_one(0, wait_ms, exiting);
if (P.play && (r = cache_save_one(1, wait_ms, exiting)) < ok) ok = r; // the worse of the two
plat_mutex_unlock(&save_lock);
return ok;
}
@@ -6810,7 +6851,7 @@ static int cache_save(uint32_t wait_ms) {
#define SAVE_WAIT_MS 5000
// Returns cache_save's answer.
static int save_with(uint32_t wait_ms) {
static int save_with(uint32_t wait_ms, int exiting) {
if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log();
if (P.ready && P.gpl) {
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; "
@@ -6865,7 +6906,7 @@ static int save_with(uint32_t wait_ms) {
}
gfx_shader_record_flush(1);
if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below
int saved = cache_save(wait_ms);
int saved = cache_save(wait_ms, exiting);
if (!saved && wait_ms)
rt_log("vk_pipeline: %s not saved: pipeline builds kept running for %u ms", P.play ? P.play_path : P.cache_path,
(unsigned)wait_ms);
@@ -6874,7 +6915,11 @@ static int save_with(uint32_t wait_ms) {
// The save at exit and after the shader preparation: it waits up to SAVE_WAIT_MS for a moment with no build running.
void vk_pipeline_save(void) {
save_with(SAVE_WAIT_MS);
save_with(SAVE_WAIT_MS, 0);
}
void vk_pipeline_save_exit(void) {
save_with(SAVE_WAIT_MS, 1);
}
// The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it
@@ -6903,7 +6948,7 @@ static void autosave_once(void) {
if (!plat_mutex_trylock(&cache_rw)) return;
plat_mutex_unlock(&cache_rw);
}
if (save_tried(&sr, n, now, save_with(0) > 0))
if (save_tried(&sr, n, now, save_with(0, 0) > 0))
rt_log("vk_pipeline: the pipeline cache was not saved in %d tries in a row; the next try is made when a new "
"pipeline is built, or at exit", SAVE_RETRIES + 1);
}

View File

@@ -0,0 +1,173 @@
"""Windows: exercise exit snapshots during builds, growing cache data, and the conservative other-driver path."""
from pathlib import Path
import subprocess
import sys
root = Path(__file__).resolve().parents[1]
source = (root / 'runtime/vk_pipeline.c').read_text()
def function(signature):
start = source.index(signature)
brace = source.index('{', start)
depth = 1
end = brace + 1
while depth:
depth += (source[end] == '{') - (source[end] == '}')
end += 1
return source[start:end]
harness = r'''
#include <windows.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
typedef SRWLOCK PlatMutex;
typedef void *VkPipelineCache;
typedef int VkResult;
#define VK_SUCCESS 0
#define VK_INCOMPLETE 5
typedef struct { struct { unsigned vendorID; } props; } VkCtx;
static VkCtx context = {{0x8086}};
static VkCtx *vk_ctx(void) { return &context; }
static size_t mock_size = 4;
static int grow_once, grow_always, query_fail;
static PlatMutex cache_rw = SRWLOCK_INIT;
static volatile int32_t cache_saving_at_exit;
static volatile LONG active, snapshots;
static HANDLE old_in, release_old, new_in, file_in, release_file;
static int block_file, fail_file, writes, save_result;
static struct {
VkPipelineCache cache, play;
int device, fold, feedback;
const char *cache_path, *play_path;
uint64_t saved[2], cache_loaded, play_loaded;
} P;
static void plat_mutex_lock_shared(PlatMutex *m) { AcquireSRWLockShared(m); }
static void plat_mutex_unlock_shared(PlatMutex *m) { ReleaseSRWLockShared(m); }
static int plat_mutex_trylock(PlatMutex *m) { return TryAcquireSRWLockExclusive(m); }
static int plat_mutex_trylock_shared(PlatMutex *m) { return TryAcquireSRWLockShared(m); }
static void plat_mutex_unlock(PlatMutex *m) { ReleaseSRWLockExclusive(m); }
static void plat_sleep_ms(uint32_t ms) { Sleep(ms); }
static uint64_t plat_now_ms(void) { return GetTickCount64(); }
static void rt_log(const char *fmt, ...) { (void)fmt; }
static void fb_log(const char *when) { (void)when; assert(!cache_saving_at_exit); }
static void plat_file_delete(const char *path) { (void)path; }
static int plat_file_write_atomic(const char *path, const void *data, size_t size) {
(void)path;
assert(!cache_saving_at_exit); // admission must reopen before IO and diagnostics
assert(size == mock_size && !memcmp(data, "test", 4));
writes++;
if (block_file) {
SetEvent(file_in);
assert(WaitForSingleObject(release_file, 2000) == WAIT_OBJECT_0);
}
return !fail_file;
}
static int vkGetPipelineCacheData(int device, VkPipelineCache cache, size_t *size, void *data) {
(void)device; (void)cache;
if (context.props.vendorID != 0x10de) assert(!active);
InterlockedIncrement(&snapshots);
if (!data && query_fail) return -1;
if (data && (grow_once || grow_always)) {
grow_once = 0; mock_size = *size + 100; return VK_INCOMPLETE;
}
if (data) { assert(*size >= mock_size); memcpy(data, "test", 4); }
*size = mock_size;
return VK_SUCCESS;
}
'''
harness += function('static void cache_build_enter(void)') + '\n'
harness += function('static int cache_save_one(int which, uint32_t wait_ms, int exiting)') + '\n'
harness += r'''
static DWORD WINAPI builder(void *old) {
cache_build_enter();
InterlockedIncrement(&active);
SetEvent(old ? old_in : new_in);
if (old) assert(WaitForSingleObject(release_old, 2000) == WAIT_OBJECT_0);
InterlockedDecrement(&active);
plat_mutex_unlock_shared(&cache_rw);
return 0;
}
static DWORD WINAPI saver(void *unused) {
(void)unused;
save_result = cache_save_one(0, 2000, 1);
return 0;
}
static void join(HANDLE t) { assert(WaitForSingleObject(t, 2000) == WAIT_OBJECT_0); CloseHandle(t); }
int main(void) {
P.cache = (void *)1; P.cache_path = "unused"; P.play_path = "unused-play"; P.feedback = 1;
old_in = CreateEventW(NULL, TRUE, FALSE, NULL);
release_old = CreateEventW(NULL, TRUE, FALSE, NULL);
new_in = CreateEventW(NULL, TRUE, FALSE, NULL);
file_in = CreateEventW(NULL, TRUE, FALSE, NULL);
release_file = CreateEventW(NULL, TRUE, FALSE, NULL);
HANDLE old = CreateThread(NULL, 0, builder, (void *)1, 0, NULL);
assert(WaitForSingleObject(old_in, 2000) == WAIT_OBJECT_0);
block_file = 1;
HANDLE saving = CreateThread(NULL, 0, saver, NULL, 0, NULL);
uint64_t until = GetTickCount64() + 2000;
while (!__atomic_load_n(&cache_saving_at_exit, __ATOMIC_ACQUIRE)) {
assert(GetTickCount64() < until); Sleep(1);
}
HANDLE next = CreateThread(NULL, 0, builder, NULL, 0, NULL);
assert(WaitForSingleObject(new_in, 20) == WAIT_TIMEOUT);
SetEvent(release_old);
assert(WaitForSingleObject(file_in, 2000) == WAIT_OBJECT_0);
assert(WaitForSingleObject(new_in, 2000) == WAIT_OBJECT_0); // IO does not hold admission closed
SetEvent(release_file);
join(old); join(next); join(saving);
assert(save_result == 1 && snapshots == 2 && writes == 1 && !cache_saving_at_exit);
// Ordinary autosave does not wait behind an active build or close admission.
cache_build_enter();
assert(cache_save_one(0, 0, 0) == 0 && !cache_saving_at_exit);
// A final save times out on a genuinely long driver call and restores admission.
assert(cache_save_one(0, 1, 1) == 0 && !cache_saving_at_exit);
plat_mutex_unlock_shared(&cache_rw);
cache_build_enter(); plat_mutex_unlock_shared(&cache_rw);
block_file = 0;
assert(cache_save_one(0, 100, 1) == 1 && writes == 1); // unchanged data is not rewritten
P.saved[0] = 0; fail_file = 1;
assert(cache_save_one(0, 100, 1) == -1 && !cache_saving_at_exit && P.saved[0] == 0);
// NVIDIA exit reads can finish while a build remains active; normal autosaves retain the conservative path.
fail_file=0;context.props.vendorID=0x10de;P.saved[0]=0;
ResetEvent(old_in);ResetEvent(release_old);
old=CreateThread(NULL,0,builder,(void *)1,0,NULL);
assert(WaitForSingleObject(old_in,2000)==WAIT_OBJECT_0);
assert(active==1);
int before=writes;
assert(cache_save_one(0,100,1)==1 && writes==before+1 && active==1 && !cache_saving_at_exit);
assert(cache_save_one(0,0,0)==0 && active==1); // no gameplay autosave behavior change
SetEvent(release_old);join(old);
// Cache growth retries a fresh size query, and a partial result never replaces an existing file.
P.saved[0]=0;grow_once=1;
assert(cache_save_one(0,100,1)==1 && P.saved[0]==mock_size);
before=writes;
assert(cache_save_one(0,100,1)==1 && writes==before);
uint64_t saved=P.saved[0];mock_size++;grow_always=1;
assert(cache_save_one(0,100,1)==-1 && writes==before && P.saved[0]==saved);
grow_always=0;query_fail=1;
assert(cache_save_one(0,100,1)==-1 && writes==before && P.saved[0]==saved);
query_fail=0;
// A merge owns exclusive access: even the concurrent snapshot must not race it.
assert(plat_mutex_trylock(&cache_rw));
assert(cache_save_one(0,0,1)==0 && writes==before);
plat_mutex_unlock(&cache_rw);
puts("Exit snapshot: active builds, merge exclusion, cache growth/retry, partial/error preservation and fallback passed");
return 0;
}
'''
out = root / 'build/test/cache-exit-snapshot'
out.mkdir(parents=True, exist_ok=True)
(out / 'check.c').write_text(harness)
cc = sys.argv[1] if len(sys.argv) > 1 else 'clang'
subprocess.run([cc, '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', str(out/'check.c'),
'-o', str(out/'check.exe')], check=True)
subprocess.run([str(out/'check.exe')], check=True, timeout=15)