Try a pipeline cache save again when it did not happen, up to six times at the save interval.

A save that found no memory for its copy, could not write its file or met a build that had just begun was not tried again until the next new pipeline was built, so a run that built none and then ended without its orderly exit lost what it had compiled since the last file. The autosave thread now repeats the try, and the same bound holds for the save owed after the play file was folded in, which before was tried at every interval for as long as it failed. tests/save_retry_check.py checks the rule.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 05:25:38 -05:00
parent d073d80e70
commit f9219180b9
5 changed files with 175 additions and 15 deletions

View File

@@ -405,6 +405,10 @@ value no machine has free. What the builders do when they are refused (`warm_bui
`lw_worker`) shows in a run's log: start the game with `build_free_mb` set above the machine's free commit and read the
`vk_pipeline:` lines that name the commit headroom.
A change to when the pipeline caches are saved in play (`runtime/vk_save_retry.h`, `runtime/vk_pipeline.c`
`autosave_once`) passes `python tests\save_retry_check.py`, which builds the header alone: a save for new content, none
without, and after a save that did not happen a bounded row of tries at the save interval.
A change to how FSR 3.1's pipelines reach the pipeline caches (`runtime/vk_ffx.cpp` `vk_ffx_proc_addr`,
`runtime/vk_pipeline.c` `vk_pipeline_create_foreign`) passes `python tests\ffx_cache_check.py <game exe>`. It runs
`--probe-ffx-cache` (one upscaling context on the device; no window, no game) twice on pipeline cache files of its own

View File

@@ -52,6 +52,7 @@
#include "vk_build_gate.h"
#include "vk_cache_file.h"
#include "vk_gpa.h"
#include "vk_save_retry.h"
#include "vk_warm_hint.h"
#define LOG_ONCE(...) do { static volatile int32_t once_; if (!plat_xchg32(&once_, 1)) rt_log(__VA_ARGS__); } while (0)
@@ -6403,14 +6404,15 @@ static int seed_pipelines(const char *dir) {
while (K.left) plat_cond_wait(&done_cv, &P.lock); // those a pool worker took
uint32_t built = K.built - built0, failed = K.failed - failed0;
plat_mutex_unlock(&P.lock);
int saved = cache_save(5000); // as a save at exit: it waits that long for a moment with no build running
int saved = cache_save(5000) > 0; // as a save at exit: it waits that long for a moment with no build running
plat_file_delete(mark);
char msg[640];
snprintf(msg, sizeof msg, "prepare-pipelines: %zu listed pipelines: %u built (%u mesh), %u failed, %u the same as one "
"built before, %u without a shader this game version or its shader cache has, %u vertex pipelines left to "
"their first draw (NX2NAVI_PREPARE_PIPELINES=%s), %u mesh pipelines this GPU cannot draw; on %d threads in "
"%.0f ms; pipeline cache %s", n, built, mesh, failed, known, dropped, other, all ? "all" : "mesh", no_mesh,
nt ? nt : 1, now_ms() - t0, saved ? "saved" : "not saved (builds kept running)");
nt ? nt : 1, now_ms() - t0,
saved ? "saved" : "not saved (builds kept running, or its file could not be written)");
printf("%s\n", msg);
fflush(stdout);
rt_log("vk_pipeline: %s", msg);
@@ -6718,7 +6720,8 @@ static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shu
// cache had held it: three saves of 358.1 MB in one run that added 0.3 MB (2026-11-11).
// The lock is only ever tried, never waited for: a waiting exclusive request would hold back every new build
// (a shared request queues behind it), so a pending save could stall the replay. wait_ms: how long to keep trying
// while builds run (0: one try). Returns 0 when no build-free moment came, 1 otherwise (saved, or nothing to save).
// while builds run (0: one try). Returns 0 when no build-free moment came, -1 when the data could not be copied or
// its file not written (the log says which), 1 otherwise (saved, or nothing to save).
static int cache_save_one(int which, uint32_t wait_ms) {
VkPipelineCache cache = which ? P.play : P.cache;
const char *path = which ? P.play_path : P.cache_path;
@@ -6735,12 +6738,14 @@ static int cache_save_one(int which, uint32_t wait_ms) {
int got = sized && !same && (data = malloc(size)) && vkGetPipelineCacheData(P.device, cache, &size, data) == VK_SUCCESS;
plat_mutex_unlock(&cache_rw);
static uint64_t skipped[2];
int failed = sized && !same && !got;
if (same) skipped[which]++;
if (sized && !same && !got) rt_log("vk_pipeline: %s not saved: no copy of its %.1f MB", path, size / 1048576.0);
if (failed) rt_log("vk_pipeline: %s not saved: no copy of its %.1f MB", path, size / 1048576.0);
if (got) {
uint64_t t0 = plat_now_ms();
if (!plat_file_write_atomic(path, data, size)) {
rt_log("vk_pipeline: cannot write %s (%.1f MB)", path, size / 1048576.0);
failed = 1;
} else {
uint64_t loaded = which ? P.play_loaded : P.cache_loaded;
rt_log("vk_pipeline: %s saved: %.1f MB (%+.1f MB since the boot) written in %llu ms; %llu saves before it were "
@@ -6761,9 +6766,10 @@ static int cache_save_one(int which, uint32_t wait_ms) {
}
}
free(data);
return 1;
return failed ? -1 : 1;
}
// Both caches' saves (cache_save_one): 1 when each is done, 0 when one found no build-free moment, -1 when one failed.
static int cache_save(uint32_t wait_ms) {
if (!P.ready || !P.cache || !P.cache_path[0]) return 1;
// Not under P.lock: holding P.lock through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped
@@ -6773,9 +6779,9 @@ static int cache_save(uint32_t wait_ms) {
plat_mutex_unlock(&save_lock);
return 1;
}
int ok = 1;
int ok = 1, r;
if (!P.play || P.fold) ok = cache_save_one(0, wait_ms);
if (P.play) ok = cache_save_one(1, wait_ms) && ok;
if (P.play && (r = cache_save_one(1, wait_ms)) < ok) ok = r; // the worse of the two
plat_mutex_unlock(&save_lock);
return ok;
}
@@ -6783,7 +6789,8 @@ static int cache_save(uint32_t wait_ms) {
// Builds still running after this long at a save that waits (exit, the shader preparation) leave the file as it was.
#define SAVE_WAIT_MS 5000
static void save_with(uint32_t wait_ms) {
// Returns cache_save's answer.
static int save_with(uint32_t wait_ms) {
if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log();
if (P.ready && P.gpl) {
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; "
@@ -6838,9 +6845,11 @@ static void save_with(uint32_t wait_ms) {
}
gfx_shader_record_flush(1);
if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below
if (!cache_save(wait_ms) && wait_ms)
int saved = cache_save(wait_ms);
if (!saved && wait_ms)
rt_log("vk_pipeline: %s not saved: pipeline builds kept running for %u ms", P.play ? P.play_path : P.cache_path,
(unsigned)wait_ms);
return saved;
}
// The save at exit and after the shader preparation: it waits up to SAVE_WAIT_MS for a moment with no build running.
@@ -6849,9 +6858,11 @@ void vk_pipeline_save(void) {
}
// The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it
// runs: at most every 20 s, and only after new pipelines were built. New pipeline keys are appended every 2 s.
// runs: at most every 20 s, and only after new pipelines were built or while a save that did not happen is still owed
// (vk_save_retry.h). New pipeline keys are appended every 2 s.
static void autosave_once(void) {
static uint64_t last_n, last_ms, last_keys;
static uint64_t last_keys;
static SaveRetry sr;
if (!P.ready) return;
uint64_t now = plat_now_ms();
if (now - last_keys >= 2000) {
@@ -6861,19 +6872,20 @@ static void autosave_once(void) {
cu_flush();
}
gfx_shader_record_flush(0);
if (now - last_ms < 20000) return;
if (now - sr.at_ms < SAVE_EVERY_MS) return;
VkPipeStats s;
vk_pipeline_stats(&s);
uint64_t n = s.graphics + s.compute + cache_merges; // a merge comes after its pipeline was counted (bg_main)
if (n == last_n && !P.fold) return; // P.fold: the play file merged at the start is to be written
if (!save_due(&sr, n, now, P.fold)) return; // P.fold: the play file merged at the start is to be written
// Only at a moment with no build running (cache_save): while one runs the save waits for the next tick, 500 ms
// on, before it logs or flushes anything, so a busy stretch does not repeat the statistics lines every tick.
if (P.cache && P.cache_path[0]) {
if (!plat_mutex_trylock(&cache_rw)) return;
plat_mutex_unlock(&cache_rw);
}
last_n = n, last_ms = now;
save_with(0);
if (save_tried(&sr, n, now, save_with(0) > 0))
rt_log("vk_pipeline: the pipeline cache was not saved in %d tries in a row; the next try is made when a new "
"pipeline is built, or at exit", SAVE_RETRIES + 1);
}
// On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to

39
runtime/vk_save_retry.h Normal file
View File

@@ -0,0 +1,39 @@
// When the autosave thread saves the pipeline caches (vk_pipeline.c autosave_once), and when it tries again after a
// save that did not happen. Own header so tests/save_retry_check.c checks it without Vulkan.
//
// A save is due when something new entered the caches since the last try. A try that did not save (no copy of the
// data for want of memory, a file that could not be written, a build that began between the look and the copy) used to
// be the end of it: the next save waited for the next new pipeline, and a run that built none, then ended without its
// orderly exit, lost what it had compiled since the last file. Now the try is repeated at the save interval, up to
// SAVE_RETRIES times in a row, so a folder that stays unwritable costs that many log lines and no more until a new
// pipeline is built. The same count bounds the save that is pending from the start (the play file folded into the
// main one, `pending`), which before was tried at every interval for as long as it failed.
#pragma once
#include <stdint.h>
#define SAVE_EVERY_MS 20000u // between two tries
#define SAVE_RETRIES 6 // tries in a row after one that did not save
typedef struct {
uint64_t n; // what had entered the caches at the last try
uint64_t at_ms; // when that try was made
uint32_t failed; // tries in a row that did not save
} SaveRetry;
// 1 when a save is to be tried now. n: the count of what entered the caches so far (it only grows); pending: a save
// is owed whatever n says.
static inline int save_due(const SaveRetry *s, uint64_t n, uint64_t now_ms, int pending) {
if (now_ms - s->at_ms < SAVE_EVERY_MS) return 0;
if (n != s->n) return 1;
return (pending || s->failed) && s->failed <= SAVE_RETRIES;
}
// A try was made at now_ms with n in the caches; saved: the files hold the caches' data now. Returns 1 when this was
// the last try in a row (the caller says so once): the next is made for new content.
static inline int save_tried(SaveRetry *s, uint64_t n, uint64_t now_ms, int saved) {
if (n != s->n) s->failed = 0; // new content starts a row of its own
s->n = n, s->at_ms = now_ms;
s->failed = saved ? 0 : s->failed + 1;
return s->failed == SAVE_RETRIES + 1;
}

65
tests/save_retry_check.c Normal file
View File

@@ -0,0 +1,65 @@
// runtime/vk_save_retry.h: when the autosave thread saves the pipeline caches and when it tries again after a save
// that did not happen (runtime/vk_pipeline.c autosave_once). Built and run by tests/save_retry_check.py; prints each
// failed check and exits 1 on any.
#include <stdio.h>
#include "vk_save_retry.h"
static int fails;
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
int main(void) {
uint64_t t = 1000000; // ms since the machine started, as the thread's clock gives them
// Nothing built, nothing owed: no save, however long the run.
SaveRetry s = {0};
CHECK(!save_due(&s, 0, t, 0), "a save with nothing new in the caches");
CHECK(!save_due(&s, 0, t + 3600000, 0), "a save after an hour with nothing new");
// A new pipeline: a save, and after it none until the next one; never two within the interval.
CHECK(save_due(&s, 1, t, 0), "no save for a new pipeline");
CHECK(!save_tried(&s, 1, t, 1), "a good save counted as the last try");
CHECK(!save_due(&s, 2, t + SAVE_EVERY_MS - 1, 0), "two saves within the interval");
CHECK(save_due(&s, 2, t + SAVE_EVERY_MS, 0), "no save for the next pipeline once the interval passed");
save_tried(&s, 2, t + SAVE_EVERY_MS, 1);
CHECK(!save_due(&s, 2, t + 10 * SAVE_EVERY_MS, 0), "a save again with nothing new after a good one");
// A save that did not happen is tried again at the interval with nothing new built, SAVE_RETRIES times; the
// last of them says so, and then nothing is tried until a new pipeline.
s = (SaveRetry){0};
uint64_t now = t;
CHECK(save_due(&s, 5, now, 0), "no first save");
CHECK(!save_tried(&s, 5, now, 0), "the first failed try counted as the last");
int tries = 0, last = 0;
for (int i = 0; i < 100 && !last; i++) {
CHECK(!save_due(&s, 5, now + SAVE_EVERY_MS - 1, 0), "a retry within the interval");
now += SAVE_EVERY_MS;
if (!save_due(&s, 5, now, 0)) break;
tries++;
last = save_tried(&s, 5, now, 0);
}
CHECK(tries == SAVE_RETRIES && last, "%d retries after a failed save, %d wanted, the last one said: %d", tries,
SAVE_RETRIES, last);
CHECK(!save_due(&s, 5, now + 100 * SAVE_EVERY_MS, 0), "a try after the last of the row with nothing new");
CHECK(save_due(&s, 6, now + SAVE_EVERY_MS, 0), "no try for a new pipeline after the row ended");
CHECK(!save_tried(&s, 6, now + SAVE_EVERY_MS, 0), "new content did not start a row of its own");
CHECK(save_due(&s, 6, now + 2 * SAVE_EVERY_MS, 0), "no retry in the new row");
// A retry that saves ends the row: nothing more until something new.
s = (SaveRetry){0};
save_tried(&s, 3, t, 0);
CHECK(save_due(&s, 3, t + SAVE_EVERY_MS, 0), "no retry after one failed save");
CHECK(!save_tried(&s, 3, t + SAVE_EVERY_MS, 1), "a good retry counted as the last try");
CHECK(!save_due(&s, 3, t + 2 * SAVE_EVERY_MS, 0), "a save again after a good retry");
// A save owed from the start (the play file folded in) is tried with nothing built, and bounded the same way.
s = (SaveRetry){0};
CHECK(save_due(&s, 0, t, 1), "no save for the one owed from the start");
now = t, tries = 1;
save_tried(&s, 0, now, 0);
while (tries < 100 && save_due(&s, 0, now += SAVE_EVERY_MS, 1)) tries++, save_tried(&s, 0, now, 0);
CHECK(tries == SAVE_RETRIES + 1, "%d tries of an owed save that keeps failing, %d wanted", tries, SAVE_RETRIES + 1);
printf("save_retry_check: %s\n", fails ? "FAILED" : "passed");
return fails ? 1 : 0;
}

40
tests/save_retry_check.py Normal file
View File

@@ -0,0 +1,40 @@
"""Build and run tests/save_retry_check.c: runtime/vk_save_retry.h, when the autosave thread saves the pipeline caches
and when it tries again after a save that did not happen (no copy of the data, a file that cannot be written).
Usage: save_retry_check.py
"""
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests' / 'diff'))
try:
from common import MSYS_BIN, compiler # noqa: E402
except ImportError: # a Python without the differential tests' packages: the same compiler choice without them
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
def compiler():
if os.environ.get('CC'):
return os.environ['CC']
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
def main():
work = ROOT / 'build' / 'test' / 'save_retry'
work.mkdir(parents=True, exist_ok=True)
exe = work / ('save_retry_check.exe' if os.name == 'nt' else 'save_retry_check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
str(ROOT / 'tests' / 'save_retry_check.c'), '-o', str(exe)], env=env)
return subprocess.run([str(exe)], env=env).returncode
if __name__ == '__main__':
sys.exit(main())