Build the shared pipeline list before play on every start, and build every listed pipeline at install.

A new install met every pipeline in play: in Hyrule Field one 600-frame stretch of a first visit had 1,769 draws wait 461 ms in all for pipelines, the longest frame 88 ms. The boot now reads seed-pipeline-keys.bin, the release's list, as recorded pipelines after the install's own (keys_open_seed, NX2NAVI_PIPE_SEED, default 1): their pairs go into the vertex output unions and the prewarm builds them as it builds the install's. The preparation builds every listed pipeline (NX2NAVI_PREPARE_PIPELINES now defaults to all), with the unions of the same list, so the first boot of a new install finds them in the caches. The list's mesh pipelines are left out at the boot on a GPU without mesh shaders, as the preparation leaves them out. Nothing of the list goes into pipeline-keys.bin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-12 06:19:39 -05:00
parent fbd4067460
commit 6ccd1e224d
5 changed files with 106 additions and 27 deletions

View File

@@ -58,7 +58,7 @@ static const char *const known[] = {
"MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOUSE_TRACE", "MOVIE_HOST", "MOVIE_PACE", "MOVIE_CLOCK", "TEST_HITCH",
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
"MV_SURVEY",
"ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_RELEASED_MAX", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC",
"ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_RELEASED_MAX", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC",
"PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW",
"POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP",
"PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC",
@@ -132,7 +132,7 @@ static const struct { const char *key, *section; } sections[] = {
{"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"},
{"PIPE_FRAME_WAIT_MS", "performance"}, {"PIPE_SKIP_STEADY", "performance"},
{"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"},
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"},
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PIPE_SEED", "performance"},
{"COMPUTE_KEEP", "performance"},
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
{"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_SPLIT_FILL", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
@@ -265,7 +265,7 @@ static const CfgMeta metas[] = {
{"PIPE_LINK_DRAW_MS", "3", "1..50", RESTART}, // vk_exec_draw.c link_draw_ms
{"PIPE_FRAME_WAIT_MS", "50", "0..500", RESTART}, // vk_exec_draw.c frame_wait
{"PIPE_SKIP_STEADY", "1", "0|1", RESTART}, // vk_exec.c steady_on
{"PREPARE_PIPELINES", "mesh", "0|mesh|all", RESTART}, // vk_pipeline.c seed_pipelines
{"PREPARE_PIPELINES", "all", "0|mesh|all", RESTART}, // vk_pipeline.c seed_pipelines
// vk_device.c: task and mesh programs through compute passes (vk_exec_draw.c "mesh fallback"); auto: on a device
// without mesh shaders. The megabytes are the buffers the passes write (vk_mesh_fb.c).
{"MESH_FALLBACK", "auto", "auto|0|1", RESTART}, {"MESH_FALLBACK_MB", "128", "16..4096", RESTART},
@@ -279,7 +279,8 @@ static const CfgMeta metas[] = {
{"GATE_WAIT_MS", "10", "0..100", RESTART}, // vk_exec.c vk_exec_gate_wait
{"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART},
{"LIB_WARM", "1", "0|1|force", RESTART}, // vk_pipeline.c lw_start: the driver's cache of stage libraries, warmed when cold
{"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "", "0|1", RESTART}, // vk_caps.c vk_caps_gpl: by the driver, off with AMD's for RDNA 2 and older
{"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PIPE_SEED", "1", "0|1", RESTART}, // vk_pipeline.c keys_open_seed
{"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "", "0|1", RESTART}, // vk_caps.c vk_caps_gpl: by the driver, off with AMD's for RDNA 2 and older
{"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART},
{"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART},
{"FFX_PIPE_CACHE", "1", "0|1", RESTART}, // vk_ffx.cpp vk_ffx_proc_addr: FSR 3.1's pipelines through the pipeline caches

View File

@@ -3762,8 +3762,73 @@ static void keys_flush(void) {
plat_mutex_unlock(&K.io);
}
// Reads the key file into K.file and its record hashes into the set. Caller holds P.lock (ensure_init).
static void keys_open_seed(void);
// Reads the key file into K.file and its record hashes into the set, then adds the shared list's records
// (keys_open_seed). Caller holds P.lock (ensure_init).
static void keys_open_own(void);
static void keys_open(void) {
keys_open_own();
if (K.path[0]) keys_open_seed();
}
static int seen_has(uint64_t h) { // caller holds P.lock
if (!K.seen_cap) return 0;
for (uint32_t j = (uint32_t)h & (K.seen_cap - 1); K.seen[j]; j = (j + 1) & (K.seen_cap - 1))
if (K.seen[j] == h) return 1;
return 0;
}
// The shared pipeline list as recorded pipelines (NX2NAVI_PIPE_SEED, default 1; 0: this install's own records only).
// seed-pipeline-keys.bin next to the game holds the list's records (tools/prepare_shaders.py writes it). Before, the boot
// built only what this install drew, so a new install met every pipeline in play: in Hyrule Field one 600-frame
// stretch of a first visit had 1,769 draws wait 461 ms in all for their pipelines, the longest frame 88 ms. The list's
// records now follow the install's own in K.file, so the prewarm counts their pairs in the vertex output unions (and the
// pipelines it builds from them are the ones draws ask for) and builds them as it builds the install's: the boot for at
// most PREWARM_PIPE_MS, the pool workers the rest. The preparation builds the same list into the caches
// (seed_pipelines, NX2NAVI_PREPARE_PIPELINES=all), so that boot finds them there. They are not added to the set or the
// file: pipeline-keys.bin keeps what this install drew, and a pipeline a draw builds that the prewarm did not still goes
// into it. A list in another record layout is left out, as the preparation leaves it.
static void keys_open_seed(void) {
if (!cfg_bool("NX2NAVI_PIPE_SEED")) return;
char path[sizeof K.path];
snprintf(path, sizeof path, "%s", K.path);
char *slash = strrchr(path, '\\'), *fslash = strrchr(path, '/');
if (fslash > slash) slash = fslash;
snprintf(slash ? slash + 1 : path, sizeof path - (size_t)(slash ? slash + 1 - path : 0), "seed-pipeline-keys.bin");
FILE *f = fopen(path, "rb");
if (!f) return;
KeysHeader h;
fseek(f, 0, SEEK_END);
long len = ftell(f);
fseek(f, 0, SEEK_SET);
size_t n = len > (long)sizeof h ? ((size_t)len - sizeof h) / sizeof(KeyRec) : 0;
KeyRec *seed = NULL;
int ok = len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8) &&
h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec) && n && (seed = malloc(n * sizeof *seed)) &&
fread(seed, sizeof *seed, n, f) == n;
fclose(f);
if (!ok) {
if (n) rt_log("vk_pipeline: %s is a list for another runtime version (its record layout differs); left out", path);
free(seed);
return;
}
KeyRec *all = realloc(K.file, (K.file_n + n) * sizeof(KeyRec));
if (!all) {
free(seed);
return;
}
K.file = (uint8_t *)all;
size_t added = 0;
for (size_t i = 0; i < n; i++) // after the install's own: those met first in play keep coming first
if (!seen_has(rec_hash(&seed[i]))) all[K.file_n + added++] = seed[i];
K.file_n += added;
free(seed);
rt_log("vk_pipeline: %s: %zu listed pipelines, %zu of them not recorded by this install; built with its own "
"(NX2NAVI_PIPE_SEED)", path, n, added);
}
static void keys_open_own(void) {
if (!A.on || !P.cache_path[0]) return;
snprintf(K.path, sizeof K.path, "%s", P.cache_path);
char *slash = strrchr(K.path, '\\'), *fslash = strrchr(K.path, '/');
@@ -4796,6 +4861,12 @@ static void prewarm_run(void) {
}
VkPipeState s = r[i].s;
s.vs = vs, s.fs = fs, s.ts = ts;
// a mesh pipeline of the shared list (keys_open_seed) on a GPU without mesh shaders, which draws them through
// the mesh fallback: none to build, as the preparation skips it (seed_pipelines)
if (is_mesh(&s) && !vk_ctx()->has_mesh_shader) {
dropped++;
continue;
}
GKey k;
gkey_fill(&s, &k);
if (!is_mesh(&s)) { // the stage libraries lb_start prebuilds are for vertex pipelines
@@ -6527,18 +6598,17 @@ static void lb_warm_prepared(const char *dir) {
// tools/prepare_shaders.py writes the list's key records there, in the layout of pipeline-keys.bin, after putting the
// shader variants they name into the seed. The preparation builds them once, after the seed is translated: each
// pipeline goes through the pipeline cache and so into pipeline-cache.bin and the driver's own disk cache, and is
// dropped with the process. Nothing is added to this folder's pipeline-keys.bin: the boot keeps building only what
// this install drew, and a listed pipeline costs no start-up time or video memory until a draw meets it. That draw
// then finds its pipeline in the caches: a few milliseconds for a mesh pipeline, which has no stage libraries and
// otherwise leaves its draws out of the picture for its whole compile (0.2 to 1.1 s each on a first visit of a forest,
// 18 of them).
// NX2NAVI_PREPARE_PIPELINES: mesh (default) builds the list's mesh pipelines only; all, every record; 0, none. A
// vertex pipeline is linked from its prepared stage libraries in 2 to 6 ms when a draw misses it, and its module
// depends on the vertex output union of the pairs this install has recorded (vs_outs), so one built here is not
// certain to be the one a draw asks for. Without the graphics pipeline library (vk_caps_gpl: AMD's Windows driver for
// RDNA 2 and older) neither holds: a vertex pipeline a draw misses is compiled whole while that draw is left out, and
// its vertex outputs come from its own pair (no independent layouts), so a listed one is the one the draw asks for.
// There mesh builds every record too.
// dropped with the process. Nothing is added to this folder's pipeline-keys.bin; every boot then counts the list as
// recorded pipelines beside the install's own (keys_open_seed) and builds them before play, from those caches.
// NX2NAVI_PREPARE_PIPELINES: all (default) builds every record; mesh, the list's mesh pipelines only (and every record
// without the graphics pipeline library); 0, none. A vertex pipeline's module depends on the vertex output union of
// the pairs known (vs_outs): here the list's, as at the first boot of a new install, which reads the list and none of
// its own yet, so a vertex pipeline built here is the one that boot asks for. An install that has drawn more pairs
// since may need some of them again with wider outputs; the boot builds those. Without the graphics pipeline library
// (vk_caps_gpl: AMD's Windows driver for RDNA 2 and older) a vertex pipeline's outputs come from its own pair (no
// independent layouts), so a listed one always is the one a draw asks for. A mesh pipeline, which has no stage
// libraries, otherwise leaves its draws out of the picture for its whole compile (0.2 to 1.1 s each on a first visit
// of a forest, 18 of them).
// A driver whose compiler crashes on one of these would crash at every preparation: seed-pipelines-running.txt marks
// the step, and a run that finds the marker of a run that died skips the step for good on this folder (the pipelines
// are then built at their first draw, as without a list) until the marker is deleted.