Keep serving the previous translator's shader translations after a translator update, and translate them again in the background.

Every shader cache key names the translator, so a release with a new translator had no cached translation at all:
the launcher re-translated about 42,000 programs before play, and play waited for any program not prepared.

The cache now lists the translator identities it holds translations of in shader-cache/identities.txt, newest first.
A shader the current translator has no translation of uses a previous one's, looked up with that identity mixed with
this machine's host options and read with the usual checks; it keeps that translation for the rest of the session, so
no pipeline is built twice. After the boot prewarm a background pass translates every stored input that has a previous
translator's entry again under the current one, on its own translator child at idle priority that waits while the
game translates. A program the new translator refuses keeps its old translation and is logged. Once the pass has gone
through every such input, the old entries of what it covered are dropped: loose files are deleted, and a pruned copy
of shader-cache.pack replaces the pack at the next start when the pack is still the one it was copied from.
--prepare-shaders still translates everything under the current translator. NX2NAVI_SHADER_CARRY=0 keeps the earlier
behaviour.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-09 10:38:09 -05:00
parent 3ccc33b3e8
commit 10ea0a2eb2
9 changed files with 800 additions and 44 deletions

View File

@@ -347,6 +347,9 @@ static void prewarm_main(void *arg) {
int cpus = (int)plat_cpu_usable();
gfx_shader_prepare_recorded(cpus, cpus / 2 > 16 ? 16 : cpus / 2 < 1 ? 1 : cpus / 2);
}
// Translations of a previous translator are made again in the background from here on, after the boot's own
// translations. Not for a --shot image: which translation a shader gets would then depend on the pass's timing.
if (!rt_config()->shot) gfx_shader_carry_start();
return;
}

View File

@@ -56,7 +56,7 @@ static const char *const known[] = {
"REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS",
"REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
"SHADER_CACHE_FLUSH", "SHADER_MEMO", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR",
"SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR",
"SVC_SLOW_MS",
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS",
"SWKBD_ANSWER",
@@ -110,7 +110,7 @@ static const struct { const char *key, *section; } sections[] = {
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"},
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
{"TEX_ALIAS", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
{"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
{"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
@@ -204,6 +204,7 @@ static const CfgMeta metas[] = {
{"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART},
{"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, {"PIPE_BINARIES", "0", "0|1", RESTART}, {"ONESHOT_WAIT_MS", "30000", "0..60000", RESTART}, {"ONESHOT_BUDGET", "0", "0|1", RESTART}, {"PIPE_BINARIES_MAX_MB", "1024", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART},
{"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART},
{"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again
// textures and memory
{"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART},
{"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART},

View File

@@ -62,8 +62,13 @@ const uint8_t *gfx_shader_input_id(const GfxShader *s); // the 32-byte input
// By input id from memory or disk; xlate 1 also translates it from its stored inputs. NULL if absent.
const GfxShader *gfx_shader_load_input(const uint8_t iid[32], int xlate);
// Boot prewarm: out[i] = gfx_shader_load_input(iids[i], 1) on `threads` threads with up to `procs` translators, then
// every other stored input not cached under this translator is translated. Logs progress and the total.
// every other stored input not cached under this translator is translated (one a previous translator's translation
// serves is left to gfx_shader_carry_start's pass). Logs progress and the total.
void gfx_shader_prewarm(const uint8_t (*iids)[32], size_t n, const GfxShader **out, int threads, int procs);
// After the boot prewarm of a game run (boot.c): when the shader cache holds translations of a previous translator
// (NX2NAVI_SHADER_CARRY), starts the background pass that translates them again under the current one for the next
// start and then drops the old ones (gfx_shader.c, "Carry").
void gfx_shader_carry_start(void);
// How far the running (or last) gfx_shader_prewarm is: shaders done of total, and how many of them it translated (the
// rest were cached). Any thread; the numbers lag the workers by at most 100 ms.
void gfx_shader_prewarm_progress(int64_t *done, int64_t *total, int64_t *translated);

File diff suppressed because it is too large Load Diff

View File

@@ -1835,6 +1835,10 @@ void plat_child_stop(PlatChild *c) {
free(c);
}
void plat_child_lower(PlatChild *c) { // a higher nice value needs no privilege
if (c) setpriority(PRIO_PROCESS, (id_t)c->pid, 19);
}
// A child that died makes the write fail with EPIPE; SIGPIPE is ignored for the process (main.c), as Windows has none.
int plat_child_write(PlatChild *c, const void *data, size_t n) { return write_all(c->in, data, n); }

View File

@@ -459,6 +459,9 @@ typedef struct PlatChild PlatChild;
// Children start one at a time so none inherits another pipe ends.
PlatChild *plat_child_start(const char *exe, const char *arg, uint32_t *err);
void plat_child_stop(PlatChild *c); // kills the child and frees c
// Runs the child below every normal-priority process (Windows: the idle priority class; elsewhere nice 19), for work
// that must not slow the game down (gfx_shader.c, the carry pass).
void plat_child_lower(PlatChild *c);
int plat_child_write(PlatChild *c, const void *data, size_t n); // 1 when all n bytes were written
#define PLAT_CHILD_READ_MS 120000 // plat_child_read gives up on a child that sends nothing for this long
long plat_child_read(PlatChild *c, void *buf, size_t cap); // bytes read (at least 1); 0 or negative: pipe broke or timed out

View File

@@ -1913,6 +1913,10 @@ void plat_child_stop(PlatChild *c) {
free(c);
}
void plat_child_lower(PlatChild *c) {
if (c) SetPriorityClass(c->proc, IDLE_PRIORITY_CLASS);
}
int plat_child_write(PlatChild *c, const void *data, size_t n) {
DWORD put = 0;
return WriteFile(c->in, data, (DWORD)n, &put, NULL) && put == n;