From 87f51f94a9d2f83b1d98b308dac82a7756dfd8b8 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Mon, 9 Nov 2026 14:34:45 -0500 Subject: [PATCH] Read shaders from the disk cache off the render thread past a small budget, and pack the cache on carried updates. A profiled traversal session had 9 of its 16 worst hitches (100 to 500 ms) inside the render thread's shader lookup, reading .res.json and .spv of shaders met for the first time: opening a loose file costs about 3 ms the first time on Windows. get_async_one now reads in place while those reads stay within 1/8 of the wall time (2 ms at once); past that three loader threads read them and the draws wait as for a translation. The perf line counts both. The launcher's updates carry the shaders over and never packed the cache, which held 44,000 loose translations in the test install. An update now packs it once it holds 1,000 loose ones, and the pack tool reads the loose files on 16 threads. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 5 ++ runtime/gfx.h | 2 + runtime/gfx_shader.c | 151 ++++++++++++++++++++++++++++++------- runtime/perf.c | 11 ++- tests/launcher_test.py | 24 ++++++ tools/make_game.py | 17 +++++ tools/pack_shader_cache.py | 36 +++++++-- 7 files changed, 210 insertions(+), 36 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 29fbf0c7..dd183d5e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -101,6 +101,11 @@ keeps per-core data without locks and relies on that, so the host models it (`ru translates, translates every stored program again for the next start, then deletes the old translations (a pruned copy of `shader-cache.pack` replaces the pack at the next start). A program the new translator refuses keeps its old translation. `--prepare-shaders` still translates everything with the current translator. +- Shader disk reads in play: a draw's shader that is in the disk cache but not in memory is read on the render thread + within a budget of 1/8 of the wall time (2 ms at once), and past it by three loader threads while its draws are + skipped as for a translation (`runtime/gfx_shader.c` `load_budget`). Opening a loose file is what costs (5.6 ms a + pair the first time on Windows), so an update that carries the shaders over packs the cache once it holds 1,000 + loose translations (`tools/make_game.py` `PACK_LOOSE_MIN`). - Mesh and task shaders run as `VK_EXT_mesh_shader`; indirect mesh draws read NVN's `{count, first}` record through a ring of Vulkan `{x, y, z}` records filled on the GPU. - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link diff --git a/runtime/gfx.h b/runtime/gfx.h index 50836592..2a1b7778 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -54,6 +54,8 @@ const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8 const GfxShaderAnswers *a); uint64_t gfx_shader_async_done(void); // translations finished by the background worker void gfx_shader_counts(uint64_t *keys, uint64_t *translated); // distinct keys in memory, translations done (NX2NAVI_TRACE=perf) +void gfx_shader_disk_counts(uint64_t *sync, uint64_t *sync_ns, uint64_t *bg); // disk cache reads on the render + // thread (count, ns) and by the loaders (gfx_shader.c load_budget) // Translator answers for this device (vk_caps.c): shaderFloat64, storage offset rounding, subgroup size per vs/fs/cs. void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals, int bda); diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 0d6c3b8f..a0356a92 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1904,12 +1904,53 @@ typedef struct XWait { static struct { PlatMutex lock; PlatCond work, done; // done: a job finished (gfx_shader_await) + PlatCond lwork; // a disk read was queued (load_worker) XJob *head, *tail; + XJob *lhead, *ltail; // disk reads past the render thread's budget (load_budget) XWait *waits[256]; int started; // 1 running, -1 no thread could be created -} X = {PLAT_MUTEX_INIT, PLAT_COND_INIT, PLAT_COND_INIT}; + int loaders; // load_worker threads running +} X = {PLAT_MUTEX_INIT, PLAT_COND_INIT, PLAT_COND_INIT, PLAT_COND_INIT}; static volatile int64_t n_async; +static volatile int64_t n_disk_sync, n_disk_sync_ns, n_disk_bg; // gfx_shader_disk_counts + +// Disk reads in get_async_one, the render thread's lookup. A shader that is in the disk cache but not in memory yet is +// read there while those reads stay within LOAD_SHARE of the wall time, at most LOAD_BURST_NS at once; past that a +// load_worker reads it, and its draws are skipped (or a one-shot draw waits) as for a shader being translated. Reading a +// pair of loose files the first time took 5.6 ms on Windows, against 0.08 ms once the system had them: the time is +// spent opening the files, not reading them. An area that needed 50 shaders not met before held one frame for 100 to 500 +// ms. A packed pair takes a fraction of a millisecond, so with a pack nearly all of them are still read in place. +#define LOAD_BURST_NS 2000000ll +#define LOAD_SHARE 8 // a budget of 1/8 of the wall time +#define LOADERS 3 +static PlatMutex load_lock = PLAT_MUTEX_INIT; +static int64_t load_tokens = LOAD_BURST_NS; // under load_lock +static uint64_t load_last; + +// Whether the render thread may read from the disk cache now. +static int load_budget(void) { + uint64_t now = plat_now_ns(); + plat_mutex_lock(&load_lock); + if (load_last) load_tokens += (int64_t)(now - load_last) / LOAD_SHARE; + if (load_tokens > LOAD_BURST_NS) load_tokens = LOAD_BURST_NS; + load_last = now; + int ok = load_tokens > 0; + plat_mutex_unlock(&load_lock); + return ok; +} + +static void load_spent(uint64_t ns) { + plat_mutex_lock(&load_lock); + load_tokens -= (int64_t)ns; + plat_mutex_unlock(&load_lock); + plat_inc64(&n_disk_sync); + __atomic_add_fetch(&n_disk_sync_ns, (int64_t)ns, __ATOMIC_RELAXED); +} + +void gfx_shader_disk_counts(uint64_t *sync, uint64_t *sync_ns, uint64_t *bg) { + *sync = (uint64_t)n_disk_sync, *sync_ns = (uint64_t)n_disk_sync_ns, *bg = (uint64_t)n_disk_bg; +} // Caller holds X.lock. static XWait **wait_slot(const uint8_t key[32]) { @@ -1938,6 +1979,67 @@ static void xlate_worker(void *arg) { } return; } + +// Reads the shaders get_async_one queued past its budget (load_budget), in get_one's disk order: the current key, then +// a previous translator's entry. One the disk cache lacks goes on to the translation queue, its key still waiting. +static void load_worker(void *arg) { + (void)arg; + for (;;) { + plat_mutex_lock(&X.lock); + while (!X.lhead) plat_cond_wait(&X.lwork, &X.lock); + XJob *j = X.lhead; + if (!(X.lhead = j->next)) X.ltail = NULL; + plat_mutex_unlock(&X.lock); + GfxShader *d = j->r.final ? disk_load(j->r.key, j->r.iid) : NULL; + const GfxShader *s = d ? mem_insert(d) : carry_find(j->r.code_hash, &j->a, j->stage, 1); + plat_mutex_lock(&X.lock); + if (s) { + XWait **w = wait_slot(j->r.key), *e = *w; + if (e) *w = e->next, free(e); + } else { + j->next = NULL; + if (X.tail) X.tail->next = j; + else X.head = j; + X.tail = j; + plat_cond_signal(&X.work); + } + plat_mutex_unlock(&X.lock); + if (!s) continue; + plat_cond_broadcast(&X.done); + plat_inc64(&n_hits), plat_inc64(&n_disk_bg); + free(j->control), free(j->code), free(j); + } +} + +// Caller holds X.lock: the translation workers and the disk readers, started at the first job. +static void start_workers(void) { + if (!X.started) { + int n = 0; + for (int i = 0; i < svc_runtime_limit(); i++) { + if (plat_thread_spawn(xlate_worker, NULL, "xlate", 0)) n++; + } + X.started = n ? 1 : -1; + if (n) rt_log("gfx_shader: graphics shaders are translated by %d background workers (NX2NAVI_XLATE_PROCS)", n); + else rt_log("gfx_shader: cannot start the translation workers; translating synchronously"); + for (int i = 0; n && i < LOADERS; i++) { + if (plat_thread_spawn(load_worker, NULL, "shload", 0)) __atomic_add_fetch(&X.loaders, 1, __ATOMIC_RELAXED); + } + } +} + +// Caller holds X.lock: the job of key to the front of its queue, so the worker takes it next (gfx_shader_await). +static void to_front(XJob **head, XJob **tail, const uint8_t key[32]) { + for (XJob *j = *head, *prev = NULL; j; prev = j, j = j->next) + if (!memcmp(j->r.key, key, 32)) { + if (prev) { + prev->next = j->next; + if (*tail == j) *tail = prev; + j->next = *head, *head = j; + } + return; + } +} + const GfxShader *gfx_shader_get_async(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, const GfxShaderAnswers *a, int *pending) { *pending = 0; @@ -1961,41 +2063,45 @@ static const GfxShader *get_async_one(Req r, int stage, const uint8_t *control, if (w) return *pending = !failed, NULL; paths(); if (!cache_dir[0]) return fail("no cache directory", NULL); - if (r.final && (s = disk_load(r.key, r.iid))) return plat_inc64(&n_hits), mem_insert(s); - if ((s = carry_find(r.code_hash, a, stage, 1))) return plat_inc64(&n_hits), s; + int read = load_budget() || __atomic_load_n(&X.loaders, __ATOMIC_RELAXED) == 0; // the disk here, or a loader + if (read) { + uint64_t t0 = plat_now_ns(); + GfxShader *d = r.final ? disk_load(r.key, r.iid) : NULL; + s = d ? mem_insert(d) : carry_find(r.code_hash, a, stage, 1); + load_spent(plat_now_ns() - t0); + if (s) return plat_inc64(&n_hits), s; + } XJob *j = calloc(1, sizeof *j); XWait *e = calloc(1, sizeof *e); uint8_t *ctl = malloc(CONTROL_SIZE), *cd = malloc(r.length); plat_mutex_lock(&X.lock); - if (!X.started) { - int n = 0; - for (int i = 0; i < svc_runtime_limit(); i++) { - if (plat_thread_spawn(xlate_worker, NULL, "xlate", 0)) n++; - } - X.started = n ? 1 : -1; - if (n) rt_log("gfx_shader: graphics shaders are translated by %d background workers (NX2NAVI_XLATE_PROCS)", n); - else rt_log("gfx_shader: cannot start the translation workers; translating synchronously"); - } + start_workers(); XWait **slot = wait_slot(r.key); int raced = *slot != NULL; // another thread queued it between the checks int queued = !raced && X.started == 1 && j && e && ctl && cd; + int load = queued && !read && X.loaders; // past the budget: a loader reads it first if (queued) { memcpy(ctl, control, CONTROL_SIZE), memcpy(cd, code, r.length); *j = (XJob){.stage = stage, .r = r, .control = ctl, .code = cd, .a = *a}; j->r.c1 = cd + (r.length - r.c1_len); memcpy(e->key, r.key, 32); *slot = e; - if (X.tail) X.tail->next = j; - else X.head = j; - X.tail = j; - plat_cond_signal(&X.work); + XJob **head = load ? &X.lhead : &X.head, **tail = load ? &X.ltail : &X.tail; + if (*tail) (*tail)->next = j; + else *head = j; + *tail = j; + plat_cond_signal(load ? &X.lwork : &X.work); } plat_mutex_unlock(&X.lock); if (queued) return *pending = 1, NULL; free(j), free(e), free(ctl), free(cd); if (raced) return *pending = 1, NULL; - return translate(&r, stage, control, code, a); // no worker or out of memory + if (!read) { // no worker or out of memory: everything here + GfxShader *d = r.final ? disk_load(r.key, r.iid) : NULL; + if ((s = d ? mem_insert(d) : carry_find(r.code_hash, a, stage, 1))) return plat_inc64(&n_hits), s; + } + return translate(&r, stage, control, code, a); } const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, @@ -2008,15 +2114,8 @@ const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8 if (!pending || !req_init(&r, stage, control, code, code_size, NULL, a)) return s; uint64_t p0 = perf_begin(); // the wait counts as the render thread's translation time plat_mutex_lock(&X.lock); - for (XJob *j = X.head, *prev = NULL; j; prev = j, j = j->next) - if (!memcmp(j->r.key, r.key, 32)) { // queued: to the front, the worker takes it next - if (prev) { - prev->next = j->next; - if (X.tail == j) X.tail = prev; - j->next = X.head, X.head = j; - } - break; - } + to_front(&X.lhead, &X.ltail, r.key); // queued: to the front, a worker takes it next + to_front(&X.head, &X.tail, r.key); XWait *w; while ((w = *wait_slot(r.key)) && !w->failed) plat_cond_wait(&X.done, &X.lock); plat_mutex_unlock(&X.lock); diff --git a/runtime/perf.c b/runtime/perf.c index 1a27ffef..f0fd0f34 100644 --- a/runtime/perf.c +++ b/runtime/perf.c @@ -564,10 +564,14 @@ void perf_present(uint64_t frame) { int64_t async = win[PERF_CNT_PIPE_ASYNC], skips = win[PERF_CNT_PIPE_SKIP], up = win[PERF_CNT_TEX_UP]; int64_t reup = win[PERF_CNT_TEX_REUP], sskip = win[PERF_CNT_SHADER_SKIP], waited = win[PERF_CNT_PIPE_WAIT]; int64_t oneshot = win[PERF_CNT_ONESHOT_WAIT]; - uint64_t shader_keys, shader_xlated; + uint64_t shader_keys, shader_xlated, disk[3]; + static uint64_t disk_prev[3]; gfx_shader_counts(&shader_keys, &shader_xlated); + gfx_shader_disk_counts(&disk[0], &disk[1], &disk[2]); + uint64_t dwin[3]; + for (int i = 0; i < 3; i++) dwin[i] = disk[i] - disk_prev[i], disk_prev[i] = disk[i]; if (n && span) - rt_log("perf: frame %llu: interval avg %.2f ms max %.2f ms (%.1f fps); render thread blocked: pacing %.0f%% fence %.0f%% drain %.0f%% pipeline build %.0f%% shader xlate %.0f%% texture %.0f%% (astc %.0f%%); pipelines built in background %lld, draws skipped pending %lld, waited %lld, waiting for a shader %lld, one-shot waited %lld; textures uploaded %lld re-uploaded %lld; slowest frame: xlate %.1f ms texture %.1f ms (astc %.1f) pipeline %.1f ms fence %.1f ms; shaders: %llu keys, %llu translated; render busy avg %.2f ms; replay avg %.2f ms; ab %llu/%llu; world %.1f s", + rt_log("perf: frame %llu: interval avg %.2f ms max %.2f ms (%.1f fps); render thread blocked: pacing %.0f%% fence %.0f%% drain %.0f%% pipeline build %.0f%% shader xlate %.0f%% texture %.0f%% (astc %.0f%%); pipelines built in background %lld, draws skipped pending %lld, waited %lld, waiting for a shader %lld, one-shot waited %lld; textures uploaded %lld re-uploaded %lld; slowest frame: xlate %.1f ms texture %.1f ms (astc %.1f) pipeline %.1f ms fence %.1f ms; shaders: %llu keys, %llu translated; render busy avg %.2f ms; replay avg %.2f ms; ab %llu/%llu; world %.1f s; shader disk reads: %llu on the render thread (%.1f ms), %llu by the loaders", (unsigned long long)frame, sum / 1e6 / n, max / 1e6, n * 1e9 / sum, 100.0 * wait_ns[PERF_PACE] / span, 100.0 * wait_ns[PERF_FENCE] / span, 100.0 * wait_ns[PERF_DRAIN] / span, 100.0 * wait_ns[PERF_PIPE] / span, 100.0 * wait_ns[PERF_XLATE] / span, @@ -576,7 +580,8 @@ void perf_present(uint64_t frame) { worst_ns[PERF_ASTC] / 1e6, worst_ns[PERF_PIPE] / 1e6, worst_ns[PERF_FENCE] / 1e6, (unsigned long long)shader_keys, (unsigned long long)shader_xlated, (span > wait_ns[PERF_PACE] ? span - wait_ns[PERF_PACE] : 0) / 1e6 / n, - plat_xchg64(&replay_ns, 0) / 1e6 / n, (unsigned long long)ab_b, (unsigned long long)n, frame_step_world_us() / 1e6); + plat_xchg64(&replay_ns, 0) / 1e6 / n, (unsigned long long)ab_b, (unsigned long long)n, frame_step_world_us() / 1e6, + (unsigned long long)dwin[0], dwin[1] / 1e6, (unsigned long long)dwin[2]); if (n && span) perf_gpu_line(frame, sum / 1e6 / n, (double)wait_ns[PERF_PACE] / span, n); sum = max = n = ab_b = 0; memset(wait_ns, 0, sizeof wait_ns); diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 0a9f6c7c..7e976808 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1392,6 +1392,30 @@ class MakeGameTest(Tmp): self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack']) self.assertIn('shader cache packed', buf.getvalue()) + def test_carried_update_packs_many_loose_shaders(self): + """An update that carries the shaders over packs the cache once it holds PACK_LOOSE_MIN loose translations, + and leaves fewer loose.""" + kit = self.tmp / 'kit' + (kit / 'tools').mkdir(parents=True) + shutil.copy(ROOT / 'tools' / 'pack_shader_cache.py', kit / 'tools') + cache = self.out / 'shader-cache' + cache.mkdir(parents=True) + for i in range(3): + (cache / (f'{i:02x}' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8) + for name, value in (('KIT', kit), ('PACK_LOOSE_MIN', 4), ('shader_carry', lambda out: True)): + self.addCleanup(setattr, self.mg, name, getattr(self.mg, name)) + setattr(self.mg, name, value) + with contextlib.redirect_stdout(io.StringIO()): + self.mg.step_shaders(self.out, update=True) + self.assertFalse((cache / 'shader-cache.pack').exists()) # 3 loose, under the minimum + (cache / ('03' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.mg.step_shaders(self.out, update=True) + self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack']) + self.assertIn('stay in use', buf.getvalue()) + self.assertIn('shader cache packed', buf.getvalue()) + def test_romfs_failure_stops_the_build(self): ran, lifting, failed = [], threading.Event(), threading.Event() diff --git a/tools/make_game.py b/tools/make_game.py index e243e849..7529e8c6 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1363,6 +1363,8 @@ def step_shaders(out, update=False): break if update and shader_carry(out): say(' the shaders prepared before this update stay in use; the game translates them again in the background') + if loose_shaders(out) >= PACK_LOOSE_MIN: + pack_shader_cache(out) return cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), '--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME] @@ -1415,6 +1417,21 @@ def step_shaders(out, update=False): pack_shader_cache(out) +# An update that carries the shaders over packs the cache when it has this many loose translations: the game writes +# each translation it makes in play, and those of its background pass after a translator change (about 42,000), as loose +# files, and reading one the first time cost 5.6 ms on Windows against a fraction of a millisecond from the pack. +PACK_LOOSE_MIN = 1000 + + +def loose_shaders(out): + """The loose translations (.spv) in the game's shader cache.""" + try: + with os.scandir(out / 'shader-cache') as it: + return sum(1 for e in it if e.name.endswith('.spv')) + except OSError: + return 0 + + def pack_shader_cache(out): """The prepared shader cache folded into shader-cache/shader-cache.pack (tools/pack_shader_cache.py): the preparation leaves four files per shader, about 175,000 for the whole game, which the game looks up one by one at diff --git a/tools/pack_shader_cache.py b/tools/pack_shader_cache.py index 10ca2f05..d14dde80 100644 --- a/tools/pack_shader_cache.py +++ b/tools/pack_shader_cache.py @@ -19,6 +19,8 @@ import os import re import struct import sys +from collections import deque +from concurrent.futures import ThreadPoolExecutor from pathlib import Path MAGIC = b'TKSPACK1' @@ -26,6 +28,11 @@ PACK = 'shader-cache.pack' KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4} NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)') USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks +# Loose files are read on READERS threads, at most WINDOW ahead of the one being written. On Windows the time goes into +# opening each file (about 3 ms the first time, against 0.04 ms once the system has seen it), which threads overlap: +# read one at a time, the 175,000 files of a launcher's cache took minutes. +READERS = 16 +WINDOW = 1024 def loose_files(cache): @@ -74,6 +81,13 @@ def valid(kind, data): return len(data) == USE_SIZE if kind == 3 else len(data) > 0 +def read_loose(path): + try: + return path.read_bytes() + except OSError: + return None + + def pack(cache, keep=False): cache = Path(cache) out = cache / PACK @@ -89,15 +103,23 @@ def pack(cache, keep=False): index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys) old = open(out, 'rb') if packed else None try: - with open(tmp, 'wb') as f: + with open(tmp, 'wb') as f, ThreadPoolExecutor(READERS) as pool: f.write(bytes(offset)) # header and index, written once the offsets are known - for key in keys: + ahead, todo = deque(), iter(keys) + + def submit(): + key = next(todo, None) + if key is not None: + ahead.append((key, pool.submit(read_loose, loose[key]) if key in loose else None)) + + for _ in range(WINDOW): + submit() + while ahead: + key, reading = ahead.popleft() + submit() data = None - if key in loose: # a valid loose file, else the packed copy, else nothing - try: - data = loose[key].read_bytes() - except OSError: - pass + if reading: # a valid loose file, else the packed copy, else nothing + data = reading.result() if data is not None and valid(key[0], data): written.append(loose[key]) else: