diff --git a/docs/architecture.md b/docs/architecture.md index 29fbf0c7..dd183d5e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -101,6 +101,11 @@ keeps per-core data without locks and relies on that, so the host models it (`ru translates, translates every stored program again for the next start, then deletes the old translations (a pruned copy of `shader-cache.pack` replaces the pack at the next start). A program the new translator refuses keeps its old translation. `--prepare-shaders` still translates everything with the current translator. +- Shader disk reads in play: a draw's shader that is in the disk cache but not in memory is read on the render thread + within a budget of 1/8 of the wall time (2 ms at once), and past it by three loader threads while its draws are + skipped as for a translation (`runtime/gfx_shader.c` `load_budget`). Opening a loose file is what costs (5.6 ms a + pair the first time on Windows), so an update that carries the shaders over packs the cache once it holds 1,000 + loose translations (`tools/make_game.py` `PACK_LOOSE_MIN`). - Mesh and task shaders run as `VK_EXT_mesh_shader`; indirect mesh draws read NVN's `{count, first}` record through a ring of Vulkan `{x, y, z}` records filled on the GPU. - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link diff --git a/runtime/gfx.h b/runtime/gfx.h index 50836592..2a1b7778 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -54,6 +54,8 @@ const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8 const GfxShaderAnswers *a); uint64_t gfx_shader_async_done(void); // translations finished by the background worker void gfx_shader_counts(uint64_t *keys, uint64_t *translated); // distinct keys in memory, translations done (NX2NAVI_TRACE=perf) +void gfx_shader_disk_counts(uint64_t *sync, uint64_t *sync_ns, uint64_t *bg); // disk cache reads on the render + // thread (count, ns) and by the loaders (gfx_shader.c load_budget) // Translator answers for this device (vk_caps.c): shaderFloat64, storage offset rounding, subgroup size per vs/fs/cs. void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals, int bda); diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 0d6c3b8f..a0356a92 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1904,12 +1904,53 @@ typedef struct XWait { static struct { PlatMutex lock; PlatCond work, done; // done: a job finished (gfx_shader_await) + PlatCond lwork; // a disk read was queued (load_worker) XJob *head, *tail; + XJob *lhead, *ltail; // disk reads past the render thread's budget (load_budget) XWait *waits[256]; int started; // 1 running, -1 no thread could be created -} X = {PLAT_MUTEX_INIT, PLAT_COND_INIT, PLAT_COND_INIT}; + int loaders; // load_worker threads running +} X = {PLAT_MUTEX_INIT, PLAT_COND_INIT, PLAT_COND_INIT, PLAT_COND_INIT}; static volatile int64_t n_async; +static volatile int64_t n_disk_sync, n_disk_sync_ns, n_disk_bg; // gfx_shader_disk_counts + +// Disk reads in get_async_one, the render thread's lookup. A shader that is in the disk cache but not in memory yet is +// read there while those reads stay within LOAD_SHARE of the wall time, at most LOAD_BURST_NS at once; past that a +// load_worker reads it, and its draws are skipped (or a one-shot draw waits) as for a shader being translated. Reading a +// pair of loose files the first time took 5.6 ms on Windows, against 0.08 ms once the system had them: the time is +// spent opening the files, not reading them. An area that needed 50 shaders not met before held one frame for 100 to 500 +// ms. A packed pair takes a fraction of a millisecond, so with a pack nearly all of them are still read in place. +#define LOAD_BURST_NS 2000000ll +#define LOAD_SHARE 8 // a budget of 1/8 of the wall time +#define LOADERS 3 +static PlatMutex load_lock = PLAT_MUTEX_INIT; +static int64_t load_tokens = LOAD_BURST_NS; // under load_lock +static uint64_t load_last; + +// Whether the render thread may read from the disk cache now. +static int load_budget(void) { + uint64_t now = plat_now_ns(); + plat_mutex_lock(&load_lock); + if (load_last) load_tokens += (int64_t)(now - load_last) / LOAD_SHARE; + if (load_tokens > LOAD_BURST_NS) load_tokens = LOAD_BURST_NS; + load_last = now; + int ok = load_tokens > 0; + plat_mutex_unlock(&load_lock); + return ok; +} + +static void load_spent(uint64_t ns) { + plat_mutex_lock(&load_lock); + load_tokens -= (int64_t)ns; + plat_mutex_unlock(&load_lock); + plat_inc64(&n_disk_sync); + __atomic_add_fetch(&n_disk_sync_ns, (int64_t)ns, __ATOMIC_RELAXED); +} + +void gfx_shader_disk_counts(uint64_t *sync, uint64_t *sync_ns, uint64_t *bg) { + *sync = (uint64_t)n_disk_sync, *sync_ns = (uint64_t)n_disk_sync_ns, *bg = (uint64_t)n_disk_bg; +} // Caller holds X.lock. static XWait **wait_slot(const uint8_t key[32]) { @@ -1938,6 +1979,67 @@ static void xlate_worker(void *arg) { } return; } + +// Reads the shaders get_async_one queued past its budget (load_budget), in get_one's disk order: the current key, then +// a previous translator's entry. One the disk cache lacks goes on to the translation queue, its key still waiting. +static void load_worker(void *arg) { + (void)arg; + for (;;) { + plat_mutex_lock(&X.lock); + while (!X.lhead) plat_cond_wait(&X.lwork, &X.lock); + XJob *j = X.lhead; + if (!(X.lhead = j->next)) X.ltail = NULL; + plat_mutex_unlock(&X.lock); + GfxShader *d = j->r.final ? disk_load(j->r.key, j->r.iid) : NULL; + const GfxShader *s = d ? mem_insert(d) : carry_find(j->r.code_hash, &j->a, j->stage, 1); + plat_mutex_lock(&X.lock); + if (s) { + XWait **w = wait_slot(j->r.key), *e = *w; + if (e) *w = e->next, free(e); + } else { + j->next = NULL; + if (X.tail) X.tail->next = j; + else X.head = j; + X.tail = j; + plat_cond_signal(&X.work); + } + plat_mutex_unlock(&X.lock); + if (!s) continue; + plat_cond_broadcast(&X.done); + plat_inc64(&n_hits), plat_inc64(&n_disk_bg); + free(j->control), free(j->code), free(j); + } +} + +// Caller holds X.lock: the translation workers and the disk readers, started at the first job. +static void start_workers(void) { + if (!X.started) { + int n = 0; + for (int i = 0; i < svc_runtime_limit(); i++) { + if (plat_thread_spawn(xlate_worker, NULL, "xlate", 0)) n++; + } + X.started = n ? 1 : -1; + if (n) rt_log("gfx_shader: graphics shaders are translated by %d background workers (NX2NAVI_XLATE_PROCS)", n); + else rt_log("gfx_shader: cannot start the translation workers; translating synchronously"); + for (int i = 0; n && i < LOADERS; i++) { + if (plat_thread_spawn(load_worker, NULL, "shload", 0)) __atomic_add_fetch(&X.loaders, 1, __ATOMIC_RELAXED); + } + } +} + +// Caller holds X.lock: the job of key to the front of its queue, so the worker takes it next (gfx_shader_await). +static void to_front(XJob **head, XJob **tail, const uint8_t key[32]) { + for (XJob *j = *head, *prev = NULL; j; prev = j, j = j->next) + if (!memcmp(j->r.key, key, 32)) { + if (prev) { + prev->next = j->next; + if (*tail == j) *tail = prev; + j->next = *head, *head = j; + } + return; + } +} + const GfxShader *gfx_shader_get_async(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, const GfxShaderAnswers *a, int *pending) { *pending = 0; @@ -1961,41 +2063,45 @@ static const GfxShader *get_async_one(Req r, int stage, const uint8_t *control, if (w) return *pending = !failed, NULL; paths(); if (!cache_dir[0]) return fail("no cache directory", NULL); - if (r.final && (s = disk_load(r.key, r.iid))) return plat_inc64(&n_hits), mem_insert(s); - if ((s = carry_find(r.code_hash, a, stage, 1))) return plat_inc64(&n_hits), s; + int read = load_budget() || __atomic_load_n(&X.loaders, __ATOMIC_RELAXED) == 0; // the disk here, or a loader + if (read) { + uint64_t t0 = plat_now_ns(); + GfxShader *d = r.final ? disk_load(r.key, r.iid) : NULL; + s = d ? mem_insert(d) : carry_find(r.code_hash, a, stage, 1); + load_spent(plat_now_ns() - t0); + if (s) return plat_inc64(&n_hits), s; + } XJob *j = calloc(1, sizeof *j); XWait *e = calloc(1, sizeof *e); uint8_t *ctl = malloc(CONTROL_SIZE), *cd = malloc(r.length); plat_mutex_lock(&X.lock); - if (!X.started) { - int n = 0; - for (int i = 0; i < svc_runtime_limit(); i++) { - if (plat_thread_spawn(xlate_worker, NULL, "xlate", 0)) n++; - } - X.started = n ? 1 : -1; - if (n) rt_log("gfx_shader: graphics shaders are translated by %d background workers (NX2NAVI_XLATE_PROCS)", n); - else rt_log("gfx_shader: cannot start the translation workers; translating synchronously"); - } + start_workers(); XWait **slot = wait_slot(r.key); int raced = *slot != NULL; // another thread queued it between the checks int queued = !raced && X.started == 1 && j && e && ctl && cd; + int load = queued && !read && X.loaders; // past the budget: a loader reads it first if (queued) { memcpy(ctl, control, CONTROL_SIZE), memcpy(cd, code, r.length); *j = (XJob){.stage = stage, .r = r, .control = ctl, .code = cd, .a = *a}; j->r.c1 = cd + (r.length - r.c1_len); memcpy(e->key, r.key, 32); *slot = e; - if (X.tail) X.tail->next = j; - else X.head = j; - X.tail = j; - plat_cond_signal(&X.work); + XJob **head = load ? &X.lhead : &X.head, **tail = load ? &X.ltail : &X.tail; + if (*tail) (*tail)->next = j; + else *head = j; + *tail = j; + plat_cond_signal(load ? &X.lwork : &X.work); } plat_mutex_unlock(&X.lock); if (queued) return *pending = 1, NULL; free(j), free(e), free(ctl), free(cd); if (raced) return *pending = 1, NULL; - return translate(&r, stage, control, code, a); // no worker or out of memory + if (!read) { // no worker or out of memory: everything here + GfxShader *d = r.final ? disk_load(r.key, r.iid) : NULL; + if ((s = d ? mem_insert(d) : carry_find(r.code_hash, a, stage, 1))) return plat_inc64(&n_hits), s; + } + return translate(&r, stage, control, code, a); } const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, @@ -2008,15 +2114,8 @@ const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8 if (!pending || !req_init(&r, stage, control, code, code_size, NULL, a)) return s; uint64_t p0 = perf_begin(); // the wait counts as the render thread's translation time plat_mutex_lock(&X.lock); - for (XJob *j = X.head, *prev = NULL; j; prev = j, j = j->next) - if (!memcmp(j->r.key, r.key, 32)) { // queued: to the front, the worker takes it next - if (prev) { - prev->next = j->next; - if (X.tail == j) X.tail = prev; - j->next = X.head, X.head = j; - } - break; - } + to_front(&X.lhead, &X.ltail, r.key); // queued: to the front, a worker takes it next + to_front(&X.head, &X.tail, r.key); XWait *w; while ((w = *wait_slot(r.key)) && !w->failed) plat_cond_wait(&X.done, &X.lock); plat_mutex_unlock(&X.lock); diff --git a/runtime/perf.c b/runtime/perf.c index 1a27ffef..f0fd0f34 100644 --- a/runtime/perf.c +++ b/runtime/perf.c @@ -564,10 +564,14 @@ void perf_present(uint64_t frame) { int64_t async = win[PERF_CNT_PIPE_ASYNC], skips = win[PERF_CNT_PIPE_SKIP], up = win[PERF_CNT_TEX_UP]; int64_t reup = win[PERF_CNT_TEX_REUP], sskip = win[PERF_CNT_SHADER_SKIP], waited = win[PERF_CNT_PIPE_WAIT]; int64_t oneshot = win[PERF_CNT_ONESHOT_WAIT]; - uint64_t shader_keys, shader_xlated; + uint64_t shader_keys, shader_xlated, disk[3]; + static uint64_t disk_prev[3]; gfx_shader_counts(&shader_keys, &shader_xlated); + gfx_shader_disk_counts(&disk[0], &disk[1], &disk[2]); + uint64_t dwin[3]; + for (int i = 0; i < 3; i++) dwin[i] = disk[i] - disk_prev[i], disk_prev[i] = disk[i]; if (n && span) - rt_log("perf: frame %llu: interval avg %.2f ms max %.2f ms (%.1f fps); render thread blocked: pacing %.0f%% fence %.0f%% drain %.0f%% pipeline build %.0f%% shader xlate %.0f%% texture %.0f%% (astc %.0f%%); pipelines built in background %lld, draws skipped pending %lld, waited %lld, waiting for a shader %lld, one-shot waited %lld; textures uploaded %lld re-uploaded %lld; slowest frame: xlate %.1f ms texture %.1f ms (astc %.1f) pipeline %.1f ms fence %.1f ms; shaders: %llu keys, %llu translated; render busy avg %.2f ms; replay avg %.2f ms; ab %llu/%llu; world %.1f s", + rt_log("perf: frame %llu: interval avg %.2f ms max %.2f ms (%.1f fps); render thread blocked: pacing %.0f%% fence %.0f%% drain %.0f%% pipeline build %.0f%% shader xlate %.0f%% texture %.0f%% (astc %.0f%%); pipelines built in background %lld, draws skipped pending %lld, waited %lld, waiting for a shader %lld, one-shot waited %lld; textures uploaded %lld re-uploaded %lld; slowest frame: xlate %.1f ms texture %.1f ms (astc %.1f) pipeline %.1f ms fence %.1f ms; shaders: %llu keys, %llu translated; render busy avg %.2f ms; replay avg %.2f ms; ab %llu/%llu; world %.1f s; shader disk reads: %llu on the render thread (%.1f ms), %llu by the loaders", (unsigned long long)frame, sum / 1e6 / n, max / 1e6, n * 1e9 / sum, 100.0 * wait_ns[PERF_PACE] / span, 100.0 * wait_ns[PERF_FENCE] / span, 100.0 * wait_ns[PERF_DRAIN] / span, 100.0 * wait_ns[PERF_PIPE] / span, 100.0 * wait_ns[PERF_XLATE] / span, @@ -576,7 +580,8 @@ void perf_present(uint64_t frame) { worst_ns[PERF_ASTC] / 1e6, worst_ns[PERF_PIPE] / 1e6, worst_ns[PERF_FENCE] / 1e6, (unsigned long long)shader_keys, (unsigned long long)shader_xlated, (span > wait_ns[PERF_PACE] ? span - wait_ns[PERF_PACE] : 0) / 1e6 / n, - plat_xchg64(&replay_ns, 0) / 1e6 / n, (unsigned long long)ab_b, (unsigned long long)n, frame_step_world_us() / 1e6); + plat_xchg64(&replay_ns, 0) / 1e6 / n, (unsigned long long)ab_b, (unsigned long long)n, frame_step_world_us() / 1e6, + (unsigned long long)dwin[0], dwin[1] / 1e6, (unsigned long long)dwin[2]); if (n && span) perf_gpu_line(frame, sum / 1e6 / n, (double)wait_ns[PERF_PACE] / span, n); sum = max = n = ab_b = 0; memset(wait_ns, 0, sizeof wait_ns); diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 0a9f6c7c..7e976808 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1392,6 +1392,30 @@ class MakeGameTest(Tmp): self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack']) self.assertIn('shader cache packed', buf.getvalue()) + def test_carried_update_packs_many_loose_shaders(self): + """An update that carries the shaders over packs the cache once it holds PACK_LOOSE_MIN loose translations, + and leaves fewer loose.""" + kit = self.tmp / 'kit' + (kit / 'tools').mkdir(parents=True) + shutil.copy(ROOT / 'tools' / 'pack_shader_cache.py', kit / 'tools') + cache = self.out / 'shader-cache' + cache.mkdir(parents=True) + for i in range(3): + (cache / (f'{i:02x}' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8) + for name, value in (('KIT', kit), ('PACK_LOOSE_MIN', 4), ('shader_carry', lambda out: True)): + self.addCleanup(setattr, self.mg, name, getattr(self.mg, name)) + setattr(self.mg, name, value) + with contextlib.redirect_stdout(io.StringIO()): + self.mg.step_shaders(self.out, update=True) + self.assertFalse((cache / 'shader-cache.pack').exists()) # 3 loose, under the minimum + (cache / ('03' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.mg.step_shaders(self.out, update=True) + self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack']) + self.assertIn('stay in use', buf.getvalue()) + self.assertIn('shader cache packed', buf.getvalue()) + def test_romfs_failure_stops_the_build(self): ran, lifting, failed = [], threading.Event(), threading.Event() diff --git a/tools/make_game.py b/tools/make_game.py index e243e849..7529e8c6 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1363,6 +1363,8 @@ def step_shaders(out, update=False): break if update and shader_carry(out): say(' the shaders prepared before this update stay in use; the game translates them again in the background') + if loose_shaders(out) >= PACK_LOOSE_MIN: + pack_shader_cache(out) return cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), '--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME] @@ -1415,6 +1417,21 @@ def step_shaders(out, update=False): pack_shader_cache(out) +# An update that carries the shaders over packs the cache when it has this many loose translations: the game writes +# each translation it makes in play, and those of its background pass after a translator change (about 42,000), as loose +# files, and reading one the first time cost 5.6 ms on Windows against a fraction of a millisecond from the pack. +PACK_LOOSE_MIN = 1000 + + +def loose_shaders(out): + """The loose translations (.spv) in the game's shader cache.""" + try: + with os.scandir(out / 'shader-cache') as it: + return sum(1 for e in it if e.name.endswith('.spv')) + except OSError: + return 0 + + def pack_shader_cache(out): """The prepared shader cache folded into shader-cache/shader-cache.pack (tools/pack_shader_cache.py): the preparation leaves four files per shader, about 175,000 for the whole game, which the game looks up one by one at diff --git a/tools/pack_shader_cache.py b/tools/pack_shader_cache.py index 10ca2f05..d14dde80 100644 --- a/tools/pack_shader_cache.py +++ b/tools/pack_shader_cache.py @@ -19,6 +19,8 @@ import os import re import struct import sys +from collections import deque +from concurrent.futures import ThreadPoolExecutor from pathlib import Path MAGIC = b'TKSPACK1' @@ -26,6 +28,11 @@ PACK = 'shader-cache.pack' KINDS = {'.spv': 1, '.res.json': 2, '.use': 3, '.in': 4} NAME = re.compile(r'([0-9a-f]{64})(\.spv|\.res\.json|\.use|\.in)') USE_SIZE = 20 # gfx_shader.c use_store: magic, version, three masks +# Loose files are read on READERS threads, at most WINDOW ahead of the one being written. On Windows the time goes into +# opening each file (about 3 ms the first time, against 0.04 ms once the system has seen it), which threads overlap: +# read one at a time, the 175,000 files of a launcher's cache took minutes. +READERS = 16 +WINDOW = 1024 def loose_files(cache): @@ -74,6 +81,13 @@ def valid(kind, data): return len(data) == USE_SIZE if kind == 3 else len(data) > 0 +def read_loose(path): + try: + return path.read_bytes() + except OSError: + return None + + def pack(cache, keep=False): cache = Path(cache) out = cache / PACK @@ -89,15 +103,23 @@ def pack(cache, keep=False): index, written, skipped, offset = [], [], 0, 16 + 48 * len(keys) old = open(out, 'rb') if packed else None try: - with open(tmp, 'wb') as f: + with open(tmp, 'wb') as f, ThreadPoolExecutor(READERS) as pool: f.write(bytes(offset)) # header and index, written once the offsets are known - for key in keys: + ahead, todo = deque(), iter(keys) + + def submit(): + key = next(todo, None) + if key is not None: + ahead.append((key, pool.submit(read_loose, loose[key]) if key in loose else None)) + + for _ in range(WINDOW): + submit() + while ahead: + key, reading = ahead.popleft() + submit() data = None - if key in loose: # a valid loose file, else the packed copy, else nothing - try: - data = loose[key].read_bytes() - except OSError: - pass + if reading: # a valid loose file, else the packed copy, else nothing + data = reading.result() if data is not None and valid(key[0], data): written.append(loose[key]) else: