From fc6d84aa63f506a2ecd2ff046b5af3f5511b64d0 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Sat, 7 Nov 2026 11:54:53 -0500 Subject: [PATCH] Prepare every one of the game's shaders when the build kit builds the game, so play does not stop to compile them. The shader reader now reads Switch 2 (NVN2) programs, whose 0xe0-byte control sections and task and mesh stages the runtime keys as it does, and runs on Python 3.12 through the zstandard package. The kit's new last step writes the 36,031 shaders of the player's game files as a seed and runs ocarina.exe --prepare-shaders, which translates them and has the GPU driver build each one's library, about 5 minutes. On a cold walk into play this took the stalls from 14 seconds of pipeline builds and 9 of translations to none. Co-Authored-By: Claude Opus 5.5 --- docs/README-BuildKit.md | 4 +++- tools/make_game.py | 16 ++++++++++++- tools/make_kit.py | 3 ++- tools/prepare_shaders.py | 32 +++++++++++++++++++------ tools/shader_census.py | 51 ++++++++++++++++++++++++++++++++++++++-- tools/zstd_compat.py | 47 ++++++++++++++++++++++++++++++++++++ 6 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 tools/zstd_compat.py diff --git a/docs/README-BuildKit.md b/docs/README-BuildKit.md index ad348b5f..85d27817 100644 --- a/docs/README-BuildKit.md +++ b/docs/README-BuildKit.md @@ -46,7 +46,9 @@ With extracted folders instead, use `--exefs "C:\path\exefs" --romfs "C:\path\ro check that your dump is the right version, add `--check` (it builds nothing). The build checks the version, unpacks the program files, packs the 26 GB RomFS image (every file is checked against -the dump's own checksums), turns the game's code into C and compiles it (about 6 minutes on a 16-thread Ryzen 7; longer on fewer cores). +the dump's own checksums), turns the game's code into C and compiles it (about 6 minutes on a 16-thread Ryzen 7; longer on fewer cores), +then prepares every one of the game's shaders for your graphics card (about 5 more minutes), so the game does not stop +to compile shaders while you play. You can use your PC meanwhile. If it stops, run the same command again: finished steps are skipped. diff --git a/tools/make_game.py b/tools/make_game.py index f63b7bed..5633f3c4 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -13,6 +13,8 @@ Steps, each skipped when a run before finished it for the same input (state in < 4. lift recomp/gen.py with recomp/patches-v111.txt -> /build-cache/lift, plus the two SDK bring-up fixes 5. compile the lifted C and runtime/ with MSYS2's clang, linked into /ocarina.exe; the MSYS2 DLLs it loads, the shader translator and romfs-index.bin beside it + 6. shaders every shader in the game files (tools/prepare_shaders.py, default bind state) translated, and each one's + GPU driver build done once, so play does not stop to compile them (`ocarina.exe --prepare-shaders`) Nothing is written outside --out. The game folder then holds only what this PC built from the player's files: never share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), and MSYS2 UCRT64's clang and libraries @@ -397,6 +399,17 @@ def step_compile(out, cache, msys, jobs): # ---------------------------------------------------------------- main +def step_shaders(out): + """Every shader of the game files into the game's shader cache and the driver's own cache, before the first start.""" + cmd = [sys.executable, str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), + '--exefs', str(out / 'exefs'), '--no-list', '--exe', 'ocarina.exe'] + if subprocess.run(cmd).returncode: + fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it ' + 'meets them. Run the build again to retry this step') + for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them + (out / seed).unlink(missing_ok=True) + + def keep_awake(): """Hold off sleep for this process's life (Windows): the build runs for many minutes with no input, and a PC that sleeps in the middle stops the compiles. The hold ends with the process.""" @@ -448,7 +461,8 @@ def main(): keep_awake() t0 = time.time() for name, run in (('exefs', lambda: step_exefs(src, out)), ('romfs', lambda: step_romfs(src, out, min(a.jobs, 8))), - ('lift', lambda: step_lift(out, cache)), ('compile', lambda: step_compile(out, cache, msys, a.jobs))): + ('lift', lambda: step_lift(out, cache)), ('compile', lambda: step_compile(out, cache, msys, a.jobs)), + ('shaders', lambda: step_shaders(out))): if name in state['done']: say(f'{name}: done before') continue diff --git a/tools/make_kit.py b/tools/make_kit.py index e6241a80..7bb9e2be 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -14,7 +14,8 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] NAME = 'OcarinaRecomp-BuildKit' -TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py') +TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', + 'shader_census.py', 'shader_list.py', 'zstd_compat.py') TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache'} SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'} diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index bbc2e001..e38c6e5d 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -47,7 +47,7 @@ sys.path.insert(0, str(ROOT / 'tools')) import shader_list # noqa: E402 from shader_census import inventory # noqa: E402 -STAGES = {'vs': 0, 'ps': 4, 'cs': 5} # runtime/gfx.h GFX_STAGE_* +STAGES = {'vs': 0, 'ps': 4, 'cs': 5, 'ts': 6, 'ms': 7} # runtime/gfx.h GFX_STAGE_* TEX = [None, '1d', '1darray', 'buffer', '2d', '2darray', '2dms', '2dmsarray', '3d', 'cube', 'cubearray'] FMT = [None, 'R8Unorm', 'R8Snorm', 'R8Uint', 'R8Sint', 'R16Float', 'R16Unorm', 'R16Snorm', 'R16Uint', 'R16Sint', 'R32Float', 'R32Uint', 'R32Sint', 'R8G8Unorm', 'R8G8Snorm', 'R8G8Uint', 'R8G8Sint', 'R16G16Float', 'R16G16Unorm', @@ -122,13 +122,28 @@ def answers(deps, stage, control): for idx, t in deps.get('fout', {}).items(): fout[int(idx)] = TYPES[t] local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for graphics stages (they are in the input id) - if stage == 5: # compute: the control section's local size, 0 read as 1 - local = [v or 1 for v in struct.unpack_from(' len(data): + return None + control_off, code_off, code_size, control_size = struct.unpack_from(' len(data) or code_off + code_size > len(data): + return None + control = data[control_off:control_off + control_size] + return (control, data[code_off:code_off + code_size]) if control[:4] == CONTROL_MAGIC else None + + +def nvn2_program(data, program): + return any(nvn2_record(data, struct.unpack_from(' len(data): + break + rec = nvn2_record(data, struct.unpack_from('> 16) + if not stage: + continue + control = control[:NVN2_CONTROL] + bytes(0x880 - NVN2_CONTROL) + size, c1_start, c1_end = struct.unpack_from('= size and c1_end >= c1_start and c1_end <= len(code) else size + if not 0 < length <= len(code): + continue + yield digest(control + code[:length]), stage, control, code[:length] + + def bnsh_shaders(data): """BNSH absolute offsets are relative to each embedded BNSH, not BFSHA.""" if data[:4] != b'BNSH' or checked(data, 12, 2) != b'\xff\xfe' or checked(data, 96, 4) != b'grsc': @@ -89,6 +133,9 @@ def bnsh_shaders(data): for i in range(count): program = struct.unpack_from('= 0 and p.content_size != zstandard.CONTENTSIZE_UNKNOWN else None + return FrameInfo(size, p.dict_id) + + def decompress(data, zstd_dict=None): + d = zstandard.ZstdDecompressor(dict_data=zstd_dict._d if zstd_dict else None) + p = zstandard.get_frame_parameters(data) + if 0 <= p.content_size != zstandard.CONTENTSIZE_UNKNOWN: + # one frame of a known size, the common case: decompressed in one call (the streaming path below is + # several times slower on the shader archives' hundreds of megabytes) + frame = zstandard.frame_content_size(data) + if frame == p.content_size: + try: + return d.decompress(data) + except zstandard.ZstdError: + pass # more than one frame: the streaming path + out, pos = [], 0 + # One frame after another, as compression.zstd decompresses concatenated frames. + while pos < len(data): + obj = d.decompressobj() + out.append(obj.decompress(data[pos:])) + rest = obj.unused_data + pos = len(data) - len(rest) + if not rest: + break + return b''.join(out)