From 65894768720e69cdb7a6d3b1aea1d68f02963193 Mon Sep 17 00:00:00 2001 From: Sebastian Choperena <64750853+schoperena@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:33:10 -0500 Subject: [PATCH] Pack the shader cache after the launcher prepares the shaders, as the installer did. The launcher builds through tools/make_game.py, whose shaders step never ran tools/pack_shader_cache.py, so a launcher install keeps about 175,000 loose files (4.9 GB) in shader-cache/. The game looks them up one by one at every start and reads the use masks of 42,000 of them for 20 s after it. make_game.py now packs the cache after the preparation; a failure is reported and the build goes on, since the game still reads loose files. The kit and the launcher ship the tool. --- tests/launcher_test.py | 23 +++++++++++++++++++++++ tools/make_game.py | 21 +++++++++++++++++++++ tools/make_kit.py | 2 +- tools/make_launcher.py | 3 ++- 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 6ed6230c..277d1c30 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1368,6 +1368,29 @@ class MakeGameTest(Tmp): self.assertTrue(any(d.startswith('Built 10 of 100') and 'still running' in d for d in details)) self.assertEqual([e['fraction'] for e in events if e['detail'] == 'Translated 50 of 100 shaders'][0], 0.25) + def test_shaders_step_packs_the_cache(self): + """After the preparation the shaders step folds the loose files of the shader cache into shader-cache.pack + (tools/pack_shader_cache.py), as the installer did; a kit without the tool leaves the cache as it is.""" + kit = self.tmp / 'kit' + (kit / 'tools').mkdir(parents=True) + (kit / 'tools' / 'prepare_shaders.py').write_text('print("prepare-shaders: progress build 1 1", flush=True)\n') + cache = self.out / 'shader-cache' + (cache / 'in').mkdir(parents=True) + (cache / ('ab' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8) + (cache / 'in' / ('cd' * 32 + '.in')).write_bytes(b'TKSI' * 4) + self.addCleanup(setattr, self.mg, 'KIT', self.mg.KIT) + self.mg.KIT = kit + with contextlib.redirect_stdout(io.StringIO()): + self.mg.step_shaders(self.out) + self.assertFalse((cache / 'shader-cache.pack').exists()) # no tool in the kit + shutil.copy(ROOT / 'tools' / 'pack_shader_cache.py', kit / 'tools') + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.mg.step_shaders(self.out) + self.assertTrue((cache / 'shader-cache.pack').is_file()) + self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack']) + self.assertIn('shader cache packed', buf.getvalue()) + def test_romfs_failure_stops_the_build(self): ran, lifting, failed = [], threading.Event(), threading.Event() diff --git a/tools/make_game.py b/tools/make_game.py index 5ec74030..566befb2 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1412,6 +1412,27 @@ def step_shaders(out, update=False): 'meets them. Run the build again to retry this step') for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them (out / seed).unlink(missing_ok=True) + pack_shader_cache(out) + + +def pack_shader_cache(out): + """The prepared shader cache folded into shader-cache/shader-cache.pack (tools/pack_shader_cache.py): the + preparation leaves four files per shader, about 175,000 for the whole game, which the game looks up one by one at + every start (gfx_shader.c gfx_shader_prewarm, the use masks read after it) and which are slow to scan, copy and + back up. The game reads the pack first and the loose files after it, so a cache left unpacked still works: a + failure here is said and the build goes on.""" + tool = KIT / 'tools' / 'pack_shader_cache.py' + if not tool.is_file() or not (out / 'shader-cache').is_dir(): + return + t0 = time.monotonic() + r = subprocess.run([sys.executable, '-u', str(tool), str(out / 'shader-cache')], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace') + for line in r.stdout.splitlines(): + say(' ' + line) + if r.returncode: + say(f' packing the shader cache failed (code {r.returncode}); the game reads the loose files instead') + else: + say(f' shader cache packed in {time.monotonic() - t0:.0f} s') def prompt_art_dir(): diff --git a/tools/make_kit.py b/tools/make_kit.py index 78d043ee..26c308ea 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -27,7 +27,7 @@ ROOT = Path(__file__).resolve().parents[1] NAME = 'NX2navi-BuildKit' TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', - 'build_game.py', 'check_isa.py', 'build_shaders.py') # make_game.py imports build_game's vma_include on + 'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on # Linux for the VMA header, and build_game.py imports check_isa and build_shaders TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache'} diff --git a/tools/make_launcher.py b/tools/make_launcher.py index ee39bbf9..3b91c77c 100644 --- a/tools/make_launcher.py +++ b/tools/make_launcher.py @@ -76,7 +76,8 @@ PINS = { 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'), } TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', - 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py') + 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', + 'pack_shader_cache.py') TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'} SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}