Pack the shader cache after the launcher prepares the shaders, as the installer did.

The launcher builds through tools/make_game.py, whose shaders step never ran tools/pack_shader_cache.py, so a
launcher install keeps about 175,000 loose files (4.9 GB) in shader-cache/. The game looks them up one by one at
every start and reads the use masks of 42,000 of them for 20 s after it. make_game.py now packs the cache after the
preparation; a failure is reported and the build goes on, since the game still reads loose files. The kit and the
launcher ship the tool.
This commit is contained in:
Sebastian Choperena
2026-11-09 12:33:10 -05:00
committed by chrg6
parent 59069502f3
commit 6589476872
4 changed files with 47 additions and 2 deletions

View File

@@ -1368,6 +1368,29 @@ class MakeGameTest(Tmp):
self.assertTrue(any(d.startswith('Built 10 of 100') and 'still running' in d for d in details))
self.assertEqual([e['fraction'] for e in events if e['detail'] == 'Translated 50 of 100 shaders'][0], 0.25)
def test_shaders_step_packs_the_cache(self):
"""After the preparation the shaders step folds the loose files of the shader cache into shader-cache.pack
(tools/pack_shader_cache.py), as the installer did; a kit without the tool leaves the cache as it is."""
kit = self.tmp / 'kit'
(kit / 'tools').mkdir(parents=True)
(kit / 'tools' / 'prepare_shaders.py').write_text('print("prepare-shaders: progress build 1 1", flush=True)\n')
cache = self.out / 'shader-cache'
(cache / 'in').mkdir(parents=True)
(cache / ('ab' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8)
(cache / 'in' / ('cd' * 32 + '.in')).write_bytes(b'TKSI' * 4)
self.addCleanup(setattr, self.mg, 'KIT', self.mg.KIT)
self.mg.KIT = kit
with contextlib.redirect_stdout(io.StringIO()):
self.mg.step_shaders(self.out)
self.assertFalse((cache / 'shader-cache.pack').exists()) # no tool in the kit
shutil.copy(ROOT / 'tools' / 'pack_shader_cache.py', kit / 'tools')
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
self.mg.step_shaders(self.out)
self.assertTrue((cache / 'shader-cache.pack').is_file())
self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack'])
self.assertIn('shader cache packed', buf.getvalue())
def test_romfs_failure_stops_the_build(self):
ran, lifting, failed = [], threading.Event(), threading.Event()

View File

@@ -1412,6 +1412,27 @@ def step_shaders(out, update=False):
'meets them. Run the build again to retry this step')
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
(out / seed).unlink(missing_ok=True)
pack_shader_cache(out)
def pack_shader_cache(out):
"""The prepared shader cache folded into shader-cache/shader-cache.pack (tools/pack_shader_cache.py): the
preparation leaves four files per shader, about 175,000 for the whole game, which the game looks up one by one at
every start (gfx_shader.c gfx_shader_prewarm, the use masks read after it) and which are slow to scan, copy and
back up. The game reads the pack first and the loose files after it, so a cache left unpacked still works: a
failure here is said and the build goes on."""
tool = KIT / 'tools' / 'pack_shader_cache.py'
if not tool.is_file() or not (out / 'shader-cache').is_dir():
return
t0 = time.monotonic()
r = subprocess.run([sys.executable, '-u', str(tool), str(out / 'shader-cache')], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace')
for line in r.stdout.splitlines():
say(' ' + line)
if r.returncode:
say(f' packing the shader cache failed (code {r.returncode}); the game reads the loose files instead')
else:
say(f' shader cache packed in {time.monotonic() - t0:.0f} s')
def prompt_art_dir():

View File

@@ -27,7 +27,7 @@ ROOT = Path(__file__).resolve().parents[1]
NAME = 'NX2navi-BuildKit'
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py',
'build_game.py', 'check_isa.py', 'build_shaders.py') # make_game.py imports build_game's vma_include on
'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on
# Linux for the VMA header, and build_game.py imports check_isa and build_shaders
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache'}

View File

@@ -76,7 +76,8 @@ PINS = {
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
}
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py')
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py',
'pack_shader_cache.py')
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}