mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 20:09:57 -04:00
Pack the shader cache after the launcher prepares the shaders, as the installer did.
The launcher builds through tools/make_game.py, whose shaders step never ran tools/pack_shader_cache.py, so a launcher install keeps about 175,000 loose files (4.9 GB) in shader-cache/. The game looks them up one by one at every start and reads the use masks of 42,000 of them for 20 s after it. make_game.py now packs the cache after the preparation; a failure is reported and the build goes on, since the game still reads loose files. The kit and the launcher ship the tool.
This commit is contained in:
committed by
chrg6
parent
59069502f3
commit
6589476872
@@ -1368,6 +1368,29 @@ class MakeGameTest(Tmp):
|
||||
self.assertTrue(any(d.startswith('Built 10 of 100') and 'still running' in d for d in details))
|
||||
self.assertEqual([e['fraction'] for e in events if e['detail'] == 'Translated 50 of 100 shaders'][0], 0.25)
|
||||
|
||||
def test_shaders_step_packs_the_cache(self):
|
||||
"""After the preparation the shaders step folds the loose files of the shader cache into shader-cache.pack
|
||||
(tools/pack_shader_cache.py), as the installer did; a kit without the tool leaves the cache as it is."""
|
||||
kit = self.tmp / 'kit'
|
||||
(kit / 'tools').mkdir(parents=True)
|
||||
(kit / 'tools' / 'prepare_shaders.py').write_text('print("prepare-shaders: progress build 1 1", flush=True)\n')
|
||||
cache = self.out / 'shader-cache'
|
||||
(cache / 'in').mkdir(parents=True)
|
||||
(cache / ('ab' * 32 + '.spv')).write_bytes(b'\x03\x02\x23\x07' * 8)
|
||||
(cache / 'in' / ('cd' * 32 + '.in')).write_bytes(b'TKSI' * 4)
|
||||
self.addCleanup(setattr, self.mg, 'KIT', self.mg.KIT)
|
||||
self.mg.KIT = kit
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
self.mg.step_shaders(self.out)
|
||||
self.assertFalse((cache / 'shader-cache.pack').exists()) # no tool in the kit
|
||||
shutil.copy(ROOT / 'tools' / 'pack_shader_cache.py', kit / 'tools')
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
self.mg.step_shaders(self.out)
|
||||
self.assertTrue((cache / 'shader-cache.pack').is_file())
|
||||
self.assertEqual(sorted(p.name for p in cache.iterdir()), ['shader-cache.pack'])
|
||||
self.assertIn('shader cache packed', buf.getvalue())
|
||||
|
||||
def test_romfs_failure_stops_the_build(self):
|
||||
ran, lifting, failed = [], threading.Event(), threading.Event()
|
||||
|
||||
|
||||
@@ -1412,6 +1412,27 @@ def step_shaders(out, update=False):
|
||||
'meets them. Run the build again to retry this step')
|
||||
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
|
||||
(out / seed).unlink(missing_ok=True)
|
||||
pack_shader_cache(out)
|
||||
|
||||
|
||||
def pack_shader_cache(out):
|
||||
"""The prepared shader cache folded into shader-cache/shader-cache.pack (tools/pack_shader_cache.py): the
|
||||
preparation leaves four files per shader, about 175,000 for the whole game, which the game looks up one by one at
|
||||
every start (gfx_shader.c gfx_shader_prewarm, the use masks read after it) and which are slow to scan, copy and
|
||||
back up. The game reads the pack first and the loose files after it, so a cache left unpacked still works: a
|
||||
failure here is said and the build goes on."""
|
||||
tool = KIT / 'tools' / 'pack_shader_cache.py'
|
||||
if not tool.is_file() or not (out / 'shader-cache').is_dir():
|
||||
return
|
||||
t0 = time.monotonic()
|
||||
r = subprocess.run([sys.executable, '-u', str(tool), str(out / 'shader-cache')], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, text=True, encoding='utf-8', errors='replace')
|
||||
for line in r.stdout.splitlines():
|
||||
say(' ' + line)
|
||||
if r.returncode:
|
||||
say(f' packing the shader cache failed (code {r.returncode}); the game reads the loose files instead')
|
||||
else:
|
||||
say(f' shader cache packed in {time.monotonic() - t0:.0f} s')
|
||||
|
||||
|
||||
def prompt_art_dir():
|
||||
|
||||
@@ -27,7 +27,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
NAME = 'NX2navi-BuildKit'
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py',
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py') # make_game.py imports build_game's vma_include on
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py', 'pack_shader_cache.py') # make_game.py imports build_game's vma_include on
|
||||
# Linux for the VMA header, and build_game.py imports check_isa and build_shaders
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache'}
|
||||
|
||||
@@ -76,7 +76,8 @@ PINS = {
|
||||
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
|
||||
}
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py')
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py',
|
||||
'pack_shader_cache.py')
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
|
||||
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}
|
||||
|
||||
Reference in New Issue
Block a user