MacKwnoX/NX2navi
1
0
Fork 0
mirror of https://github.com/tangerineturtle1/NX2navi.git synced 2026-11-15 17:09:57 -04:00
Code Issues Packages Projects Releases Wiki Activity

Prepare every one of the game's shaders when the build kit builds the game, so play does not stop to compile them.

Browse Source 
The shader reader now reads Switch 2 (NVN2) programs, whose 0xe0-byte control sections and task and mesh stages the
runtime keys as it does, and runs on Python 3.12 through the zstandard package. The kit's new last step writes the
36,031 shaders of the player's game files as a seed and runs ocarina.exe --prepare-shaders, which translates them and
has the GPU driver build each one's library, about 5 minutes. On a cold walk into play this took the stalls from
14 seconds of pipeline builds and 9 of translations to none.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-07 11:54:53 -05:00
parent ee6fbaf0bb
commit fc6d84aa63
6 changed files with 141 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/README-BuildKit.md
View File

@@ -46,7 +46,9 @@ With extracted folders instead, use `--exefs "C:\path\exefs" --romfs "C:\path\ro
check that your dump is the right version, add `--check` (it builds nothing).
The build checks the version, unpacks the program files, packs the 26 GB RomFS image (every file is checked against
the dump's own checksums), turns the game's code into C and compiles it (about 6 minutes on a 16-thread Ryzen 7; longer on fewer cores).
the dump's own checksums), turns the game's code into C and compiles it (about 6 minutes on a 16-thread Ryzen 7; longer on fewer cores),
then prepares every one of the game's shaders for your graphics card (about 5 more minutes), so the game does not stop
to compile shaders while you play.
You can use your PC meanwhile. If it stops,
run the same command again: finished steps are skipped.

16
tools/make_game.py
View File

@@ -13,6 +13,8 @@ Steps, each skipped when a run before finished it for the same input (state in <
4. lift recomp/gen.py with recomp/patches-v111.txt -> <out>/build-cache/lift, plus the two SDK bring-up fixes
5. compile the lifted C and runtime/ with MSYS2's clang, linked into <out>/ocarina.exe; the MSYS2 DLLs it loads,
the shader translator and romfs-index.bin beside it
6. shaders every shader in the game files (tools/prepare_shaders.py, default bind state) translated, and each one's
GPU driver build done once, so play does not stop to compile them (`ocarina.exe --prepare-shaders`)
Nothing is written outside --out. The game folder then holds only what this PC built from the player's files: never
share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), and MSYS2 UCRT64's clang and libraries
@@ -397,6 +399,17 @@ def step_compile(out, cache, msys, jobs):
# ---------------------------------------------------------------- main
def step_shaders(out):
"""Every shader of the game files into the game's shader cache and the driver's own cache, before the first start."""
cmd = [sys.executable, str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
'--exefs', str(out / 'exefs'), '--no-list', '--exe', 'ocarina.exe']
if subprocess.run(cmd).returncode:
fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it '
'meets them. Run the build again to retry this step')
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
(out / seed).unlink(missing_ok=True)
def keep_awake():
"""Hold off sleep for this process's life (Windows): the build runs for many minutes with no input, and a PC that
sleeps in the middle stops the compiles. The hold ends with the process."""
@@ -448,7 +461,8 @@ def main():
keep_awake()
t0 = time.time()
for name, run in (('exefs', lambda: step_exefs(src, out)), ('romfs', lambda: step_romfs(src, out, min(a.jobs, 8))),
('lift', lambda: step_lift(out, cache)), ('compile', lambda: step_compile(out, cache, msys, a.jobs))):
('lift', lambda: step_lift(out, cache)), ('compile', lambda: step_compile(out, cache, msys, a.jobs)),
('shaders', lambda: step_shaders(out))):
if name in state['done']:
say(f'{name}: done before')
continue

3
tools/make_kit.py
View File

@@ -14,7 +14,8 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
NAME = 'OcarinaRecomp-BuildKit'
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py')
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
'shader_census.py', 'shader_list.py', 'zstd_compat.py')
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example')
SKIP_DIRS = {'__pycache__', '.pytest_cache'}
SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'}

32
tools/prepare_shaders.py
View File

@@ -47,7 +47,7 @@ sys.path.insert(0, str(ROOT / 'tools'))
import shader_list # noqa: E402
from shader_census import inventory # noqa: E402
STAGES = {'vs': 0, 'ps': 4, 'cs': 5} # runtime/gfx.h GFX_STAGE_*
STAGES = {'vs': 0, 'ps': 4, 'cs': 5, 'ts': 6, 'ms': 7} # runtime/gfx.h GFX_STAGE_*
TEX = [None, '1d', '1darray', 'buffer', '2d', '2darray', '2dms', '2dmsarray', '3d', 'cube', 'cubearray']
FMT = [None, 'R8Unorm', 'R8Snorm', 'R8Uint', 'R8Sint', 'R16Float', 'R16Unorm', 'R16Snorm', 'R16Uint', 'R16Sint',
'R32Float', 'R32Uint', 'R32Sint', 'R8G8Unorm', 'R8G8Snorm', 'R8G8Uint', 'R8G8Sint', 'R16G16Float', 'R16G16Unorm',
@@ -122,13 +122,28 @@ def answers(deps, stage, control):
for idx, t in deps.get('fout', {}).items():
fout[int(idx)] = TYPES[t]
local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for graphics stages (they are in the input id)
if stage == 5: # compute: the control section's local size, 0 read as 1
local = [v or 1 for v in struct.unpack_from('<III', control, 0x748)]
if stage == 5: # compute: the control section's local size (NVN2 at +0x58), 0 read as 1
local = struct.unpack_from('<III', control, 0x58 if nvn2(control) else 0x748)
local = [1, 1, 1] if not any(local) else [v or 1 for v in local]
packed = bytes(tex + img + fmt + attr + fout) + struct.pack('<HHHIB', *local, 0, 0)
assert len(packed) == 147
return packed
def nvn2(control):
"""An NVN2 (Switch 2) control section as shader_census.nvn2_stages keeps it: 0xe0 bytes, zeros to 0x880."""
return control[:4] == bytes.fromhex('34127698') and not any(control[0xe0:0x880])
def bounds(control, code):
"""(c1 offset, length) as runtime/gfx_shader.c req_init takes them."""
if nvn2(control):
size, c1_start = struct.unpack_from('<I', control, 0x38)[0], struct.unpack_from('<I', control, 0x40)[0]
length = len(code) # shader_census cut the code at the end of its c1 part
return (c1_start if size <= c1_start <= length else length), length
return struct.unpack_from('<II', control, 0x700)
def code_hash(stage, control, code, length, c1):
"""gfx_shader.c build_code_hash."""
return hashlib.sha256(struct.pack('<I', stage) + control + struct.pack('<Q', length) + code[:length] +
@@ -150,9 +165,12 @@ def main():
ap.add_argument('--exefs', type=Path, help='ExeFS directory (default build/extracted/<version>/exefs)')
ap.add_argument('--list', help='a shader_list.tsv instead of tools/data/shader_list.tsv or refs/MacNativeGames')
ap.add_argument('--no-run', action='store_true', help='write the seed files only')
ap.add_argument('--no-list', action='store_true', help='default variants only, without a shared shader list')
ap.add_argument('--exe', default='nx2navi.exe' if os.name == 'nt' else 'nx2navi',
help='the game executable in the build directory that runs --prepare-shaders')
a = ap.parse_args()
t0 = time.monotonic()
variants, prio = parse_list(read_list(a.list))
variants, prio = parse_list('' if a.no_list else read_list(a.list))
extracted = ROOT / 'build' / 'extracted' / a.version
listed = len(variants)
shaders, _ = inventory(a.romfs or extracted / 'romfs', a.exefs or extracted / 'exefs', everything=False)
@@ -176,7 +194,7 @@ def main():
skipped['stage'] += 1
continue
control, code = s['control'][:CONTROL], s['code']
c1_off, length = struct.unpack_from('<II', control, 0x700)
c1_off, length = bounds(control, code)
c1 = code[c1_off:length]
if key not in progs:
h = code_hash(stage, control, code, length, c1)
@@ -205,7 +223,7 @@ def main():
f'seed written in {time.monotonic() - t0:.1f} s', flush=True)
if a.no_run:
return 0
exe = out / ('nx2navi.exe' if os.name == 'nt' else 'nx2navi')
exe = out / a.exe
r = subprocess.run([str(exe), '--prepare-shaders'], cwd=str(out))
for _ in range(RETRIES):
if not r.returncode or not driver_crashed(out):
@@ -213,7 +231,7 @@ def main():
print(f'prepare_shaders: the graphics driver crashed while building shaders (rc={r.returncode}); building them '
'again without the variants it was building (nx2navi.exe --warm-shaders)', flush=True)
r = subprocess.run([str(exe), '--warm-shaders'], cwd=str(out))
print(f'nx2navi.exe --prepare-shaders: rc={r.returncode}; total {time.monotonic() - t0:.1f} s')
print(f'{a.exe} --prepare-shaders: rc={r.returncode}; total {time.monotonic() - t0:.1f} s')
return r.returncode

51
tools/shader_census.py
View File

@@ -17,7 +17,7 @@ The key is sha256(control[:0x880] + code), the shared shader list's key. Needs P
.venv-py314\\Scripts\\python tools\\shader_census.py --version v121 --translate
"""
import argparse
import compression.zstd as zstd
import zstd_compat as zstd # compression.zstd on 3.14, the zstandard package before
import concurrent.futures
import hashlib
import json
@@ -73,11 +73,55 @@ def sarc(data):
raise ValueError('invalid SARC member')
name = data[off:data.index(0, off, start)].decode('utf-8')
if name.startswith('/') or '\\' in name or any(p in ('', '.', '..') for p in name.split('/')) or name in result:
raise ValueError('unsafe or duplicate SARC name')
continue # names only key the members here, nothing is written under them: an odd or repeated one is skipped
result[name] = checked(data, start + lo, hi - lo)
return result
NVN2_STAGES = {0: 'vs', 1: 'ps', 5: 'cs', 6: 'ts', 7: 'ms'} # the stage number in bits 16+ of control +0x54
NVN2_CONTROL = 0xe0 # the part of an NVN2 control section the runtime keeps (runtime/gfx_shader.c, "NVN2")
NVN2_SLOTS = range(8, 96, 8) # program slots that may point at a stage record: fragment +40, task +64, mesh +72 seen
def nvn2_record(data, offset):
"""(control, code) of the stage record at offset when it is an NVN2 one: {u64, control offset, code offset,
u32 code size, u32 control size}, the control shorter than NVN1's 0x880 and starting with the magic. Else None."""
if not offset or offset + 32 > len(data):
return None
control_off, code_off, code_size, control_size = struct.unpack_from('<QQII', data, offset + 8)
if not NVN2_CONTROL <= control_size < 0x880 or control_off + control_size > len(data) or code_off + code_size > len(data):
return None
control = data[control_off:control_off + control_size]
return (control, data[code_off:code_off + code_size]) if control[:4] == CONTROL_MAGIC else None
def nvn2_program(data, program):
return any(nvn2_record(data, struct.unpack_from('<Q', data, program + r)[0]) for r in NVN2_SLOTS
if program + r + 8 <= len(data))
def nvn2_stages(data, program):
"""The stages of an NVN2 (Switch 2) program as the runtime keys them: the control section's first 0xe0 bytes in a
zeroed 0x880 one, and the code from its first byte up to the end of its c1 constants (+0x40 to +0x44, after the
code of +0x38 bytes), the c1 part included."""
for r in NVN2_SLOTS:
if program + r + 8 > len(data):
break
rec = nvn2_record(data, struct.unpack_from('<Q', data, program + r)[0])
if not rec:
continue
control, code = rec
stage = NVN2_STAGES.get(struct.unpack_from('<I', control, 0x54)[0] >> 16)
if not stage:
continue
control = control[:NVN2_CONTROL] + bytes(0x880 - NVN2_CONTROL)
size, c1_start, c1_end = struct.unpack_from('<I', control, 0x38)[0], *struct.unpack_from('<II', control, 0x40)
length = c1_end if c1_start >= size and c1_end >= c1_start and c1_end <= len(code) else size
if not 0 < length <= len(code):
continue
yield digest(control + code[:length]), stage, control, code[:length]
def bnsh_shaders(data):
"""BNSH absolute offsets are relative to each embedded BNSH, not BFSHA."""
if data[:4] != b'BNSH' or checked(data, 12, 2) != b'\xff\xfe' or checked(data, 96, 4) != b'grsc':
@@ -89,6 +133,9 @@ def bnsh_shaders(data):
for i in range(count):
program = struct.unpack_from('<Q', data, variations + i * 64 + 16)[0]
checked(data, program, 56)
if nvn2_program(data, program):
yield from nvn2_stages(data, program)
continue
for stage, relative in [('vs', 8), ('ps', 40), ('cs', 48)]:
offset = struct.unpack_from('<Q', data, program + relative)[0]
if not offset:

47
tools/zstd_compat.py Normal file
View File

@@ -0,0 +1,47 @@
"""The part of Python 3.14's compression.zstd the shader tools use, on older Pythons through the zstandard package.
decompress(data, zstd_dict=None), ZstdDict(raw) with .dict_id, and get_frame_info(data) with .decompressed_size (None
when the frame does not say) and .dictionary_id (0 for none). On 3.14 the standard module itself is used.
"""
try:
from compression.zstd import ZstdDict, decompress, get_frame_info # noqa: F401
except ImportError:
from collections import namedtuple
import zstandard
FrameInfo = namedtuple('FrameInfo', 'decompressed_size dictionary_id')
LIMIT = 1 << 30 # the largest output a frame without a content size may decompress to
class ZstdDict:
def __init__(self, raw):
self._d = zstandard.ZstdCompressionDict(raw, dict_type=zstandard.DICT_TYPE_AUTO)
self.dict_id = self._d.dict_id()
def get_frame_info(data):
p = zstandard.get_frame_parameters(data)
size = p.content_size if p.content_size >= 0 and p.content_size != zstandard.CONTENTSIZE_UNKNOWN else None
return FrameInfo(size, p.dict_id)
def decompress(data, zstd_dict=None):
d = zstandard.ZstdDecompressor(dict_data=zstd_dict._d if zstd_dict else None)
p = zstandard.get_frame_parameters(data)
if 0 <= p.content_size != zstandard.CONTENTSIZE_UNKNOWN:
# one frame of a known size, the common case: decompressed in one call (the streaming path below is
# several times slower on the shader archives' hundreds of megabytes)
frame = zstandard.frame_content_size(data)
if frame == p.content_size:
try:
return d.decompress(data)
except zstandard.ZstdError:
pass # more than one frame: the streaming path
out, pos = [], 0
# One frame after another, as compression.zstd decompresses concatenated frames.
while pos < len(data):
obj = d.decompressobj()
out.append(obj.decompress(data[pos:]))
rest = obj.unused_data
pos = len(data) - len(rest)
if not rest:
break
return b''.join(out)