diff --git a/docs/README-BuildKit.md b/docs/README-BuildKit.md index ad348b5f..85d27817 100644 --- a/docs/README-BuildKit.md +++ b/docs/README-BuildKit.md @@ -46,7 +46,9 @@ With extracted folders instead, use `--exefs "C:\path\exefs" --romfs "C:\path\ro check that your dump is the right version, add `--check` (it builds nothing). The build checks the version, unpacks the program files, packs the 26 GB RomFS image (every file is checked against -the dump's own checksums), turns the game's code into C and compiles it (about 6 minutes on a 16-thread Ryzen 7; longer on fewer cores). +the dump's own checksums), turns the game's code into C and compiles it (about 6 minutes on a 16-thread Ryzen 7; longer on fewer cores), +then prepares every one of the game's shaders for your graphics card (about 5 more minutes), so the game does not stop +to compile shaders while you play. You can use your PC meanwhile. If it stops, run the same command again: finished steps are skipped. diff --git a/tools/make_game.py b/tools/make_game.py index f63b7bed..5633f3c4 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -13,6 +13,8 @@ Steps, each skipped when a run before finished it for the same input (state in < 4. lift recomp/gen.py with recomp/patches-v111.txt -> /build-cache/lift, plus the two SDK bring-up fixes 5. compile the lifted C and runtime/ with MSYS2's clang, linked into /ocarina.exe; the MSYS2 DLLs it loads, the shader translator and romfs-index.bin beside it + 6. shaders every shader in the game files (tools/prepare_shaders.py, default bind state) translated, and each one's + GPU driver build done once, so play does not stop to compile them (`ocarina.exe --prepare-shaders`) Nothing is written outside --out. The game folder then holds only what this PC built from the player's files: never share it. Needs Python 3.10+ with numpy, lz4, zstandard and zbic (pip), and MSYS2 UCRT64's clang and libraries @@ -397,6 +399,17 @@ def step_compile(out, cache, msys, jobs): # ---------------------------------------------------------------- main +def step_shaders(out): + """Every shader of the game files into the game's shader cache and the driver's own cache, before the first start.""" + cmd = [sys.executable, str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), + '--exefs', str(out / 'exefs'), '--no-list', '--exe', 'ocarina.exe'] + if subprocess.run(cmd).returncode: + fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it ' + 'meets them. Run the build again to retry this step') + for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them + (out / seed).unlink(missing_ok=True) + + def keep_awake(): """Hold off sleep for this process's life (Windows): the build runs for many minutes with no input, and a PC that sleeps in the middle stops the compiles. The hold ends with the process.""" @@ -448,7 +461,8 @@ def main(): keep_awake() t0 = time.time() for name, run in (('exefs', lambda: step_exefs(src, out)), ('romfs', lambda: step_romfs(src, out, min(a.jobs, 8))), - ('lift', lambda: step_lift(out, cache)), ('compile', lambda: step_compile(out, cache, msys, a.jobs))): + ('lift', lambda: step_lift(out, cache)), ('compile', lambda: step_compile(out, cache, msys, a.jobs)), + ('shaders', lambda: step_shaders(out))): if name in state['done']: say(f'{name}: done before') continue diff --git a/tools/make_kit.py b/tools/make_kit.py index e6241a80..7bb9e2be 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -14,7 +14,8 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] NAME = 'OcarinaRecomp-BuildKit' -TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py') +TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', + 'shader_census.py', 'shader_list.py', 'zstd_compat.py') TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'VERSION', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache'} SKIP_SUFFIXES = {'.o', '.obj', '.pyc', '.exe', '.dll', '.a', '.lib'} diff --git a/tools/prepare_shaders.py b/tools/prepare_shaders.py index bbc2e001..e38c6e5d 100644 --- a/tools/prepare_shaders.py +++ b/tools/prepare_shaders.py @@ -47,7 +47,7 @@ sys.path.insert(0, str(ROOT / 'tools')) import shader_list # noqa: E402 from shader_census import inventory # noqa: E402 -STAGES = {'vs': 0, 'ps': 4, 'cs': 5} # runtime/gfx.h GFX_STAGE_* +STAGES = {'vs': 0, 'ps': 4, 'cs': 5, 'ts': 6, 'ms': 7} # runtime/gfx.h GFX_STAGE_* TEX = [None, '1d', '1darray', 'buffer', '2d', '2darray', '2dms', '2dmsarray', '3d', 'cube', 'cubearray'] FMT = [None, 'R8Unorm', 'R8Snorm', 'R8Uint', 'R8Sint', 'R16Float', 'R16Unorm', 'R16Snorm', 'R16Uint', 'R16Sint', 'R32Float', 'R32Uint', 'R32Sint', 'R8G8Unorm', 'R8G8Snorm', 'R8G8Uint', 'R8G8Sint', 'R16G16Float', 'R16G16Unorm', @@ -122,13 +122,28 @@ def answers(deps, stage, control): for idx, t in deps.get('fout', {}).items(): fout[int(idx)] = TYPES[t] local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for graphics stages (they are in the input id) - if stage == 5: # compute: the control section's local size, 0 read as 1 - local = [v or 1 for v in struct.unpack_from(' len(data): + return None + control_off, code_off, code_size, control_size = struct.unpack_from(' len(data) or code_off + code_size > len(data): + return None + control = data[control_off:control_off + control_size] + return (control, data[code_off:code_off + code_size]) if control[:4] == CONTROL_MAGIC else None + + +def nvn2_program(data, program): + return any(nvn2_record(data, struct.unpack_from(' len(data): + break + rec = nvn2_record(data, struct.unpack_from('> 16) + if not stage: + continue + control = control[:NVN2_CONTROL] + bytes(0x880 - NVN2_CONTROL) + size, c1_start, c1_end = struct.unpack_from('= size and c1_end >= c1_start and c1_end <= len(code) else size + if not 0 < length <= len(code): + continue + yield digest(control + code[:length]), stage, control, code[:length] + + def bnsh_shaders(data): """BNSH absolute offsets are relative to each embedded BNSH, not BFSHA.""" if data[:4] != b'BNSH' or checked(data, 12, 2) != b'\xff\xfe' or checked(data, 96, 4) != b'grsc': @@ -89,6 +133,9 @@ def bnsh_shaders(data): for i in range(count): program = struct.unpack_from('= 0 and p.content_size != zstandard.CONTENTSIZE_UNKNOWN else None + return FrameInfo(size, p.dict_id) + + def decompress(data, zstd_dict=None): + d = zstandard.ZstdDecompressor(dict_data=zstd_dict._d if zstd_dict else None) + p = zstandard.get_frame_parameters(data) + if 0 <= p.content_size != zstandard.CONTENTSIZE_UNKNOWN: + # one frame of a known size, the common case: decompressed in one call (the streaming path below is + # several times slower on the shader archives' hundreds of megabytes) + frame = zstandard.frame_content_size(data) + if frame == p.content_size: + try: + return d.decompress(data) + except zstandard.ZstdError: + pass # more than one frame: the streaming path + out, pos = [], 0 + # One frame after another, as compression.zstd decompresses concatenated frames. + while pos < len(data): + obj = d.decompressobj() + out.append(obj.decompress(data[pos:])) + rest = obj.unused_data + pos = len(data) - len(rest) + if not rest: + break + return b''.join(out)