"""Every shader program in the game's archives, for the precompile before the first start. python -I nx2/shader_census.py [--out ] Walks romfs/Shader/*.bfsha.zs (and the shader packages inside other archives), and each BNSH inside them. A BNSH's variations are programs: which stages go together, so what comes out are real pairs, not a heap of stages. --out writes them for the back end to translate (gpu/vk/vk_shader.cpp); nothing here goes into the repository: the file is made from the player's own game. The game's compute programs, and many of its graphics ones, are not in a BNSH: agl's shader archives (BAHS, in Lib/ and in Shader/ApplicationPackage), agl's networks (NDSR, Lib/agl), the movie upscaler's (NDVE, Movie/Ndve) and the particle sets (Effect/*.vsetb.byml.zs, each a BNSH inside a byml) are walked too, each in its own way (below). A stage is its control section (0xd8 bytes) and the code area it sizes, the bytes the game hands the GPU: those are what the back end keys by. After tools/shader_census.py of TotK Recomp (github.com/valentderah/totk-pc-recomp, MIT), itself after MacNativeGames' prepare_shader_seed.py (MIT): the BNSH layout and the walk of its variations. BAHS, NDSR and NDVE were read from the stages a play run dumped (RX_VK_DUMP_STAGES), after the first reading of BAHS (the code offset 0x68 bytes before a control section, where the control section follows its own stage record). : "HKSEED01", u32 programs, u32 stages; then the stages, each once: u64 key, u32 stage, u32 control bytes, u32 code bytes, control, code then the programs: u32 count, u32 stage indices """ import argparse import collections import hashlib import os import re import struct import sys import time CONTROL_MAGIC = b"\x34\x12\x76\x98" ZSTD = b"\x28\xb5\x2f\xfd" # the stages a variation's program points at (a 0 offset: none), as far as the structure reaches SLOTS = (8, 16, 24, 32, 40, 48, 56, 64, 72) def dictionaries(romfs, zs): """The game's zstd dictionaries: the files of a SARC, Pack/ZsDic.pack.zs""" raw = zs.ZstdDecompressor().decompress(open(os.path.join(romfs, "Pack", "ZsDic.pack.zs"), "rb").read(), max_output_size=1 << 28) start = struct.unpack_from(" size: stats["no program"] += 1 continue stages = [] for rel in SLOTS: offset = struct.unpack_from(" size: continue control_off, code_off, code_size, control_size = struct.unpack_from(" size or code_off + code_size > size: continue control = data[control_off:control_off + control_size] if control[:4] != CONTROL_MAGIC: continue stage = struct.unpack_from(" 7: return None, 0 return control, struct.unpack_from(" len(data): raise ValueError("stage at %#x" % at) stages.append(stage(control, data[blob + offset:blob + offset + end])) at += step table += base + 0x150 at, heads = table + 8, [] for _ in range(struct.unpack_from(" len(data): continue s = stage(control, data[base + x:base + x + end]) if ct & 15 == 6: yield [s] elif ct & 15 == 0: if graphics: yield graphics graphics = [s] elif graphics: graphics.append(s) if graphics: yield graphics def ndve_programs(data): """The movie upscaler's networks (Movie/Ndve, magic NDVE): an NDSR at +0x28, then tables {u32 stages, u32 blobs, blobs x {u32 offset, u32 bytes}, stages x {u32 control section from the table, u32 code from the control, u32 code bytes, u32 0}}. Each control's entry is found from its code bytes (the pair "bytes, 0") and checked against its table's header.""" if data[0x28:0x2c] == b"NDSR": yield from ndsr_programs(data, 0x28) for m in re.finditer(re.escape(CONTROL_MAGIC), data): c = m.start() control, end = control_at(data, c) if control is None: continue for e in re.finditer(re.escape(struct.pack(" len(data): continue n, blobs = struct.unpack_from("> 20, "MB") if __name__ == "__main__": sys.exit(main())