mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-19 15:09:58 -04:00
Prepare shaders under the code hashes play asks for, from every shader package, with better guessed bind states.
The install step seeded only 63% of the programs a play session met under their run-time code hash. A Switch 2 program with an empty c1 range past its code was hashed with that range, the Switch 2 agl and NDSR packages had no reader, and Layout2Shader/ was not walked. It now seeds 99.3% of the programs play asks for. A default variant's answers are read again from its first translation (runtime/gfx_shader_guess.c): vertex inputs the shader reads as integers become uint, and a compute shader takes the texture and image types its translation declares. Task and mesh defaults carry the workgroup words play answers with. Against a play session's recorded variants, --prepare-shaders now matches 98.3%; the plain default guess matched 89.5%. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -13,7 +13,10 @@ build's nx2navi.exe in the record layouts of runtime/gfx_shader.c ("answers met
|
||||
seed-answers.bin "NX2NSANS": stage, code hash, input id (zero: the runtime computes it), answers, priority
|
||||
|
||||
Priority (the pad byte): 0 shaders the list's warm rows flag as title/menu (t), 1 other warm rows (S, C, R), 2 the
|
||||
rest; a default variant takes its shader's priority. Then it runs `nx2navi.exe --prepare-shaders`, which translates every record into the build's shader cache, writes
|
||||
rest; a default variant takes its shader's priority and sets DEFAULT_FLAG (bit 7). The runtime reads a flagged
|
||||
record's first translation and translates it again with better answers where it reads other ones
|
||||
(runtime/gfx_shader_guess.c): integer vertex inputs, and the texture and image types a compute shader's translation
|
||||
declares, for which a compute default variant goes without texture answers. Then it runs `nx2navi.exe --prepare-shaders`, which translates every record into the build's shader cache, writes
|
||||
shader-prepared.bin (one record per variant, in priority order) and builds every variant's stage library or compute
|
||||
pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files
|
||||
can be deleted afterwards.
|
||||
@@ -59,6 +62,7 @@ TYPES = {'float': 0, 'sint': 1, 'uint': 2}
|
||||
DEFAULT = {'tex': {str(slot): '2d' for slot in range(32)}} # answers outside a shader's use mask are dropped
|
||||
CONTROL = 0x880
|
||||
ANS_REC = 4 + 32 + 32 + 147 + 1
|
||||
DEFAULT_FLAG = 0x80 # pad byte: a default variant, its answers refined from its first translation (gfx_shader_guess.c)
|
||||
JOURNAL = 'shader-prepare-journal.bin' # vk_pipeline.c skip_init: a 64-byte slot per worker, input id first
|
||||
RETRIES = 3
|
||||
|
||||
@@ -121,10 +125,12 @@ def answers(deps, stage, control):
|
||||
attr[int(loc)] = TYPES[t]
|
||||
for idx, t in deps.get('fout', {}).items():
|
||||
fout[int(idx)] = TYPES[t]
|
||||
local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for graphics stages (they are in the input id)
|
||||
if stage == 5: # compute: the control section's local size (NVN2 at +0x58), 0 read as 1
|
||||
local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for vertex and fragment programs (they are in the input id)
|
||||
if stage not in (STAGES['vs'], STAGES['ps']):
|
||||
# compute, task and mesh: the three words at the control section's local size (NVN2 at +0x58), 0 read as 1,
|
||||
# each cut to 16 bits as answers_for stores them (task and mesh programs keep other fields there)
|
||||
local = struct.unpack_from('<III', control, 0x58 if nvn2(control) else 0x748)
|
||||
local = [1, 1, 1] if not any(local) else [v or 1 for v in local]
|
||||
local = [(v or 1) & 0xffff for v in local]
|
||||
packed = bytes(tex + img + fmt + attr + fout) + struct.pack('<HHHIB', *local, 0, 0)
|
||||
assert len(packed) == 147
|
||||
return packed
|
||||
@@ -136,11 +142,18 @@ def nvn2(control):
|
||||
|
||||
|
||||
def bounds(control, code):
|
||||
"""(c1 offset, length) as runtime/gfx_shader.c req_init takes them."""
|
||||
"""(c1 offset, length) as runtime/gfx_shader.c req_init takes them.
|
||||
|
||||
NVN2: play hands req_init the code size of runtime/vk_exec_draw.c ctl_stage_len, which reaches the end of the c1
|
||||
constants only when there are some (+0x40 below +0x44). With an empty c1 range past the code the length is the code
|
||||
size alone; cutting at the range's end (as shader_census does) seeded most of Ocarina's vertex programs under keys
|
||||
play never asks for."""
|
||||
if nvn2(control):
|
||||
size, c1_start = struct.unpack_from('<I', control, 0x38)[0], struct.unpack_from('<I', control, 0x40)[0]
|
||||
length = len(code) # shader_census cut the code at the end of its c1 part
|
||||
return (c1_start if size <= c1_start <= length else length), length
|
||||
size, c1_start, c1_end = struct.unpack_from('<III', control, 0x38)[0], *struct.unpack_from('<II', control, 0x40)
|
||||
code_size = c1_end if c1_start >= size and c1_end > c1_start and c1_end - size < 1 << 20 else size
|
||||
if c1_start >= size and c1_end >= c1_start and c1_end <= min(code_size, len(code)):
|
||||
return c1_start, c1_end
|
||||
return min(size, len(code)), min(size, len(code))
|
||||
return struct.unpack_from('<II', control, 0x700)
|
||||
|
||||
|
||||
@@ -200,11 +213,12 @@ def main():
|
||||
h = code_hash(stage, control, code, length, c1)
|
||||
progs[key] = (h, struct.pack('<III', stage, length, len(c1)) + h + control + code[:length] + c1)
|
||||
h = progs[key][0]
|
||||
packed = answers(deps, stage, control)
|
||||
default = deps is DEFAULT
|
||||
packed = answers({} if default and stage == STAGES['cs'] else deps, stage, control)
|
||||
if (h, packed) in seen: # a list row of the default bind state
|
||||
continue
|
||||
seen.add((h, packed))
|
||||
rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2)])
|
||||
rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2) | (DEFAULT_FLAG if default else 0)])
|
||||
assert len(rec) == ANS_REC
|
||||
recs.append((prio.get(key, 2), rec))
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
|
||||
@@ -8,7 +8,7 @@ a control section must start with CONTROL_MAGIC (sharcb_shaders says why).
|
||||
Reads build/extracted/<version>/ (RomFS image plus index.json as tools/romfs_index.py reads them, and exefs/). Writes under
|
||||
build/<version>/shaders/ only:
|
||||
code/<key>.code translator input (graphics: SPH + instructions, compute: instructions)
|
||||
jobs.json key -> job, source path, origin (bnsh|sharcb|main)
|
||||
jobs.json key -> job, source path, origin (bnsh|sharcb|ndsr|main)
|
||||
spv/<key>.spv with --translate
|
||||
failures.txt with --translate, when something failed: groups by message
|
||||
|
||||
@@ -173,7 +173,7 @@ def sharcb_shaders(data, rejected):
|
||||
if not 32 <= start <= pool or pool + pool_size > len(data):
|
||||
raise ValueError('agl shader package layout outside the file')
|
||||
seen = set()
|
||||
for size in (b'\x80\x08\x00\x00', b'\x8c\x08\x00\x00'):
|
||||
for size in (b'\x80\x08\x00\x00', b'\x8c\x08\x00\x00', b'\xe0\x00\x00\x00'):
|
||||
at = start
|
||||
while (at := data.find(size, at, pool)) >= 0:
|
||||
record, at = at - 8, at + 4
|
||||
@@ -181,6 +181,12 @@ def sharcb_shaders(data, rejected):
|
||||
continue
|
||||
code_off = struct.unpack_from('<Q', data, record)[0]
|
||||
control_at = record + struct.unpack_from('<i', data, record + 12)[0]
|
||||
if size[0] == NVN2_CONTROL: # NVN2 (Switch 2) packages: the same records with a 0xe0-byte control section
|
||||
found = nvn2_sharcb_record(data, start, pool, pool_size, code_off, control_at)
|
||||
if found and found[0] not in seen:
|
||||
seen.add(found[0])
|
||||
yield found
|
||||
continue
|
||||
if not start <= control_at <= pool - 0x880 or code_off >= pool_size:
|
||||
continue
|
||||
control = data[control_at:control_at + 0x880]
|
||||
@@ -198,6 +204,52 @@ def sharcb_shaders(data, rejected):
|
||||
yield key, stage, control, code
|
||||
|
||||
|
||||
def ndsr_shaders(data):
|
||||
"""NDSR packages (Lib/agl/agl_ndsr*.pck: compute programs). Items of 8 bytes, <u32 offset, u32 size << 4 | kind>:
|
||||
an NVN2 control section is kind 6 with size 0xe0 and the item after it, kind 7, is its code."""
|
||||
if data[:4] != b'NDSR':
|
||||
raise ValueError('expected an NDSR package')
|
||||
seen = set()
|
||||
for at in range(4, len(data) - 16, 4):
|
||||
off, word, code_off, code_word = struct.unpack_from('<4I', data, at)
|
||||
if word != NVN2_CONTROL << 4 | 6 or code_word & 15 != 7 or off + NVN2_CONTROL > len(data):
|
||||
continue
|
||||
control = data[off:off + NVN2_CONTROL]
|
||||
stage = NVN2_STAGES.get(struct.unpack_from('<I', control, 0x54)[0] >> 16)
|
||||
room = code_word >> 4
|
||||
if control[:4] != CONTROL_MAGIC or not stage or code_off + room > len(data):
|
||||
continue
|
||||
control += bytes(0x880 - NVN2_CONTROL)
|
||||
size, c1_start, c1_end = struct.unpack_from('<I', control, 0x38)[0], *struct.unpack_from('<II', control, 0x40)
|
||||
length = c1_end if c1_start >= size and c1_end >= c1_start and c1_end <= room else size
|
||||
if not 0 < length <= room:
|
||||
continue
|
||||
code = data[code_off:code_off + length]
|
||||
key = digest(control + code)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
yield key, stage, control, code
|
||||
|
||||
|
||||
def nvn2_sharcb_record(data, start, pool, pool_size, code_off, control_at):
|
||||
"""(key, stage, control, code) of an NVN2 .sharcb record, as nvn2_stages cuts them, or None. A size word of 0xe0
|
||||
is common, so a candidate counts only with the control magic and a known stage."""
|
||||
if not start <= control_at <= pool - NVN2_CONTROL or code_off >= pool_size:
|
||||
return None
|
||||
control = data[control_at:control_at + NVN2_CONTROL]
|
||||
stage = NVN2_STAGES.get(struct.unpack_from('<I', control, 0x54)[0] >> 16)
|
||||
if control[:4] != CONTROL_MAGIC or not stage:
|
||||
return None
|
||||
control += bytes(0x880 - NVN2_CONTROL)
|
||||
size, c1_start, c1_end = struct.unpack_from('<I', control, 0x38)[0], *struct.unpack_from('<II', control, 0x40)
|
||||
room = pool_size - code_off
|
||||
length = c1_end if c1_start >= size and c1_end >= c1_start and c1_end <= room else size
|
||||
if not 0 < length <= room:
|
||||
return None
|
||||
code = data[pool + code_off:pool + code_off + length]
|
||||
return digest(control + code), stage, control, code
|
||||
|
||||
|
||||
def nso_image(path):
|
||||
"""The decompressed text, rodata and data segments of an NSO, concatenated (enough to find embedded BNSH files)."""
|
||||
raw = path.read_bytes()
|
||||
@@ -222,7 +274,8 @@ def startup_package(name):
|
||||
# The RomFS directories that can hold shaders: the agl and gsys packages, the effects, terrain, the UI layout archives
|
||||
# and the game's own shader packs. Every shader of 1.1.0 (27,576) comes from these six; they are 1,996 entries and
|
||||
# 147 MiB, 0.9% of a RomFS whose every file the walk below used to read and decompress first (65 s against 2.8 s).
|
||||
SHADER_DIRS = ('Shader/', 'Effect/', 'Lib/agl/', 'Lib/Terrain/', 'Lib/gsys/', 'UI/LayoutArchive/')
|
||||
# Ocarina keeps its UI layout shaders (MergeShader) in Layout2Shader/.
|
||||
SHADER_DIRS = ('Shader/', 'Effect/', 'Lib/agl/', 'Lib/Terrain/', 'Lib/gsys/', 'UI/LayoutArchive/', 'Layout2Shader/')
|
||||
|
||||
|
||||
def inventory(romfs, exefs, everything=True):
|
||||
@@ -268,6 +321,9 @@ def inventory(romfs, exefs, everything=True):
|
||||
if data[:4] == b'BAHS':
|
||||
add('sharcb', name, sharcb_shaders(data, rejected), True)
|
||||
return
|
||||
if data[:4] == b'NDSR':
|
||||
add('ndsr', name, ndsr_shaders(data), True)
|
||||
return
|
||||
if data[:4] == b'SARC':
|
||||
if depth > 4:
|
||||
return
|
||||
@@ -284,7 +340,7 @@ def inventory(romfs, exefs, everything=True):
|
||||
if not everything and not name.startswith(SHADER_DIRS):
|
||||
continue
|
||||
raw = read(name)
|
||||
if raw[:4] not in (ZSTD, b'SARC', b'FSHA', b'BNSH', b'BAHS') and b'BNSH\0\0\0\0' not in raw:
|
||||
if raw[:4] not in (ZSTD, b'SARC', b'FSHA', b'BNSH', b'BAHS', b'NDSR') and b'BNSH\0\0\0\0' not in raw:
|
||||
continue
|
||||
visit('bnsh', name, decode(raw) if raw[:4] == ZSTD else raw, startup_package(name))
|
||||
for module in sorted(exefs.iterdir()):
|
||||
@@ -387,10 +443,10 @@ def main():
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
shaders, rejected = inventory(extracted / 'romfs', extracted / 'exefs')
|
||||
jobs = write_shaders(shaders, out, args.all)
|
||||
a, b, c = (sum(1 for j in jobs.values() if j['origin'] == o) for o in ('bnsh', 'sharcb', 'main'))
|
||||
a, b, n, c = (sum(1 for j in jobs.values() if j['origin'] == o) for o in ('bnsh', 'sharcb', 'ndsr', 'main'))
|
||||
print(f'{len(shaders)} unique shaders found in all, {len(jobs)} selected; {rejected} .sharcb record candidates '
|
||||
f'without a control section rejected', flush=True)
|
||||
summary = f'shaders {len(jobs)} (bnsh {a}, sharcb {b}, main {c})'
|
||||
summary = f'shaders {len(jobs)} (bnsh {a}, sharcb {b}, ndsr {n}, main {c})'
|
||||
print(f'extracted {summary} in {time.monotonic() - t0:.1f} s', flush=True)
|
||||
if not args.translate:
|
||||
print(f'{summary}, total {time.monotonic() - t0:.1f} s')
|
||||
|
||||
Reference in New Issue
Block a user