Prepare shaders under the code hashes play asks for, from every shader package, with better guessed bind states.

The install step seeded only 63% of the programs a play session met under their run-time code hash. A Switch 2 program with an empty c1 range past its code was hashed with that range, the Switch 2 agl and NDSR packages had no reader, and Layout2Shader/ was not walked. It now seeds 99.3% of the programs play asks for.

A default variant's answers are read again from its first translation (runtime/gfx_shader_guess.c): vertex inputs the shader reads as integers become uint, and a compute shader takes the texture and image types its translation declares. Task and mesh defaults carry the workgroup words play answers with. Against a play session's recorded variants, --prepare-shaders now matches 98.3%; the plain default guess matched 89.5%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-07 22:21:28 -05:00
parent d0da862c87
commit fc8a9523a4
5 changed files with 204 additions and 23 deletions

View File

@@ -79,6 +79,10 @@ void gfx_shader_prepare_recorded(int threads, int procs);
// nx2navi.exe --prepare-shaders: translates <dir>/seed-answers.bin + seed-programs.bin (tools/prepare_shaders.py) and // nx2navi.exe --prepare-shaders: translates <dir>/seed-answers.bin + seed-programs.bin (tools/prepare_shaders.py) and
// writes <dir>/shader-prepared.bin (stage, priority, input id per variant). 0 on success. // writes <dir>/shader-prepared.bin (stage, priority, input id per variant). 0 on success.
int gfx_shader_prepare_seed(const char *dir, int threads, int procs); int gfx_shader_prepare_seed(const char *dir, int threads, int procs);
// gfx_shader_guess.c: the answers of a prepared default variant, read from its first translation s: vertex inputs the
// shader reads as integers become uint, a compute shader's textures and images take the types its translation
// declares. 1 when *out differs from *a (translate again with *out).
int gfx_shader_guess_refine(const GfxShader *s, int stage, const GfxShaderAnswers *a, GfxShaderAnswers *out);
typedef struct { uint8_t iid[32]; uint8_t stage, prio; } GfxWarmEnt; // prio 0: met in play; 1..3: prepared typedef struct { uint8_t iid[32]; uint8_t stage, prio; } GfxWarmEnt; // prio 0: met in play; 1..3: prepared
size_t gfx_shader_warm_list(GfxWarmEnt **out); // malloc'd; recorded first, then prepared; needs recording on size_t gfx_shader_warm_list(GfxWarmEnt **out); // malloc'd; recorded first, then prepared; needs recording on
// Disk-cache load the caller owns (*owned 1: release it), or the memory cache's entry (*owned 0). NULL if not cached. // Disk-cache load the caller owns (*owned 1: release it), or the memory cache's entry (*owned 0). NULL if not cached.

View File

@@ -2118,7 +2118,10 @@ void gfx_shader_record_flush(int force) {
// ---- preparation from record files: the answers met in play (NX2NAVI_TEST_PREPARE_SHADERS=1) or the seed that // ---- preparation from record files: the answers met in play (NX2NAVI_TEST_PREPARE_SHADERS=1) or the seed that
// tools/prepare_shaders.py writes from its shader list (nx2navi.exe --prepare-shaders). Both use the record // tools/prepare_shaders.py writes from its shader list (nx2navi.exe --prepare-shaders). Both use the record
// layouts above; a seed record has a zero input id (the use mask, hence the id, is the translator's to find) and its // layouts above; a seed record has a zero input id (the use mask, hence the id, is the translator's to find) and its
// pad byte holds the list priority (0 title, 1 warm-up, 2 rest). // pad byte holds the list priority (0 title, 1 warm-up, 2 rest), with SEED_DEFAULT set on a default variant: its
// answers are read again from its first translation (gfx_shader_guess.c) and the variant it resolves to is the
// translation with those, when they differ.
#define SEED_DEFAULT 0x80 // tools/prepare_shaders.py DEFAULT_FLAG
typedef struct { const uint8_t *rec; } PrgIdx; typedef struct { const uint8_t *rec; } PrgIdx;
typedef struct { PrgIdx *idx; size_t n; } PrgList; typedef struct { PrgIdx *idx; size_t n; } PrgList;
typedef struct { typedef struct {
@@ -2126,7 +2129,7 @@ typedef struct {
const uint8_t *ans; const uint8_t *ans;
size_t n_ans; size_t n_ans;
uint8_t (*iid)[32]; // per answers record: the resolved input id, zero on failure uint8_t (*iid)[32]; // per answers record: the resolved input id, zero on failure
volatile int64_t next, done, missing, failed; volatile int64_t next, done, missing, failed, refined;
} Prep; } Prep;
static int prg_cmp(const void *x, const void *y) { return memcmp(((const PrgIdx *)x)->rec + 12, ((const PrgIdx *)y)->rec + 12, 32); } static int prg_cmp(const void *x, const void *y) { return memcmp(((const PrgIdx *)x)->rec + 12, ((const PrgIdx *)y)->rec + 12, 32); }
@@ -2152,9 +2155,12 @@ static void prep_worker(void *arg) {
uint32_t st, len; uint32_t st, len;
memcpy(&st, e->rec, 4), memcpy(&len, e->rec + 4, 4); memcpy(&st, e->rec, 4), memcpy(&len, e->rec + 4, 4);
const uint8_t *control = e->rec + 44, *code = control + CONTROL_SIZE, *c1 = code + len; const uint8_t *control = e->rec + 44, *code = control + CONTROL_SIZE, *c1 = code + len;
GfxShaderAnswers a; GfxShaderAnswers a, b;
unpack_answers(r + 68, &a); unpack_answers(r + 68, &a);
const GfxShader *s = gfx_shader_get((int)st, control, code, len, c1, &a); const GfxShader *s = gfx_shader_get((int)st, control, code, len, c1, &a), *t;
if (s && r[ANS_REC - 1] & SEED_DEFAULT && gfx_shader_guess_refine(s, (int)st, &a, &b) &&
(t = gfx_shader_get((int)st, control, code, len, c1, &b)))
s = t, plat_inc64(&p->refined);
if (!s) plat_inc64(&p->failed); if (!s) plat_inc64(&p->failed);
else if (p->iid) memcpy(p->iid[i], s->iid, 32); else if (p->iid) memcpy(p->iid[i], s->iid, 32);
plat_inc64(&p->done); plat_inc64(&p->done);
@@ -2195,8 +2201,9 @@ static long long prepare_files(const char *ans_path, const char *prg_path, int t
(unsigned long long)((uint64_t)n_xlated - xl0), (double)plat_now_ms() - t0); (unsigned long long)((uint64_t)n_xlated - xl0), (double)plat_now_ms() - t0);
svc_set_limit(svc_runtime_limit()); svc_set_limit(svc_runtime_limit());
rt_log("gfx_shader: prepare: %zu %s of %zu programs: %llu translated, %lld failed, %lld without a program, the rest " rt_log("gfx_shader: prepare: %zu %s of %zu programs: %llu translated, %lld failed, %lld without a program, the rest "
"cached; %.0f ms", p.n_ans, what, p.progs.n, (unsigned long long)((uint64_t)n_xlated - xl0), (long long)p.failed, "cached; %lld default variants with refined answers; %.0f ms", p.n_ans, what, p.progs.n,
(long long)p.missing, (double)plat_now_ms() - t0); (unsigned long long)((uint64_t)n_xlated - xl0), (long long)p.failed, (long long)p.missing, (long long)p.refined,
(double)plat_now_ms() - t0);
free(p.progs.idx), free(pb); free(p.progs.idx), free(pb);
if (ans_out) *ans_out = ab; if (ans_out) *ans_out = ab;
else free(ab); else free(ab);
@@ -2236,7 +2243,8 @@ int gfx_shader_prepare_seed(const char *dir, int threads, int procs) {
if (!memcmp(iid[i], zero, 32) || !set_add(&seen, hash8(iid[i]))) continue; if (!memcmp(iid[i], zero, 32) || !set_add(&seen, hash8(iid[i]))) continue;
uint8_t *r = recs + u * PRP_REC; uint8_t *r = recs + u * PRP_REC;
const uint8_t *a = ab + (size_t)i * ANS_REC; const uint8_t *a = ab + (size_t)i * ANS_REC;
r[0] = a[0] < 6 ? a[0] : 0, r[1] = a[ANS_REC - 1] > 2 ? 2 : a[ANS_REC - 1]; uint8_t prio = a[ANS_REC - 1] & ~SEED_DEFAULT;
r[0] = a[0] < 6 ? a[0] : 0, r[1] = prio > 2 ? 2 : prio;
memcpy(r + 4, iid[i], 32); memcpy(r + 4, iid[i], 32);
by[r[1]]++, stages[r[0]]++, u++; by[r[1]]++, stages[r[0]]++, u++;
} }

View File

@@ -0,0 +1,99 @@
// Bind-state answers for a prepared default variant, read from its first translation (gfx_shader.c prep_worker, for
// seed records tools/prepare_shaders.py flags as default variants). A default variant guesses the state play binds:
// every texture a 2D one, float vertex inputs and colour outputs. Against the 3,093 (program, answers) pairs a play
// session of Ocarina recorded, that guess matched 89.5%: vertex 77.3%, compute 88.1%, fragment 99.6%. With two readings
// of the first translation, one more translation for the programs they change, --prepare-shaders matched 98.3% of them
// (vertex 97.1%, compute 96.0%, fragment 99.6%; last-frame vertex variants, vk_mvprev.c, are not prepared):
//
// - Vertex inputs. A vertex attribute without an answer is declared float, and a shader that reads it as an integer
// then bitcasts the loaded value to an integer type. Such inputs are answered uint: of the 313 integer inputs met
// in play, all were uint and all were bitcast, against 36 of 3,339 float ones.
// - Compute textures and images. Translated without texture answers (the seed gives a compute default variant none),
// the translator declares each texture and image with the type its instructions name; those types become the
// answers, image formats included, and a slot it queries without declaring one keeps the 2D default. Compute misses
// dropped from 43 to 13. A compute shader play meets untranslated is translated and its pipeline built on the replay
// thread (vk_exec_draw.c, the dispatch's shader_for call), so each miss is a stall; the prepare step's driver builds
// of compute pipelines averaged 0.96 s each, 16 at a time, on an RTX 5090.
#include <stdlib.h>
#include <string.h>
#include "gfx.h"
enum { OP_TYPE_INT = 21, OP_TYPE_VECTOR = 23, OP_VARIABLE = 59, OP_LOAD = 61, OP_ACCESS_CHAIN = 65,
OP_IN_BOUNDS_ACCESS_CHAIN = 66, OP_DECORATE = 71, OP_BITCAST = 124 };
enum { DEC_LOCATION = 30, SC_INPUT = 1 };
#define ID_CAP (1u << 20)
// Input Locations (bit mask) whose loaded value some OpBitcast turns into an integer scalar or vector.
static uint32_t int_read_inputs(const uint32_t *w, uint64_t n) {
if (n < 5) return 0;
uint32_t bound = w[3] < ID_CAP ? w[3] : ID_CAP, found = 0;
int8_t *loc = malloc(bound); // per id: the input Location it reads, -1 none
uint8_t *is_int = calloc(bound, 1), *located = calloc(bound, 1);
int8_t *dec = malloc(bound); // Location decorations
if (!loc || !is_int || !located || !dec) goto out;
memset(loc, -1, bound), memset(dec, -1, bound);
for (uint64_t i = 5; i < n;) { // declarations come before the code that uses them
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
if (!len || i + len > n) break;
const uint32_t *a = w + i + 1;
if (op == OP_DECORATE && len >= 4 && a[0] < bound && a[1] == DEC_LOCATION && a[2] < 32) dec[a[0]] = (int8_t)a[2];
else if (op == OP_TYPE_INT && len >= 2 && a[0] < bound) is_int[a[0]] = 1;
else if (op == OP_TYPE_VECTOR && len >= 3 && a[0] < bound && a[1] < bound) is_int[a[0]] = is_int[a[1]];
else if (op == OP_VARIABLE && len >= 4 && a[1] < bound && a[2] == SC_INPUT && dec[a[1]] >= 0)
loc[a[1]] = dec[a[1]], located[a[1]] = 1;
else if ((op == OP_ACCESS_CHAIN || op == OP_IN_BOUNDS_ACCESS_CHAIN) && len >= 4 && a[1] < bound && a[2] < bound &&
located[a[2]])
loc[a[1]] = loc[a[2]], located[a[1]] = 1;
else if (op == OP_LOAD && len >= 4 && a[1] < bound && a[2] < bound && located[a[2]])
loc[a[1]] = loc[a[2]]; // a loaded value: not a pointer, so chains do not continue from it
else if (op == OP_BITCAST && len >= 4 && a[0] < bound && a[2] < bound && is_int[a[0]] && !located[a[2]] &&
loc[a[2]] >= 0)
found |= 1u << loc[a[2]];
i += len;
}
out:
free(loc), free(is_int), free(located), free(dec);
return found;
}
// The translator's SamplerType (GfxShaderRes.type) as a GfxShaderAnswers tex_type / img_type code; 0 unknown.
static uint8_t answer_type(uint16_t t) {
int arr = t & 0x100, ms = t & 0x200;
switch (t & 0xff) {
case 1: return arr ? GFX_TEX_1D_ARRAY : GFX_TEX_1D;
case 2: return GFX_TEX_BUFFER;
case 3: return ms ? (arr ? GFX_TEX_2D_MS_ARRAY : GFX_TEX_2D_MS) : arr ? GFX_TEX_2D_ARRAY : GFX_TEX_2D;
case 4: return GFX_TEX_3D;
case 5: return arr ? GFX_TEX_CUBE_ARRAY : GFX_TEX_CUBE;
default: return 0;
}
}
int gfx_shader_guess_refine(const GfxShader *s, int stage, const GfxShaderAnswers *a, GfxShaderAnswers *out) {
*out = *a;
if (!s) return 0;
if (stage == GFX_STAGE_VS) {
uint64_t words = 0;
const uint32_t *w = gfx_shader_spirv(s, &words);
GfxShaderIo io[32];
uint32_t ints = w ? int_read_inputs(w, words) & gfx_shader_inputs(s, io) : 0;
for (int l = 0; l < 32; l++)
if (ints >> l & 1 && out->attr_type[l] == GFX_TYPE_FLOAT) out->attr_type[l] = GFX_TYPE_UINT;
} else if (stage == GFX_STAGE_CS) {
// a slot the translation queries without declaring a texture there (a bindless one) keeps the 2D default
for (int l = 0; l < 32; l++)
if (!out->tex_type[l]) out->tex_type[l] = GFX_TEX_2D;
const GfxShaderRes *r;
int n = gfx_shader_resources(s, GFX_RES_TEX, &r);
for (int i = 0; i < n; i++)
if (r[i].slot >= 0 && r[i].slot < 32 && answer_type(r[i].type)) out->tex_type[r[i].slot] = answer_type(r[i].type);
n = gfx_shader_resources(s, GFX_RES_IMG, &r);
for (int i = 0; i < n; i++)
if (r[i].slot >= 0 && r[i].slot < 32 && answer_type(r[i].type)) {
out->img_type[r[i].slot] = answer_type(r[i].type);
if (r[i].format) out->img_format[r[i].slot] = r[i].format;
}
}
return memcmp(out, a, sizeof *a) != 0;
}

View File

@@ -13,7 +13,10 @@ build's nx2navi.exe in the record layouts of runtime/gfx_shader.c ("answers met
seed-answers.bin "NX2NSANS": stage, code hash, input id (zero: the runtime computes it), answers, priority seed-answers.bin "NX2NSANS": stage, code hash, input id (zero: the runtime computes it), answers, priority
Priority (the pad byte): 0 shaders the list's warm rows flag as title/menu (t), 1 other warm rows (S, C, R), 2 the Priority (the pad byte): 0 shaders the list's warm rows flag as title/menu (t), 1 other warm rows (S, C, R), 2 the
rest; a default variant takes its shader's priority. Then it runs `nx2navi.exe --prepare-shaders`, which translates every record into the build's shader cache, writes rest; a default variant takes its shader's priority and sets DEFAULT_FLAG (bit 7). The runtime reads a flagged
record's first translation and translates it again with better answers where it reads other ones
(runtime/gfx_shader_guess.c): integer vertex inputs, and the texture and image types a compute shader's translation
declares, for which a compute default variant goes without texture answers. Then it runs `nx2navi.exe --prepare-shaders`, which translates every record into the build's shader cache, writes
shader-prepared.bin (one record per variant, in priority order) and builds every variant's stage library or compute shader-prepared.bin (one record per variant, in priority order) and builds every variant's stage library or compute
pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files
can be deleted afterwards. can be deleted afterwards.
@@ -59,6 +62,7 @@ TYPES = {'float': 0, 'sint': 1, 'uint': 2}
DEFAULT = {'tex': {str(slot): '2d' for slot in range(32)}} # answers outside a shader's use mask are dropped DEFAULT = {'tex': {str(slot): '2d' for slot in range(32)}} # answers outside a shader's use mask are dropped
CONTROL = 0x880 CONTROL = 0x880
ANS_REC = 4 + 32 + 32 + 147 + 1 ANS_REC = 4 + 32 + 32 + 147 + 1
DEFAULT_FLAG = 0x80 # pad byte: a default variant, its answers refined from its first translation (gfx_shader_guess.c)
JOURNAL = 'shader-prepare-journal.bin' # vk_pipeline.c skip_init: a 64-byte slot per worker, input id first JOURNAL = 'shader-prepare-journal.bin' # vk_pipeline.c skip_init: a 64-byte slot per worker, input id first
RETRIES = 3 RETRIES = 3
@@ -121,10 +125,12 @@ def answers(deps, stage, control):
attr[int(loc)] = TYPES[t] attr[int(loc)] = TYPES[t]
for idx, t in deps.get('fout', {}).items(): for idx, t in deps.get('fout', {}).items():
fout[int(idx)] = TYPES[t] fout[int(idx)] = TYPES[t]
local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for graphics stages (they are in the input id) local = [0, 0, 0] # vk_exec_draw.c answers_for: zero for vertex and fragment programs (they are in the input id)
if stage == 5: # compute: the control section's local size (NVN2 at +0x58), 0 read as 1 if stage not in (STAGES['vs'], STAGES['ps']):
# compute, task and mesh: the three words at the control section's local size (NVN2 at +0x58), 0 read as 1,
# each cut to 16 bits as answers_for stores them (task and mesh programs keep other fields there)
local = struct.unpack_from('<III', control, 0x58 if nvn2(control) else 0x748) local = struct.unpack_from('<III', control, 0x58 if nvn2(control) else 0x748)
local = [1, 1, 1] if not any(local) else [v or 1 for v in local] local = [(v or 1) & 0xffff for v in local]
packed = bytes(tex + img + fmt + attr + fout) + struct.pack('<HHHIB', *local, 0, 0) packed = bytes(tex + img + fmt + attr + fout) + struct.pack('<HHHIB', *local, 0, 0)
assert len(packed) == 147 assert len(packed) == 147
return packed return packed
@@ -136,11 +142,18 @@ def nvn2(control):
def bounds(control, code): def bounds(control, code):
"""(c1 offset, length) as runtime/gfx_shader.c req_init takes them.""" """(c1 offset, length) as runtime/gfx_shader.c req_init takes them.
NVN2: play hands req_init the code size of runtime/vk_exec_draw.c ctl_stage_len, which reaches the end of the c1
constants only when there are some (+0x40 below +0x44). With an empty c1 range past the code the length is the code
size alone; cutting at the range's end (as shader_census does) seeded most of Ocarina's vertex programs under keys
play never asks for."""
if nvn2(control): if nvn2(control):
size, c1_start = struct.unpack_from('<I', control, 0x38)[0], struct.unpack_from('<I', control, 0x40)[0] size, c1_start, c1_end = struct.unpack_from('<III', control, 0x38)[0], *struct.unpack_from('<II', control, 0x40)
length = len(code) # shader_census cut the code at the end of its c1 part code_size = c1_end if c1_start >= size and c1_end > c1_start and c1_end - size < 1 << 20 else size
return (c1_start if size <= c1_start <= length else length), length if c1_start >= size and c1_end >= c1_start and c1_end <= min(code_size, len(code)):
return c1_start, c1_end
return min(size, len(code)), min(size, len(code))
return struct.unpack_from('<II', control, 0x700) return struct.unpack_from('<II', control, 0x700)
@@ -200,11 +213,12 @@ def main():
h = code_hash(stage, control, code, length, c1) h = code_hash(stage, control, code, length, c1)
progs[key] = (h, struct.pack('<III', stage, length, len(c1)) + h + control + code[:length] + c1) progs[key] = (h, struct.pack('<III', stage, length, len(c1)) + h + control + code[:length] + c1)
h = progs[key][0] h = progs[key][0]
packed = answers(deps, stage, control) default = deps is DEFAULT
packed = answers({} if default and stage == STAGES['cs'] else deps, stage, control)
if (h, packed) in seen: # a list row of the default bind state if (h, packed) in seen: # a list row of the default bind state
continue continue
seen.add((h, packed)) seen.add((h, packed))
rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2)]) rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2) | (DEFAULT_FLAG if default else 0)])
assert len(rec) == ANS_REC assert len(rec) == ANS_REC
recs.append((prio.get(key, 2), rec)) recs.append((prio.get(key, 2), rec))
recs.sort(key=lambda r: r[0]) # stable: list order within a priority recs.sort(key=lambda r: r[0]) # stable: list order within a priority

View File

@@ -8,7 +8,7 @@ a control section must start with CONTROL_MAGIC (sharcb_shaders says why).
Reads build/extracted/<version>/ (RomFS image plus index.json as tools/romfs_index.py reads them, and exefs/). Writes under Reads build/extracted/<version>/ (RomFS image plus index.json as tools/romfs_index.py reads them, and exefs/). Writes under
build/<version>/shaders/ only: build/<version>/shaders/ only:
code/<key>.code translator input (graphics: SPH + instructions, compute: instructions) code/<key>.code translator input (graphics: SPH + instructions, compute: instructions)
jobs.json key -> job, source path, origin (bnsh|sharcb|main) jobs.json key -> job, source path, origin (bnsh|sharcb|ndsr|main)
spv/<key>.spv with --translate spv/<key>.spv with --translate
failures.txt with --translate, when something failed: groups by message failures.txt with --translate, when something failed: groups by message
@@ -173,7 +173,7 @@ def sharcb_shaders(data, rejected):
if not 32 <= start <= pool or pool + pool_size > len(data): if not 32 <= start <= pool or pool + pool_size > len(data):
raise ValueError('agl shader package layout outside the file') raise ValueError('agl shader package layout outside the file')
seen = set() seen = set()
for size in (b'\x80\x08\x00\x00', b'\x8c\x08\x00\x00'): for size in (b'\x80\x08\x00\x00', b'\x8c\x08\x00\x00', b'\xe0\x00\x00\x00'):
at = start at = start
while (at := data.find(size, at, pool)) >= 0: while (at := data.find(size, at, pool)) >= 0:
record, at = at - 8, at + 4 record, at = at - 8, at + 4
@@ -181,6 +181,12 @@ def sharcb_shaders(data, rejected):
continue continue
code_off = struct.unpack_from('<Q', data, record)[0] code_off = struct.unpack_from('<Q', data, record)[0]
control_at = record + struct.unpack_from('<i', data, record + 12)[0] control_at = record + struct.unpack_from('<i', data, record + 12)[0]
if size[0] == NVN2_CONTROL: # NVN2 (Switch 2) packages: the same records with a 0xe0-byte control section
found = nvn2_sharcb_record(data, start, pool, pool_size, code_off, control_at)
if found and found[0] not in seen:
seen.add(found[0])
yield found
continue
if not start <= control_at <= pool - 0x880 or code_off >= pool_size: if not start <= control_at <= pool - 0x880 or code_off >= pool_size:
continue continue
control = data[control_at:control_at + 0x880] control = data[control_at:control_at + 0x880]
@@ -198,6 +204,52 @@ def sharcb_shaders(data, rejected):
yield key, stage, control, code yield key, stage, control, code
def ndsr_shaders(data):
"""NDSR packages (Lib/agl/agl_ndsr*.pck: compute programs). Items of 8 bytes, <u32 offset, u32 size << 4 | kind>:
an NVN2 control section is kind 6 with size 0xe0 and the item after it, kind 7, is its code."""
if data[:4] != b'NDSR':
raise ValueError('expected an NDSR package')
seen = set()
for at in range(4, len(data) - 16, 4):
off, word, code_off, code_word = struct.unpack_from('<4I', data, at)
if word != NVN2_CONTROL << 4 | 6 or code_word & 15 != 7 or off + NVN2_CONTROL > len(data):
continue
control = data[off:off + NVN2_CONTROL]
stage = NVN2_STAGES.get(struct.unpack_from('<I', control, 0x54)[0] >> 16)
room = code_word >> 4
if control[:4] != CONTROL_MAGIC or not stage or code_off + room > len(data):
continue
control += bytes(0x880 - NVN2_CONTROL)
size, c1_start, c1_end = struct.unpack_from('<I', control, 0x38)[0], *struct.unpack_from('<II', control, 0x40)
length = c1_end if c1_start >= size and c1_end >= c1_start and c1_end <= room else size
if not 0 < length <= room:
continue
code = data[code_off:code_off + length]
key = digest(control + code)
if key not in seen:
seen.add(key)
yield key, stage, control, code
def nvn2_sharcb_record(data, start, pool, pool_size, code_off, control_at):
"""(key, stage, control, code) of an NVN2 .sharcb record, as nvn2_stages cuts them, or None. A size word of 0xe0
is common, so a candidate counts only with the control magic and a known stage."""
if not start <= control_at <= pool - NVN2_CONTROL or code_off >= pool_size:
return None
control = data[control_at:control_at + NVN2_CONTROL]
stage = NVN2_STAGES.get(struct.unpack_from('<I', control, 0x54)[0] >> 16)
if control[:4] != CONTROL_MAGIC or not stage:
return None
control += bytes(0x880 - NVN2_CONTROL)
size, c1_start, c1_end = struct.unpack_from('<I', control, 0x38)[0], *struct.unpack_from('<II', control, 0x40)
room = pool_size - code_off
length = c1_end if c1_start >= size and c1_end >= c1_start and c1_end <= room else size
if not 0 < length <= room:
return None
code = data[pool + code_off:pool + code_off + length]
return digest(control + code), stage, control, code
def nso_image(path): def nso_image(path):
"""The decompressed text, rodata and data segments of an NSO, concatenated (enough to find embedded BNSH files).""" """The decompressed text, rodata and data segments of an NSO, concatenated (enough to find embedded BNSH files)."""
raw = path.read_bytes() raw = path.read_bytes()
@@ -222,7 +274,8 @@ def startup_package(name):
# The RomFS directories that can hold shaders: the agl and gsys packages, the effects, terrain, the UI layout archives # The RomFS directories that can hold shaders: the agl and gsys packages, the effects, terrain, the UI layout archives
# and the game's own shader packs. Every shader of 1.1.0 (27,576) comes from these six; they are 1,996 entries and # and the game's own shader packs. Every shader of 1.1.0 (27,576) comes from these six; they are 1,996 entries and
# 147 MiB, 0.9% of a RomFS whose every file the walk below used to read and decompress first (65 s against 2.8 s). # 147 MiB, 0.9% of a RomFS whose every file the walk below used to read and decompress first (65 s against 2.8 s).
SHADER_DIRS = ('Shader/', 'Effect/', 'Lib/agl/', 'Lib/Terrain/', 'Lib/gsys/', 'UI/LayoutArchive/') # Ocarina keeps its UI layout shaders (MergeShader) in Layout2Shader/.
SHADER_DIRS = ('Shader/', 'Effect/', 'Lib/agl/', 'Lib/Terrain/', 'Lib/gsys/', 'UI/LayoutArchive/', 'Layout2Shader/')
def inventory(romfs, exefs, everything=True): def inventory(romfs, exefs, everything=True):
@@ -268,6 +321,9 @@ def inventory(romfs, exefs, everything=True):
if data[:4] == b'BAHS': if data[:4] == b'BAHS':
add('sharcb', name, sharcb_shaders(data, rejected), True) add('sharcb', name, sharcb_shaders(data, rejected), True)
return return
if data[:4] == b'NDSR':
add('ndsr', name, ndsr_shaders(data), True)
return
if data[:4] == b'SARC': if data[:4] == b'SARC':
if depth > 4: if depth > 4:
return return
@@ -284,7 +340,7 @@ def inventory(romfs, exefs, everything=True):
if not everything and not name.startswith(SHADER_DIRS): if not everything and not name.startswith(SHADER_DIRS):
continue continue
raw = read(name) raw = read(name)
if raw[:4] not in (ZSTD, b'SARC', b'FSHA', b'BNSH', b'BAHS') and b'BNSH\0\0\0\0' not in raw: if raw[:4] not in (ZSTD, b'SARC', b'FSHA', b'BNSH', b'BAHS', b'NDSR') and b'BNSH\0\0\0\0' not in raw:
continue continue
visit('bnsh', name, decode(raw) if raw[:4] == ZSTD else raw, startup_package(name)) visit('bnsh', name, decode(raw) if raw[:4] == ZSTD else raw, startup_package(name))
for module in sorted(exefs.iterdir()): for module in sorted(exefs.iterdir()):
@@ -387,10 +443,10 @@ def main():
out.mkdir(parents=True, exist_ok=True) out.mkdir(parents=True, exist_ok=True)
shaders, rejected = inventory(extracted / 'romfs', extracted / 'exefs') shaders, rejected = inventory(extracted / 'romfs', extracted / 'exefs')
jobs = write_shaders(shaders, out, args.all) jobs = write_shaders(shaders, out, args.all)
a, b, c = (sum(1 for j in jobs.values() if j['origin'] == o) for o in ('bnsh', 'sharcb', 'main')) a, b, n, c = (sum(1 for j in jobs.values() if j['origin'] == o) for o in ('bnsh', 'sharcb', 'ndsr', 'main'))
print(f'{len(shaders)} unique shaders found in all, {len(jobs)} selected; {rejected} .sharcb record candidates ' print(f'{len(shaders)} unique shaders found in all, {len(jobs)} selected; {rejected} .sharcb record candidates '
f'without a control section rejected', flush=True) f'without a control section rejected', flush=True)
summary = f'shaders {len(jobs)} (bnsh {a}, sharcb {b}, main {c})' summary = f'shaders {len(jobs)} (bnsh {a}, sharcb {b}, ndsr {n}, main {c})'
print(f'extracted {summary} in {time.monotonic() - t0:.1f} s', flush=True) print(f'extracted {summary} in {time.monotonic() - t0:.1f} s', flush=True)
if not args.translate: if not args.translate:
print(f'{summary}, total {time.monotonic() - t0:.1f} s') print(f'{summary}, total {time.monotonic() - t0:.1f} s')