mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 13:59:56 -04:00
Merge branch 'claude/pipeline-list' into test-all-fixes
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -17,6 +17,9 @@ __pycache__/
|
||||
/generated/
|
||||
# Button art a release carries (tools/make_launcher.py --prompt-art); a checkout may hold it for a test build
|
||||
/prompt-art/
|
||||
# The shared pipeline list a release carries (tools/pipeline_list.py merge; tools/prepare_shaders.py reads it here)
|
||||
/tools/data/pipeline_list.bin
|
||||
/data/pipeline_list.bin
|
||||
*.xci
|
||||
*.xcz
|
||||
*.nsp
|
||||
|
||||
@@ -158,6 +158,22 @@ of them.
|
||||
version; they build at their first draw instead, another driver version builds them again, and so does deleting the
|
||||
file.
|
||||
|
||||
What this step leaves for play, and the shared pipeline list. The step translates every shader's default variant
|
||||
and the shader list's variants and builds each one's stage library or compute pipeline. A whole graphics pipeline
|
||||
is built ahead only at the start of a game folder that has drawn it before (`pipeline-keys.bin`); a pipeline met
|
||||
for the first time is linked from its stage libraries in 2 to 6 ms, except a mesh pipeline, which has no libraries
|
||||
and compiles whole (0.2 to 1.1 s, its draws left out of the picture meanwhile). `tools/pipeline_list.py merge`
|
||||
folds the `pipeline-keys.bin` and `shader-answers.bin` of game folders that were played into a pipeline list;
|
||||
`prepare_shaders.py` reads it from `tools\data\pipeline_list.bin` (`data\pipeline_list.bin` in a release folder, or
|
||||
`--pipeline-list`), adds the shader variants it names to the seed and writes its records next to the game as
|
||||
`seed-pipeline-keys.bin`, and `--prepare-shaders` builds the mesh pipelines among them into the pipeline cache and
|
||||
the driver's (`runtime/vk_pipeline.c` `seed_pipelines`; `NX2NAVI_PREPARE_PIPELINES=all` builds the vertex pipelines
|
||||
too). `--warm-shaders` builds them again, and `nx2navi.exe --prepare-pipelines` does that step alone. The list is
|
||||
derived from game data: it is not in the repository (`tools/check_content.py` refuses it), and a kit, a launcher
|
||||
release or an installer release takes one with `--pipeline-list` (`tools/make_kit.py`, `make_launcher.py`,
|
||||
`make_release.py`). A list belongs to the record layout of the runtime that wrote it; another runtime says so and
|
||||
builds nothing from it. `python tests\pipeline_list_test.py` checks the tool on made-up folders.
|
||||
|
||||
7. Run. The game keeps running when its window loses the focus (`assume_focus = 1` in `nx2navi.ini`, the default; the
|
||||
Game tab switches it). With 0 it pauses, as it does on the Switch when Home is pressed.
|
||||
|
||||
|
||||
@@ -148,7 +148,8 @@ llvm/ llvm-mingw, x86_64-only subset (Apache-2.0 with LLVM excep
|
||||
runtime/ libnx2navirt.a, libnx2navirt-v2.a and the headers the generated code and modules.c/romfs.c/patches.c
|
||||
include
|
||||
bin/ nx2navi_xlate.exe, hactool.exe, SDL3.dll, libopus-0.dll, ninja.exe
|
||||
data/ shader_list.tsv (MIT, THIRD_PARTY_NOTICES.md); pgo-<ver>.profdata + .json, the execution profile an
|
||||
data/ shader_list.tsv (MIT, THIRD_PARTY_NOTICES.md); pipeline_list.bin when the release was made with
|
||||
one (building.md, step 6); pgo-<ver>.profdata + .json, the execution profile an
|
||||
update's code and the runtime libraries compile with (building.md, "Profile-guided build")
|
||||
THIRD_PARTY_NOTICES.md
|
||||
nx2navi.ini.example, VERSION the default ini; the release string (the repository's VERSION plus the commit)
|
||||
|
||||
@@ -53,7 +53,7 @@ static const char *const known[] = {
|
||||
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
|
||||
"MV_SURVEY",
|
||||
"ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC",
|
||||
"PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW",
|
||||
"PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW",
|
||||
"POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP",
|
||||
"PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC",
|
||||
"REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", "SCENE_SHARE",
|
||||
@@ -234,6 +234,7 @@ static const CfgMeta metas[] = {
|
||||
{"PIPE_LINK_DRAW_MS", "3", "1..50", RESTART}, // vk_exec_draw.c link_draw_ms
|
||||
{"PIPE_FRAME_WAIT_MS", "50", "0..500", RESTART}, // vk_exec_draw.c frame_wait
|
||||
{"PIPE_SKIP_STEADY", "1", "0|1", RESTART}, // vk_exec.c steady_on
|
||||
{"PREPARE_PIPELINES", "mesh", "0|mesh|all", RESTART}, // vk_pipeline.c seed_pipelines
|
||||
{"PIPE_AHEAD", "1", "0|1", RESTART}, // vk_exec.c vk_exec_ahead, vk_pipeline.c vk_pipeline_hint
|
||||
{"GPL_FEEDBACK", "1", "0|1", RESTART}, // vk_pipeline.c gpl_feedback
|
||||
{"PIPE_FAST_WORKERS", "1", "0|1", RESTART}, // vk_pipeline.c lib_worker, fast_take
|
||||
|
||||
@@ -389,6 +389,7 @@ int probe_gamepad(void); // input.c
|
||||
int probe_movie(void); // movie_probe.c: host VP9 movie::Decoder on one RomFS movie
|
||||
int vk_pipeline_prepare_shaders(void); // vk_pipeline.c: --prepare-shaders (tools/prepare_shaders.py)
|
||||
int vk_pipeline_warm_shaders(void); // vk_pipeline.c: --warm-shaders (tools/build_game.py)
|
||||
int vk_pipeline_prepare_pipelines(void); // vk_pipeline.c: --prepare-pipelines (the shared pipeline list)
|
||||
int vk_pipeline_probe_prewarm(void); // vk_pipeline.c: --probe-prewarm
|
||||
// vk_device.c: the drivers' own shader caches beside the exe (MESA_SHADER_CACHE_DIR, on NVIDIA's Linux driver
|
||||
// __GL_SHADER_DISK_CACHE_PATH and _SIZE) in the environment, once; before any thread can load a Vulkan driver
|
||||
|
||||
@@ -529,7 +529,7 @@ static int usage(const char *argv0) {
|
||||
"--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | "
|
||||
"--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | "
|
||||
"--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | "
|
||||
"--prepare-shaders | --warm-shaders | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0);
|
||||
"--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0);
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -586,7 +586,8 @@ int main(int argc, char **argv) {
|
||||
// threads, so narrowing them to one die would only make the preparation take longer.
|
||||
int prepare = 0;
|
||||
for (int i = 1; i < argc; i++)
|
||||
if (!strcmp(argv[i], "--prepare-shaders") || !strcmp(argv[i], "--warm-shaders")) prepare = 1;
|
||||
if (!strcmp(argv[i], "--prepare-shaders") || !strcmp(argv[i], "--warm-shaders") ||
|
||||
!strcmp(argv[i], "--prepare-pipelines")) prepare = 1;
|
||||
if (!prepare) perf_cpu_start();
|
||||
{ // NX2NAVI_POWER_FULL=0 leaves Windows' power throttling of an unfocused window in place (A/B runs)
|
||||
const char *pf = cfg_get("NX2NAVI_POWER_FULL");
|
||||
@@ -679,6 +680,7 @@ int main(int argc, char **argv) {
|
||||
if (!strcmp(mode, "--probe-movie")) return probe_movie();
|
||||
if (!strcmp(mode, "--prepare-shaders")) return vk_pipeline_prepare_shaders();
|
||||
if (!strcmp(mode, "--warm-shaders")) return vk_pipeline_warm_shaders();
|
||||
if (!strcmp(mode, "--prepare-pipelines")) return vk_pipeline_prepare_pipelines();
|
||||
if (!strcmp(mode, "--probe-prewarm")) return vk_pipeline_probe_prewarm();
|
||||
if (!strcmp(mode, "--boot")) { cfg.boot = 1; rt_boot(); return 0; }
|
||||
return usage(argv[0]);
|
||||
|
||||
@@ -3974,6 +3974,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
|
||||
if (linking) t0 = kern_now_ns();
|
||||
else w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0;
|
||||
vk_exec_ahead(ex);
|
||||
// A mesh pipeline that is known to be compiling gets no fast link, so the few milliseconds a draw waits
|
||||
// for one are lost in every frame that meets it until the compile ends, 0.2 to 1.1 s later (MESH_WAIT_MS).
|
||||
if (wait_ms && linking && mesh && vk_pipeline_building_ms(&s) > MESH_WAIT_MS) wait_ms = 0;
|
||||
if (wait_ms) p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
|
||||
}
|
||||
if (t0 && budget) {
|
||||
|
||||
@@ -5823,6 +5823,167 @@ static void lb_warm_prepared(const char *dir) {
|
||||
free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0, LB.skipped = 0;
|
||||
}
|
||||
|
||||
// ---- the shared pipeline list (tools/pipeline_list.py; seed-pipeline-keys.bin next to the game)
|
||||
// tools/prepare_shaders.py writes the list's key records there, in the layout of pipeline-keys.bin, after putting the
|
||||
// shader variants they name into the seed. The preparation builds them once, after the seed is translated: each
|
||||
// pipeline goes through the pipeline cache and so into pipeline-cache.bin and the driver's own disk cache, and is
|
||||
// dropped with the process. Nothing is added to this folder's pipeline-keys.bin: the boot keeps building only what
|
||||
// this install drew, and a listed pipeline costs no start-up time or video memory until a draw meets it. That draw
|
||||
// then finds its pipeline in the caches: a few milliseconds for a mesh pipeline, which has no stage libraries and
|
||||
// otherwise leaves its draws out of the picture for its whole compile (0.2 to 1.1 s each on a first visit of a forest,
|
||||
// 18 of them).
|
||||
// NX2NAVI_PREPARE_PIPELINES: mesh (default) builds the list's mesh pipelines only; all, every record; 0, none. A
|
||||
// vertex pipeline is linked from its prepared stage libraries in 2 to 6 ms when a draw misses it, and its module
|
||||
// depends on the vertex output union of the pairs this install has recorded (vs_outs), so one built here is not
|
||||
// certain to be the one a draw asks for.
|
||||
// A driver whose compiler crashes on one of these would crash at every preparation: seed-pipelines-running.txt marks
|
||||
// the step, and a run that finds the marker of a run that died skips the step for good on this folder (the pipelines
|
||||
// are then built at their first draw, as without a list) until the marker is deleted.
|
||||
static int cache_save(uint32_t wait_ms);
|
||||
|
||||
static int seed_pipelines(const char *dir) {
|
||||
const char *mode = cfg_value("NX2NAVI_PREPARE_PIPELINES");
|
||||
int all = !strcmp(mode, "all");
|
||||
char path[PLAT_PATH_MAX * 2 + 48], mark[PLAT_PATH_MAX * 2 + 48];
|
||||
snprintf(path, sizeof path, "%s/seed-pipeline-keys.bin", dir);
|
||||
snprintf(mark, sizeof mark, "%s/seed-pipelines-running.txt", dir);
|
||||
FILE *f = fopen(path, "rb");
|
||||
if (!f) return 0; // no list: nothing to say
|
||||
if (!strcmp(mode, "0")) {
|
||||
fclose(f);
|
||||
printf("prepare-pipelines: off (NX2NAVI_PREPARE_PIPELINES=0)\n");
|
||||
return 0;
|
||||
}
|
||||
if (plat_file_exists(mark)) {
|
||||
fclose(f);
|
||||
printf("prepare-pipelines: skipped: an earlier preparation ended while it built the listed pipelines (%s); they "
|
||||
"are built at their first draw\n", mark);
|
||||
return 0;
|
||||
}
|
||||
KeysHeader h;
|
||||
fseek(f, 0, SEEK_END);
|
||||
long len = ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
size_t n = 0;
|
||||
KeyRec *r = NULL;
|
||||
if (len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8) &&
|
||||
h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec)) {
|
||||
n = ((size_t)len - sizeof h) / sizeof(KeyRec);
|
||||
r = malloc(n * sizeof *r + 1);
|
||||
if (r && fread(r, sizeof *r, n, f) != n) n = 0;
|
||||
} else {
|
||||
printf("prepare-pipelines: %s is a list for another version of the game's runtime (its record layout differs); "
|
||||
"no pipeline is built from it\n", path);
|
||||
}
|
||||
fclose(f);
|
||||
uint8_t (*ids)[32] = n ? malloc(3 * n * 32) : NULL;
|
||||
const GfxShader **sh = n ? calloc(3 * n, sizeof *sh) : NULL;
|
||||
if (!r || !ids || !sh) {
|
||||
free(r), free(ids), free(sh);
|
||||
return 0;
|
||||
}
|
||||
double t0 = now_ms();
|
||||
size_t m = 0, u = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
memcpy(ids[m++], r[i].vs, 32);
|
||||
if (r[i].has_fs) memcpy(ids[m++], r[i].fs, 32);
|
||||
if (r[i].has_ts) memcpy(ids[m++], r[i].ts, 32);
|
||||
}
|
||||
qsort(ids, m, 32, id_cmp);
|
||||
for (size_t i = 0; i < m; i++)
|
||||
if (!u || memcmp(ids[u - 1], ids[i], 32)) memcpy(ids[u++], ids[i], 32);
|
||||
int cpus = (int)plat_cpu_usable();
|
||||
const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE");
|
||||
int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2;
|
||||
gfx_shader_prewarm((const uint8_t(*)[32])ids, u, sh, cpus, procs < 1 ? 1 : procs);
|
||||
static const char note[] = "the listed pipelines are being built; a run that finds this file skips them\n";
|
||||
plat_file_write_atomic(mark, note, sizeof note - 1);
|
||||
uint32_t queued = 0, dropped = 0, known = 0, other = 0, mesh = 0, no_mesh = 0;
|
||||
plat_mutex_lock(&P.lock);
|
||||
uint32_t built0 = K.built, failed0 = K.failed;
|
||||
K.loading = 1;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp);
|
||||
const uint8_t (*fi)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL;
|
||||
const uint8_t (*t)[32] = r[i].has_ts ? bsearch(r[i].ts, ids, u, 32, id_cmp) : NULL;
|
||||
const GfxShader *vs = v ? sh[v - ids] : NULL, *fs = fi ? sh[fi - ids] : NULL, *ts = t ? sh[t - ids] : NULL;
|
||||
if (!vs || (r[i].has_fs && !fs) || (r[i].has_ts && !ts)) {
|
||||
dropped++;
|
||||
continue;
|
||||
}
|
||||
VkPipeState s = r[i].s;
|
||||
s.vs = vs, s.fs = fs, s.ts = ts;
|
||||
if (!is_mesh(&s) && !all) {
|
||||
other++;
|
||||
continue;
|
||||
}
|
||||
if (is_mesh(&s) && !vk_ctx()->has_mesh_shader) { // this GPU draws no mesh pipeline
|
||||
no_mesh++;
|
||||
continue;
|
||||
}
|
||||
if (!is_mesh(&s) && P.indep && fs && gfx_shader_stage(vs) == GFX_STAGE_VS) { // its vertex output union, as the boot
|
||||
GfxShaderIo io[32];
|
||||
vs_outs(vs, fs, io);
|
||||
}
|
||||
GKey k;
|
||||
gkey_fill(&s, &k);
|
||||
uint64_t hk = key_hash(&k, sizeof k);
|
||||
int q = 0;
|
||||
if (pipe_find(&k, sizeof k, hk)) known++;
|
||||
else pipe_miss(&s, &k, hk, 2, &q), queued += (uint32_t)q, mesh += q && is_mesh(&s);
|
||||
}
|
||||
K.loading = 0;
|
||||
K.until = 0; // every one of them: no boot waits for this
|
||||
plat_mutex_unlock(&P.lock);
|
||||
free(ids), free(sh), free(r);
|
||||
PlatThread *th[64];
|
||||
int nt = 0;
|
||||
for (int i = 0; i < cpus && i < 64 && queued; i++)
|
||||
if ((th[nt] = plat_thread_new(warm_builder, NULL, "pipe list"))) nt++;
|
||||
if (!nt && queued) warm_builder(NULL);
|
||||
for (int i = 0; i < nt; i++)
|
||||
while (!plat_thread_join_ns(th[i], 2000000000u)) {
|
||||
printf("prepare-pipelines: progress %u %u\n",
|
||||
__atomic_load_n(&K.built, __ATOMIC_RELAXED) + __atomic_load_n(&K.failed, __ATOMIC_RELAXED) - built0 - failed0,
|
||||
queued);
|
||||
fflush(stdout);
|
||||
}
|
||||
plat_mutex_lock(&P.lock);
|
||||
while (K.left) plat_cond_wait(&done_cv, &P.lock); // those a pool worker took
|
||||
uint32_t built = K.built - built0, failed = K.failed - failed0;
|
||||
plat_mutex_unlock(&P.lock);
|
||||
int saved = cache_save(5000); // as a save at exit: it waits that long for a moment with no build running
|
||||
plat_file_delete(mark);
|
||||
char msg[640];
|
||||
snprintf(msg, sizeof msg, "prepare-pipelines: %zu listed pipelines: %u built (%u mesh), %u failed, %u the same as one "
|
||||
"built before, %u without a shader this game version or its shader cache has, %u vertex pipelines left to "
|
||||
"their first draw (NX2NAVI_PREPARE_PIPELINES=%s), %u mesh pipelines this GPU cannot draw; on %d threads in "
|
||||
"%.0f ms; pipeline cache %s", n, built, mesh, failed, known, dropped, other, all ? "all" : "mesh", no_mesh,
|
||||
nt ? nt : 1, now_ms() - t0, saved ? "saved" : "not saved (builds kept running)");
|
||||
printf("%s\n", msg);
|
||||
fflush(stdout);
|
||||
rt_log("vk_pipeline: %s", msg);
|
||||
fb_log("after the listed pipelines");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// nx2navi.exe --prepare-pipelines: the list's pipelines alone, for a folder whose shaders are prepared already.
|
||||
int vk_pipeline_prepare_pipelines(void) {
|
||||
if (!ensure_init()) {
|
||||
fprintf(stderr, "prepare-pipelines: no Vulkan device\n");
|
||||
return 1;
|
||||
}
|
||||
char dir[PLAT_PATH_MAX * 2];
|
||||
if (!cache_dir(dir, sizeof dir)) return 1;
|
||||
char path[PLAT_PATH_MAX * 2 + 48];
|
||||
snprintf(path, sizeof path, "%s/seed-pipeline-keys.bin", dir);
|
||||
if (!plat_file_exists(path)) {
|
||||
fprintf(stderr, "prepare-pipelines: no %s (tools/prepare_shaders.py writes it from the pipeline list)\n", path);
|
||||
return 1;
|
||||
}
|
||||
return seed_pipelines(dir);
|
||||
}
|
||||
|
||||
// nx2navi.exe --prepare-shaders (tools/prepare_shaders.py): the device sets the translator's host answers (they are in every
|
||||
// cache key), the seed next to nx2navi.exe is translated on every core, then every prepared variant's stage library or
|
||||
// compute pipeline is built once into the driver's disk cache (lb_warm_prepared). No game runs.
|
||||
@@ -5837,6 +5998,9 @@ int vk_pipeline_prepare_shaders(void) {
|
||||
const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset
|
||||
int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2;
|
||||
int rc = gfx_shader_prepare_seed(dir, cpus, procs < 1 ? 1 : procs);
|
||||
// The listed pipelines before the stage libraries: the pipeline cache is saved with them in it and without the
|
||||
// tens of thousands of libraries the next step builds, which would take the file past its size limit.
|
||||
if (!rc) seed_pipelines(dir);
|
||||
if (!rc) lb_warm_prepared(dir);
|
||||
return rc;
|
||||
}
|
||||
@@ -5858,6 +6022,7 @@ int vk_pipeline_warm_shaders(void) {
|
||||
fprintf(stderr, "warm-shaders: no %s (tools/prepare_shaders.py writes it)\n", prp);
|
||||
return 1;
|
||||
}
|
||||
seed_pipelines(dir); // before the libraries, as in vk_pipeline_prepare_shaders
|
||||
lb_warm_prepared(dir);
|
||||
return 0;
|
||||
}
|
||||
|
||||
132
tests/pipeline_list_test.py
Normal file
132
tests/pipeline_list_test.py
Normal file
@@ -0,0 +1,132 @@
|
||||
"""tools/pipeline_list.py on made-up game folders: records folded once, a key whose shader the folder's answers do not
|
||||
name left out, --mesh-only, a folder of another record layout left out, the file read back, and what
|
||||
tools/prepare_shaders.py takes from a list for its seed. Also tools/check_content.py refusing such files in a commit.
|
||||
No game files; stdlib only.
|
||||
|
||||
Run: python tests/pipeline_list_test.py
|
||||
"""
|
||||
import hashlib
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import check_content # noqa: E402
|
||||
import pipeline_list as pl # noqa: E402
|
||||
|
||||
REC = 464 # a KeyRec of the runtime that wrote the test's files: three ids, two flags, the state
|
||||
VS, FS, TS, MS = 0, 4, 6, 7
|
||||
|
||||
|
||||
def h(seed):
|
||||
return hashlib.sha256(seed.encode()).digest()
|
||||
|
||||
|
||||
def answer(seed, stage, salt=0):
|
||||
"""A shader-answers.bin record: stage, code hash, input id, 147 bytes of answers, the pad byte."""
|
||||
return struct.pack('<I', stage) + h('code ' + seed) + h('id ' + seed) + bytes([salt]) * 147 + b'\0'
|
||||
|
||||
|
||||
def key(vs, fs=None, ts=None, state=0):
|
||||
ids = h('id ' + vs) + (h('id ' + fs) if fs else bytes(32)) + (h('id ' + ts) if ts else bytes(32))
|
||||
return ids + struct.pack('<II', fs is not None, ts is not None) + bytes([state]) * (REC - 104)
|
||||
|
||||
|
||||
def folder(root, name, keys, answers, version=4, rec=REC):
|
||||
d = root / name
|
||||
d.mkdir()
|
||||
(d / 'pipeline-keys.bin').write_bytes(b'NX2NPKEY' + struct.pack('<II', version, rec) + b''.join(keys))
|
||||
(d / 'shader-answers.bin').write_bytes(pl.ANS_MAGIC + struct.pack('<II', 1, pl.ANS_REC) + b''.join(answers))
|
||||
return d
|
||||
|
||||
|
||||
class PipelineList(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.tmp.name)
|
||||
self.log = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def merge(self, dirs, **kw):
|
||||
return pl.merge(dirs, log=self.log.append, **kw)
|
||||
|
||||
def test_merge(self):
|
||||
a = folder(self.root, 'a', [key('v1', 'f1'), key('m1', 'f2', 't1'), key('v1', 'f1', state=1)],
|
||||
[answer('v1', VS), answer('f1', FS), answer('m1', MS), answer('f2', FS), answer('t1', TS)])
|
||||
b = folder(self.root, 'b', [key('v1', 'f1'), key('m2', 'f1'), key('v9', 'f1')],
|
||||
[answer('v1', VS), answer('f1', FS), answer('m2', MS)]) # v9's answers are not recorded there
|
||||
header, keys, answers = self.merge([a, b])
|
||||
self.assertEqual(header, (b'NX2NPKEY', 4, REC))
|
||||
self.assertEqual(keys, [key('v1', 'f1'), key('m1', 'f2', 't1'), key('v1', 'f1', state=1), key('m2', 'f1')])
|
||||
self.assertEqual({x[36:68] for x in answers}, {h('id ' + s) for s in ('v1', 'f1', 'm1', 'f2', 't1', 'm2')})
|
||||
self.assertIn('1 without their shaders\' answers', self.log[1])
|
||||
|
||||
def test_mesh_only(self):
|
||||
a = folder(self.root, 'a', [key('v1', 'f1'), key('m1', 'f2', 't1'), key('m1', None)],
|
||||
[answer('v1', VS), answer('f1', FS), answer('m1', MS), answer('f2', FS), answer('t1', TS)])
|
||||
_, keys, answers = self.merge([a], mesh_only=True)
|
||||
self.assertEqual(keys, [key('m1', 'f2', 't1'), key('m1', None)])
|
||||
self.assertEqual({x[36:68] for x in answers}, {h('id ' + s) for s in ('m1', 'f2', 't1')})
|
||||
|
||||
def test_other_layout_left_out(self):
|
||||
a = folder(self.root, 'a', [key('v1', 'f1')], [answer('v1', VS), answer('f1', FS)])
|
||||
b = folder(self.root, 'b', [key('v2', 'f1')], [answer('v2', VS), answer('f1', FS)], version=5)
|
||||
_, keys, _ = self.merge([a, b])
|
||||
self.assertEqual(keys, [key('v1', 'f1')])
|
||||
self.assertIn('another layout', self.log[1])
|
||||
self.assertIsNone(self.merge([self.root / 'none']))
|
||||
|
||||
def test_file(self):
|
||||
a = folder(self.root, 'a', [key('v1', 'f1'), key('m1', 'f1')],
|
||||
[answer('v1', VS), answer('f1', FS), answer('m1', MS)])
|
||||
header, keys, answers = self.merge([a])
|
||||
data = pl.pack(header, keys, answers)
|
||||
got_answers, image = pl.unpack(data)
|
||||
self.assertEqual(got_answers, answers)
|
||||
self.assertEqual(image, (a / 'pipeline-keys.bin').read_bytes()) # the layout the runtime reads as it is
|
||||
self.assertEqual(pl.read_keys(image)[3], keys)
|
||||
self.assertIsNone(pl.unpack(data[:-1]))
|
||||
self.assertIsNone(pl.unpack(b'NX2NPKEY' + data[8:]))
|
||||
self.assertIsNone(pl.unpack(b''))
|
||||
|
||||
def test_seed_variants(self):
|
||||
"""prepare_shaders.list_variants: variants of shaders the game version has, once, with the id left to the
|
||||
runtime."""
|
||||
try:
|
||||
import prepare_shaders as ps
|
||||
except ImportError as e: # its shader inventory needs Python 3.14's compression.zstd
|
||||
self.skipTest(f'tools/prepare_shaders.py does not import here: {e}')
|
||||
recs = [answer('v1', VS), answer('f1', FS), answer('gone', FS), answer('v1', VS, salt=1)]
|
||||
have = {h('code v1'), h('code f1')}
|
||||
seen = {(h('code f1'), bytes(147))} # the seed has f1's variant already (its default one, say)
|
||||
out, absent = ps.list_variants(recs, have, seen)
|
||||
self.assertEqual(absent, 1)
|
||||
self.assertEqual(len(out), 2)
|
||||
for r in out:
|
||||
self.assertEqual(len(r), pl.ANS_REC)
|
||||
self.assertEqual(r[4:36], h('code v1'))
|
||||
self.assertEqual(r[36:68], bytes(32))
|
||||
self.assertEqual(r[-1], ps.LIST_PRIORITY)
|
||||
self.assertEqual([r[68] for r in out], [0, 1])
|
||||
self.assertEqual(ps.list_variants(recs, have, seen), ([], 1)) # all in the seed now
|
||||
|
||||
def test_refused_in_a_commit(self):
|
||||
a = folder(self.root, 'a', [key('v1', 'f1')], [answer('v1', VS), answer('f1', FS)])
|
||||
lst = self.root / 'pipeline_list.bin'
|
||||
lst.write_bytes(pl.pack(*self.merge([a])))
|
||||
for p in (lst, a / 'pipeline-keys.bin', a / 'shader-answers.bin'):
|
||||
self.assertTrue(check_content.recorded_in_play(p, None), p.name)
|
||||
self.assertTrue(check_content.recorded_in_play(p, p.read_bytes()), p.name)
|
||||
other = self.root / 'notes.txt'
|
||||
other.write_text('NX2N is the prefix of the recorded files\n')
|
||||
self.assertFalse(check_content.recorded_in_play(other, None))
|
||||
self.assertFalse(check_content.recorded_in_play(self.root / 'missing.bin', None))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -91,6 +91,23 @@ def compiled_shader(path: Path) -> bool:
|
||||
return path.suffix.lower() == ".spv" or path.name.lower().endswith("_spv.h")
|
||||
|
||||
|
||||
# What a game folder records in play and what tools/pipeline_list.py folds from it: the game's shader code, the
|
||||
# answers it was translated for, the pipelines it drew. All of it is derived from game data.
|
||||
RECORD_MAGICS = (b"NX2NSPRG", b"NX2NSANS", b"NX2NPKEY", b"NX2NPLST")
|
||||
|
||||
|
||||
def recorded_in_play(path: Path, data: bytes | None) -> bool:
|
||||
"""A shader-programs.bin, shader-answers.bin, pipeline-keys.bin or a pipeline list, by its first bytes. A build kit
|
||||
or a release folder may carry a pipeline list (data/pipeline_list.bin), so make_kit.py and make_release.py, which
|
||||
call problems_for, are not held to this."""
|
||||
if data is None:
|
||||
if not path.is_file():
|
||||
return False
|
||||
with open(path, "rb") as f:
|
||||
data = f.read(8)
|
||||
return data[:8] in RECORD_MAGICS
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--staged", action="store_true", help="check staged files only")
|
||||
@@ -100,9 +117,12 @@ def main() -> int:
|
||||
for path in list_files(args.staged):
|
||||
# --staged checks what the commit takes: a marker staged and then removed from the working copy alone
|
||||
# passed when the file on disk was read
|
||||
problems = problems_for(path, data=staged_bytes(path) if args.staged else None)
|
||||
data = staged_bytes(path) if args.staged else None
|
||||
problems = problems_for(path, data=data)
|
||||
if compiled_shader(path):
|
||||
problems.append("compiled shader: the repository holds the source, tools/build_shaders.py builds this")
|
||||
if recorded_in_play(path, data):
|
||||
problems.append("recorded from a game folder (shaders, answers, pipeline keys or a pipeline list)")
|
||||
for problem in problems:
|
||||
print(f"{path}: {problem}")
|
||||
failed = True
|
||||
|
||||
@@ -1622,6 +1622,20 @@ def shader_run(cmd, cwd=None):
|
||||
return rc
|
||||
|
||||
|
||||
def refresh_pipeline_seed(out):
|
||||
"""The kit's pipeline list (tools/pipeline_list.py) as the game folder's seed-pipeline-keys.bin, so that an update
|
||||
which carries the shaders over builds a newer list's pipelines too: `--warm-shaders` builds from that file. The
|
||||
list's shader variants are not translated on this path, so a record whose variant this folder has neither prepared
|
||||
nor met is left out by the game, which counts those in its line."""
|
||||
lst = KIT / 'tools' / 'data' / 'pipeline_list.bin'
|
||||
if not lst.is_file():
|
||||
return
|
||||
import pipeline_list
|
||||
got = pipeline_list.unpack(lst.read_bytes())
|
||||
if got:
|
||||
(out / 'seed-pipeline-keys.bin').write_bytes(got[1])
|
||||
|
||||
|
||||
def warm_driver_cache(out):
|
||||
"""After an update that carries the shaders over: every prepared stage library is built into the graphics driver's
|
||||
cache again (`nx2navi.exe --warm-shaders`). It takes seconds while the driver still has them. It takes minutes when
|
||||
@@ -1633,6 +1647,7 @@ def warm_driver_cache(out):
|
||||
exe = out / EXE_NAME
|
||||
if not exe.is_file() or not (out / 'shader-prepared.bin').is_file():
|
||||
return
|
||||
refresh_pipeline_seed(out)
|
||||
say(" the prepared shaders are built into the graphics driver's cache again")
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
@@ -1672,7 +1687,8 @@ def step_shaders(out, update=False):
|
||||
if rc:
|
||||
fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it '
|
||||
'meets them. Run the build again to retry this step')
|
||||
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
|
||||
# translated now: the shader cache holds them. seed-pipeline-keys.bin stays: --warm-shaders builds from it again
|
||||
for seed in ('seed-programs.bin', 'seed-answers.bin'):
|
||||
(out / seed).unlink(missing_ok=True)
|
||||
pack_shader_cache(out)
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
NAME = 'NX2navi-BuildKit'
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py',
|
||||
'pack_shader_cache.py', 'vulkan_pin.py') # make_game.py imports build_game's vma_include and vulkan_include on
|
||||
# Linux for the VMA header and for Vulkan headers newer than the system's (which build_game takes from vulkan_pin), and
|
||||
@@ -60,6 +60,8 @@ def main():
|
||||
help='the Streamline SDK folder instead of downloading the SDK release with --dlssg')
|
||||
ap.add_argument('--prompt-art', type=Path, nargs='+', metavar='FOLDER',
|
||||
help='the button art: folders holding XGamepad, P5Gamepad and Keyboard_Mouse, or those folders')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--out', type=Path, default=ROOT / 'build' / 'kit')
|
||||
ap.add_argument('--zip', action='store_true')
|
||||
a = ap.parse_args()
|
||||
@@ -100,6 +102,12 @@ def main():
|
||||
if a.prompt_art: # make_game.py draws the button prompts from it (its prompts step)
|
||||
n, size = prompt_art.copy_art(a.prompt_art, kit / prompt_art.FOLDER)
|
||||
print(f'button art: {n} pictures, {size / 1e6:.1f} MB')
|
||||
if a.pipeline_list: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
import pipeline_list
|
||||
if pipeline_list.unpack(a.pipeline_list.read_bytes()) is None:
|
||||
sys.exit(f'{a.pipeline_list} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
(kit / 'tools' / 'data').mkdir(exist_ok=True)
|
||||
shutil.copy2(a.pipeline_list, kit / 'tools' / 'data' / 'pipeline_list.bin')
|
||||
import check_content
|
||||
bad = [f'{p.relative_to(kit)}: {m}' for p in sorted(kit.rglob('*')) if p.is_file()
|
||||
for m in check_content.problems_for(p, allow_large=p.suffix in ('.exe', '.dll'))] # the translator, the DLLs
|
||||
|
||||
@@ -77,7 +77,7 @@ PINS = {
|
||||
'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'),
|
||||
}
|
||||
TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py',
|
||||
'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py',
|
||||
'pack_shader_cache.py')
|
||||
TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example')
|
||||
SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'}
|
||||
@@ -177,7 +177,7 @@ def build_python(py):
|
||||
|
||||
# ---------------------------------------------------------------- app/<version>/
|
||||
|
||||
def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None):
|
||||
def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None, pipelines=None):
|
||||
# The runtime's own shaders are source in the repository and SPIR-V headers in a release: a player's PC compiles
|
||||
# runtime/ (make_game.py) and has no shader compiler. Build the missing and the stale ones before the copy.
|
||||
build_shaders.build(ROOT / 'runtime')
|
||||
@@ -205,6 +205,10 @@ def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, d
|
||||
if art: # make_game.py draws the button prompts from it (its prompts step)
|
||||
n, size = prompt_art.copy_art(art, app / prompt_art.FOLDER)
|
||||
say(f'button art: {n} pictures, {size / 1e6:.1f} MB')
|
||||
if pipelines: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
(app / 'tools' / 'data').mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(pipelines, app / 'tools' / 'data' / 'pipeline_list.bin')
|
||||
say(f'pipeline list: {pipelines.stat().st_size / 1e6:.1f} MB')
|
||||
(app / 'toolchain.lock.json').write_text(json.dumps(lock, indent=1), encoding='utf-8')
|
||||
(app / 'feed.json').write_text(json.dumps({'type': 'github', 'repo': feed_repo}, indent=1), encoding='utf-8')
|
||||
|
||||
@@ -324,6 +328,8 @@ def main():
|
||||
'nvngx_dlssg.dll files, instead of downloading the SDK release with --dlssg')
|
||||
ap.add_argument('--prompt-art', type=Path, nargs='+', metavar='FOLDER',
|
||||
help='the button art: folders holding XGamepad, P5Gamepad and Keyboard_Mouse, or those folders')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--out', type=Path, default=MAIN_ROOT / 'build' / 'launcher-release')
|
||||
ap.add_argument('--feed', default=DEFAULT_FEED, help='the GitHub repository the launcher looks for releases in')
|
||||
ap.add_argument('--notes', type=Path, help='release notes (markdown) for the manifest and the GitHub release')
|
||||
@@ -356,7 +362,11 @@ def main():
|
||||
dlssg = (a.dlssg, streamline_pin.resolve(a.streamline, DL)) if a.dlssg else None
|
||||
say(f'DLSS Frame Generation: {a.dlssg}, Streamline {streamline_pin.VERSION}: {dlssg[1]}' if dlssg
|
||||
else 'DLSS Frame Generation left out (no --dlssg)')
|
||||
build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx, dlss, a.prompt_art, dlssg)
|
||||
if a.pipeline_list:
|
||||
import pipeline_list
|
||||
if pipeline_list.unpack(a.pipeline_list.read_bytes()) is None:
|
||||
sys.exit(f'{a.pipeline_list} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx, dlss, a.prompt_art, dlssg, a.pipeline_list)
|
||||
(folder / 'app' / 'current.txt').write_text(ver + '\n', encoding='utf-8')
|
||||
build_python(folder / 'python')
|
||||
cc, windres = native_compiler(lock)
|
||||
|
||||
@@ -113,7 +113,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
|
||||
# tools/ files the installer steps run (and what they import), copied from the repo.
|
||||
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py',
|
||||
'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py',
|
||||
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py',
|
||||
'romfs_index.py', 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'ultrawide_layout.py', 'cheats.py',
|
||||
'exefs_patches.py']
|
||||
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
|
||||
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
|
||||
@@ -427,7 +427,7 @@ def runtime_lib(out, llvm, pgo):
|
||||
shutil.copy2(ROOT / 'runtime' / name, rt / name)
|
||||
|
||||
|
||||
def build_sources(out, pgo):
|
||||
def build_sources(out, pgo, pipelines=None):
|
||||
copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r)
|
||||
for f in TOOLS_FILES:
|
||||
(out / 'tools').mkdir(exist_ok=True)
|
||||
@@ -437,6 +437,8 @@ def build_sources(out, pgo):
|
||||
(out / 'data').mkdir(exist_ok=True)
|
||||
text = shader_list.history(MAIN_ROOT / 'refs' / 'MacNativeGames', MNG_COMMIT)
|
||||
(out / 'data' / 'shader_list.tsv').write_text(text, encoding='utf-8', newline='\n')
|
||||
if pipelines: # prepare_shaders.py reads it there (BUNDLED_PIPELINES)
|
||||
shutil.copy2(pipelines, out / 'data' / 'pipeline_list.bin')
|
||||
for p in pgo.values():
|
||||
for f in (p, p.with_suffix('.json')):
|
||||
if f.is_file():
|
||||
@@ -586,6 +588,8 @@ def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)')
|
||||
ap.add_argument('--zip', type=Path, help='also write the folder as this zip')
|
||||
ap.add_argument('--pipeline-list', type=Path, metavar='FILE',
|
||||
help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)')
|
||||
ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder')
|
||||
ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3',
|
||||
help='--prove: build and run the check for this CPU level (default v3)')
|
||||
@@ -604,7 +608,7 @@ def main():
|
||||
build_deps(out)
|
||||
pgo = profiles()
|
||||
runtime_lib(out, llvm, pgo)
|
||||
build_sources(out, pgo)
|
||||
build_sources(out, pgo, args.pipeline_list)
|
||||
build_launcher(out, llvm)
|
||||
collect_licenses(out)
|
||||
check_folder(out)
|
||||
|
||||
196
tools/pipeline_list.py
Normal file
196
tools/pipeline_list.py
Normal file
@@ -0,0 +1,196 @@
|
||||
"""The shared pipeline list: graphics pipelines players met, for an install to build before play.
|
||||
|
||||
A game folder records every graphics pipeline a run builds in pipeline-keys.bin (runtime/vk_pipeline.c "pipeline
|
||||
keys": the pipeline state with its shaders named by input id) and every shader variant in shader-answers.bin
|
||||
(runtime/gfx_shader.c "answers met in play": stage, code hash, input id, translator answers). The next start of that
|
||||
folder builds its recorded pipelines ahead of play; another install, or the same one after its caches were removed,
|
||||
meets each of them in play first. A mesh pipeline met that way has no stage libraries to be linked from, so its draws
|
||||
are left out of the picture for its whole compile, 0.2 to 1.1 s.
|
||||
|
||||
This tool folds the two files of any number of game folders into one list file. tools/prepare_shaders.py reads the
|
||||
list beside the shared shader list: the shader variants it names go into the seed, so the preparation translates them,
|
||||
and its pipeline records go next to the game as seed-pipeline-keys.bin, which `nx2navi.exe --prepare-shaders` (and
|
||||
`--prepare-pipelines` on a folder that is prepared already) builds once into the pipeline cache and the graphics
|
||||
driver's own.
|
||||
|
||||
What a record depends on. Its shaders are named by input id, a hash of the game's shader code and the answers, which
|
||||
no translator, GPU or driver changes: a record made on one machine names the same shaders on another with the same
|
||||
game version. Its state is the game's (formats, blend, depth and stencil, topology, sample count) and a few choices of
|
||||
the runtime's settings (a culled foliage shadow pass, the depth range); everything that depends on the GPU is derived
|
||||
from it when the pipeline is built. A record whose state this install never reaches costs one pipeline in the caches,
|
||||
and one whose shader this game version lacks is left out by the preparation. The record layout belongs to the runtime
|
||||
that wrote it (KEYS_VERSION and the record size, in the file's header): a list is made again for a runtime that
|
||||
changes either, and the runtime ignores one that does not match.
|
||||
|
||||
The list is derived from game data: it stays out of the repository (tools/check_content.py refuses it) and travels
|
||||
with the installer archive as data/pipeline_list.bin, as the shader list does.
|
||||
|
||||
python tools/pipeline_list.py merge OUT.bin GAME_DIR [GAME_DIR ...] [--mesh-only]
|
||||
python tools/pipeline_list.py info FILE_OR_GAME_DIR [...]
|
||||
|
||||
File: "NX2NPLST", version u32, answers bytes u32, keys bytes u32, then a shader-answers.bin image (header and the
|
||||
records the keys name, input id kept) and a pipeline-keys.bin image (header and records).
|
||||
"""
|
||||
import argparse
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MAGIC = b'NX2NPLST'
|
||||
VERSION = 1
|
||||
ANS_MAGIC, ANS_REC = b'NX2NSANS', 4 + 32 + 32 + 147 + 1 # gfx_shader.c: stage, code hash, input id, answers, pad
|
||||
KEYS_MAGIC = b'NX2NPKEY'
|
||||
ID = 32
|
||||
STAGE_NAMES = {0: 'vertex', 4: 'fragment', 5: 'compute', 6: 'task', 7: 'mesh'} # runtime/gfx.h GFX_STAGE_*
|
||||
STAGE_MESH = 7
|
||||
|
||||
|
||||
def read_answers(data):
|
||||
"""shader-answers.bin image -> its records (bytes each); [] for another magic or record size."""
|
||||
if len(data) < 16 or data[:8] != ANS_MAGIC:
|
||||
return []
|
||||
_, rec = struct.unpack_from('<II', data, 8)
|
||||
if rec != ANS_REC:
|
||||
return []
|
||||
n = (len(data) - 16) // rec
|
||||
return [data[16 + i * rec:16 + (i + 1) * rec] for i in range(n)]
|
||||
|
||||
|
||||
def read_keys(data):
|
||||
"""pipeline-keys.bin image -> (magic, version, record size, records); records [] for a file too short."""
|
||||
if len(data) < 16:
|
||||
return b'', 0, 0, []
|
||||
magic = data[:8]
|
||||
version, rec = struct.unpack_from('<II', data, 8)
|
||||
if rec < 3 * ID + 8:
|
||||
return magic, version, rec, []
|
||||
n = (len(data) - 16) // rec
|
||||
return magic, version, rec, [data[16 + i * rec:16 + (i + 1) * rec] for i in range(n)]
|
||||
|
||||
|
||||
def key_ids(rec):
|
||||
"""The input ids a key record names: vertex (or mesh), then fragment and task when it has them."""
|
||||
has_fs, has_ts = struct.unpack_from('<II', rec, 3 * ID)
|
||||
ids = [rec[:ID]]
|
||||
if has_fs:
|
||||
ids.append(rec[ID:2 * ID])
|
||||
if has_ts:
|
||||
ids.append(rec[2 * ID:3 * ID])
|
||||
return ids
|
||||
|
||||
|
||||
def read_dir(d):
|
||||
"""A game folder's recorded keys and answers; a missing file gives no records."""
|
||||
d = Path(d)
|
||||
kp, ap = d / 'pipeline-keys.bin', d / 'shader-answers.bin'
|
||||
keys = read_keys(kp.read_bytes()) if kp.is_file() else (b'', 0, 0, [])
|
||||
return keys, read_answers(ap.read_bytes()) if ap.is_file() else []
|
||||
|
||||
|
||||
def merge(dirs, mesh_only=False, log=print):
|
||||
"""One list from the folders' files: each key record once, and the answers record of every shader they name."""
|
||||
header, keys, seen, answers = None, [], set(), {}
|
||||
for d in dirs:
|
||||
(magic, version, rec, recs), ans = read_dir(d)
|
||||
if not recs:
|
||||
log(f'{d}: no pipeline key records')
|
||||
continue
|
||||
if header is None:
|
||||
header = (magic, version, rec)
|
||||
elif header != (magic, version, rec):
|
||||
log(f'{d}: its pipeline-keys.bin is of another layout ({magic!r}, version {version}, {rec} bytes a record); '
|
||||
'left out')
|
||||
continue
|
||||
by_id = {a[36:68]: a for a in ans}
|
||||
added = lacking = 0
|
||||
for r in recs:
|
||||
ids = key_ids(r)
|
||||
if r in seen:
|
||||
continue
|
||||
if any(i not in by_id and i not in answers for i in ids):
|
||||
lacking += 1 # a shader the folder's answers do not name: another install could not translate it
|
||||
continue
|
||||
if mesh_only and by_id.get(ids[0], answers.get(ids[0]))[0:4] != struct.pack('<I', STAGE_MESH):
|
||||
continue
|
||||
seen.add(r)
|
||||
keys.append(r)
|
||||
added += 1
|
||||
for i in ids:
|
||||
answers.setdefault(i, by_id.get(i) or answers[i])
|
||||
log(f'{d}: {len(recs)} key records, {added} new to the list, {lacking} without their shaders\' answers')
|
||||
if header is None:
|
||||
return None
|
||||
return header, keys, list(answers.values())
|
||||
|
||||
|
||||
def pack(header, keys, answers):
|
||||
magic, version, rec = header
|
||||
a = ANS_MAGIC + struct.pack('<II', 1, ANS_REC) + b''.join(answers)
|
||||
k = magic + struct.pack('<II', version, rec) + b''.join(keys)
|
||||
return MAGIC + struct.pack('<III', VERSION, len(a), len(k)) + a + k
|
||||
|
||||
|
||||
def unpack(data):
|
||||
"""A list file -> (answers records, pipeline-keys.bin image); None when it is not one."""
|
||||
if len(data) < 20 or data[:8] != MAGIC:
|
||||
return None
|
||||
version, na, nk = struct.unpack_from('<III', data, 8)
|
||||
if version != VERSION or len(data) != 20 + na + nk:
|
||||
return None
|
||||
return read_answers(data[20:20 + na]), data[20 + na:]
|
||||
|
||||
|
||||
def describe(name, keys_image, answers):
|
||||
magic, version, rec, recs = read_keys(keys_image)
|
||||
stage = {a[36:68]: struct.unpack_from('<I', a)[0] for a in answers}
|
||||
mesh = sum(1 for r in recs if stage.get(r[:ID]) == STAGE_MESH)
|
||||
task = sum(1 for r in recs if struct.unpack_from('<I', r, 3 * ID + 4)[0])
|
||||
named = {i for r in recs for i in key_ids(r)}
|
||||
by = {}
|
||||
for i in named:
|
||||
s = STAGE_NAMES.get(stage.get(i), 'not in the answers')
|
||||
by[s] = by.get(s, 0) + 1
|
||||
print(f'{name}: {len(recs)} pipelines ({mesh} mesh, {task} of them with a task stage), layout {magic.decode("ascii", "replace")} '
|
||||
f'version {version}, {rec} bytes a record; {len(named)} shaders named: '
|
||||
+ ', '.join(f'{v} {k}' for k, v in sorted(by.items())))
|
||||
return len(recs), mesh
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest='cmd', required=True)
|
||||
m = sub.add_parser('merge', help='fold game folders into a list file')
|
||||
m.add_argument('out', type=Path)
|
||||
m.add_argument('dirs', nargs='+', type=Path)
|
||||
m.add_argument('--mesh-only', action='store_true', help='keep the mesh pipelines only')
|
||||
i = sub.add_parser('info', help='what a list file or a game folder holds')
|
||||
i.add_argument('paths', nargs='+', type=Path)
|
||||
a = ap.parse_args()
|
||||
if a.cmd == 'info':
|
||||
for p in a.paths:
|
||||
if p.is_dir():
|
||||
(magic, version, rec, recs), ans = read_dir(p)
|
||||
image = magic + struct.pack('<II', version, rec) + b''.join(recs)
|
||||
describe(str(p), image, ans)
|
||||
else:
|
||||
got = unpack(p.read_bytes())
|
||||
if got is None:
|
||||
print(f'{p}: not a pipeline list')
|
||||
return 1
|
||||
describe(str(p), got[1], got[0])
|
||||
return 0
|
||||
got = merge(a.dirs, a.mesh_only)
|
||||
if got is None:
|
||||
print('no pipeline key records in any folder')
|
||||
return 1
|
||||
header, keys, answers = got
|
||||
data = pack(header, keys, answers)
|
||||
a.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.out.write_bytes(data)
|
||||
describe(str(a.out), unpack(data)[1], answers)
|
||||
print(f'{a.out}: {len(data)} bytes')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -21,6 +21,13 @@ shader-prepared.bin (one record per variant, in priority order) and builds every
|
||||
pipeline once, so the GPU driver's own disk cache holds them (vk_pipeline.c "stage library prebuild"). The seed files
|
||||
can be deleted afterwards.
|
||||
|
||||
The shared pipeline list (tools/pipeline_list.py: graphics pipelines players met, with the shader variants they name)
|
||||
is read beside the shader list when there is one: tools/data/pipeline_list.bin, data/pipeline_list.bin in a release
|
||||
folder, or --pipeline-list. Its variants of shaders this game version has join the seed, and its key records are
|
||||
written as seed-pipeline-keys.bin, which --prepare-shaders builds into the pipeline caches once the seed is translated
|
||||
(vk_pipeline.c "the shared pipeline list"; mesh pipelines by default). That file stays in the build directory:
|
||||
`nx2navi.exe --warm-shaders` and `--prepare-pipelines` build from it again after an update or a new driver.
|
||||
|
||||
A driver whose shader compiler crashes on a variant crashes on it in every run (NVIDIA 615.78.08 on an RTX 5080, issue
|
||||
#30). The translations and shader-prepared.bin are written by then, and the journal the run leaves names the variants
|
||||
the driver was building (vk_pipeline.c skip_init), so only the driver step runs again (`nx2navi.exe --warm-shaders`), up to
|
||||
@@ -47,6 +54,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
while not (ROOT / 'tools' / 'shader_census.py').is_file() and ROOT != ROOT.parent:
|
||||
ROOT = ROOT.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import pipeline_list # noqa: E402
|
||||
import shader_list # noqa: E402
|
||||
from shader_census import inventory # noqa: E402
|
||||
|
||||
@@ -83,6 +91,44 @@ if not BUNDLED_LIST.is_file():
|
||||
BUNDLED_LIST = ROOT / 'data' / 'shader_list.tsv' # release folder layout (tools/make_release.py)
|
||||
|
||||
|
||||
BUNDLED_PIPELINES = ROOT / 'tools' / 'data' / 'pipeline_list.bin' # as BUNDLED_LIST: out of the repository
|
||||
if not BUNDLED_PIPELINES.is_file():
|
||||
BUNDLED_PIPELINES = ROOT / 'data' / 'pipeline_list.bin'
|
||||
SEED_KEYS = 'seed-pipeline-keys.bin'
|
||||
LIST_PRIORITY = 1 # a pipeline list variant's seed priority: with the warm-up rows
|
||||
|
||||
|
||||
def read_pipeline_list(path):
|
||||
"""(answers records, pipeline-keys.bin image) of the pipeline list at path, or of the bundled one; None without
|
||||
a list. A file that is not a list stops the run: a release that names one must not go on without it."""
|
||||
p = Path(path) if path else BUNDLED_PIPELINES
|
||||
if not p.is_file():
|
||||
if path:
|
||||
sys.exit(f'prepare_shaders: no pipeline list at {p}')
|
||||
return None
|
||||
got = pipeline_list.unpack(p.read_bytes())
|
||||
if got is None:
|
||||
sys.exit(f'prepare_shaders: {p} is not a pipeline list (tools/pipeline_list.py merge writes one)')
|
||||
return got
|
||||
|
||||
|
||||
def list_variants(answers_recs, have, seen):
|
||||
"""Seed records for the pipeline list's shader variants: those whose shader (runtime code hash) is in `have` and
|
||||
whose (hash, answers) the seed does not hold yet. The input id is left zero, as for every seed record: the runtime
|
||||
finds it, and finds the id the list's key records name, since it depends on the code and the answers alone."""
|
||||
out, absent = [], 0
|
||||
for a in answers_recs:
|
||||
h, packed = a[4:36], a[68:68 + 147]
|
||||
if h not in have:
|
||||
absent += 1
|
||||
continue
|
||||
if (h, packed) in seen:
|
||||
continue
|
||||
seen.add((h, packed))
|
||||
out.append(a[:36] + bytes(32) + packed + bytes([LIST_PRIORITY]))
|
||||
return out, absent
|
||||
|
||||
|
||||
def read_list(path):
|
||||
if path:
|
||||
return Path(path).read_text(encoding='utf-8')
|
||||
@@ -190,6 +236,8 @@ def main():
|
||||
ap.add_argument('--list', help='a shader_list.tsv instead of tools/data/shader_list.tsv or refs/MacNativeGames')
|
||||
ap.add_argument('--no-run', action='store_true', help='write the seed files only')
|
||||
ap.add_argument('--no-list', action='store_true', help='default variants only, without a shared shader list')
|
||||
ap.add_argument('--pipeline-list', help='a pipeline list instead of tools/data/pipeline_list.bin (tools/pipeline_list.py)')
|
||||
ap.add_argument('--no-pipeline-list', action='store_true', help='no shared pipeline list, also when one is bundled')
|
||||
ap.add_argument('--all-shaders', action='store_true',
|
||||
help='also the default variants SLOW_UNUSED leaves out (an hour more of GPU driver builds)')
|
||||
ap.add_argument('--exe', default='nx2navi.exe' if os.name == 'nt' else 'nx2navi',
|
||||
@@ -235,8 +283,17 @@ def main():
|
||||
rec = struct.pack('<I', stage) + h + bytes(32) + packed + bytes([prio.get(key, 2) | (DEFAULT_FLAG if default else 0)])
|
||||
assert len(rec) == ANS_REC
|
||||
recs.append((prio.get(key, 2), rec))
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
out = a.build.resolve()
|
||||
plist = None if a.no_pipeline_list else read_pipeline_list(a.pipeline_list)
|
||||
(out / SEED_KEYS).unlink(missing_ok=True) # a list this build no longer has
|
||||
if plist:
|
||||
extra, absent = list_variants(plist[0], {h for h, _ in progs.values()}, seen)
|
||||
recs += [(LIST_PRIORITY, r) for r in extra]
|
||||
(out / SEED_KEYS).write_bytes(plist[1])
|
||||
n_keys = len(pipeline_list.read_keys(plist[1])[3])
|
||||
print(f'pipeline list: {n_keys} pipelines; of the {len(plist[0])} shader variants they name, {len(extra)} join the '
|
||||
f'seed, {absent} are of shaders {a.version} does not have, the others are in it already', flush=True)
|
||||
recs.sort(key=lambda r: r[0]) # stable: list order within a priority
|
||||
with open(out / 'seed-programs.bin', 'wb') as f:
|
||||
f.write(b'NX2NSPRG' + struct.pack('<II', 1, 0))
|
||||
for _, body in progs.values():
|
||||
|
||||
Reference in New Issue
Block a user