From 682d9c5c50e647ca6a634b3e959b437be089f045 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 14:52:25 -0500 Subject: [PATCH 1/3] A draw does not wait for the fast link of a mesh pipeline that is known to be compiling. A draw whose pipeline is missing waits up to 3 ms for its fast link before it is skipped, 6 ms a frame in all. A mesh pipeline has no stage libraries and gets no fast link, so for the 0.2 to 1.1 s of its compile every frame that drew with it lost those milliseconds and skipped the draw all the same. Once such a pipeline has been building for longer than the 8 ms a cached one needs, its draws are skipped at once, as the one-shot mesh draws already are. Not run in the game yet. Co-Authored-By: Claude Fable 5.2 --- runtime/vk_exec_draw.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 1f22d537..8e5eea68 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -3974,6 +3974,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int if (linking) t0 = kern_now_ns(); else w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; vk_exec_ahead(ex); + // A mesh pipeline that is known to be compiling gets no fast link, so the few milliseconds a draw waits + // for one are lost in every frame that meets it until the compile ends, 0.2 to 1.1 s later (MESH_WAIT_MS). + if (wait_ms && linking && mesh && vk_pipeline_building_ms(&s) > MESH_WAIT_MS) wait_ms = 0; if (wait_ms) p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); } if (t0 && budget) { From 918d399d55c80f1e88fbd0009c5c05c5baeebf3b Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 14:52:46 -0500 Subject: [PATCH 2/3] Add the shared pipeline list: mesh pipelines players met are built at the shader preparation, before the first start. The preparation translates every shader and builds each one's stage library, so a pipeline a draw meets for the first time is linked in 2 to 6 ms. A mesh pipeline has no stage libraries: on an install that never drew it, it compiles whole in play, 0.2 to 1.1 s with its draws left out of the picture (18 of them on a first visit of Kokiri Forest). Whole pipelines were built ahead only from the folder's own pipeline-keys.bin. tools/pipeline_list.py merge folds the pipeline-keys.bin and shader-answers.bin of game folders that were played into one list file: every key record once, with the answers record of each shader it names. A record names its shaders by input id, which depends on the game's code and the answers alone, and holds the game's pipeline state; what depends on the GPU is derived when the pipeline is built, so a record made on one machine is good on another with the same game version and runtime. Four folders on this PC give 5,432 records, 317 of them mesh pipelines (147 KB of keys). tools/prepare_shaders.py reads the list (tools/data/pipeline_list.bin, data/pipeline_list.bin in a release folder, or --pipeline-list), adds its shader variants to the seed and writes its records as seed-pipeline-keys.bin. nx2navi.exe --prepare-shaders builds the mesh pipelines among them after the seed is translated and saves the pipeline cache before the stage libraries are built (NX2NAVI_PREPARE_PIPELINES=all builds the vertex pipelines too); --warm-shaders does it again after an update or a new driver, and --prepare-pipelines is that step alone. The pipelines go into the caches and are not kept or added to the folder's own key file, so they cost no start-up time or video memory until a draw meets them. A marker file keeps a driver that crashes on one of them from failing every preparation. The list is derived from game data and stays out of the repository: check_content.py refuses files recorded from a game folder by their first bytes, and make_kit.py, make_launcher.py and make_release.py carry a list given with --pipeline-list. All three now also carry pipeline_list.py, which prepare_shaders.py imports. Checked without the game: tests/pipeline_list_test.py, and prepare_shaders.py --no-run on the game files with a list of 317 mesh pipelines (all 266 shader variants found in this game version, 177 new to the seed). The build step in nx2navi.exe compiles and links and has not been run. Co-Authored-By: Claude Fable 5.2 --- .gitignore | 3 + docs/building.md | 16 +++ docs/installer.md | 3 +- runtime/cfg.c | 3 +- runtime/host.h | 1 + runtime/main.c | 6 +- runtime/vk_pipeline.c | 161 +++++++++++++++++++++++++++++ tests/pipeline_list_test.py | 132 ++++++++++++++++++++++++ tools/check_content.py | 22 +++- tools/make_game.py | 3 +- tools/make_kit.py | 10 +- tools/make_launcher.py | 16 ++- tools/make_release.py | 10 +- tools/pipeline_list.py | 196 ++++++++++++++++++++++++++++++++++++ tools/prepare_shaders.py | 59 ++++++++++- 15 files changed, 627 insertions(+), 14 deletions(-) create mode 100644 tests/pipeline_list_test.py create mode 100644 tools/pipeline_list.py diff --git a/.gitignore b/.gitignore index 464aaae3..bea70934 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ __pycache__/ /generated/ # Button art a release carries (tools/make_launcher.py --prompt-art); a checkout may hold it for a test build /prompt-art/ +# The shared pipeline list a release carries (tools/pipeline_list.py merge; tools/prepare_shaders.py reads it here) +/tools/data/pipeline_list.bin +/data/pipeline_list.bin *.xci *.xcz *.nsp diff --git a/docs/building.md b/docs/building.md index de709c79..7174b864 100644 --- a/docs/building.md +++ b/docs/building.md @@ -158,6 +158,22 @@ of them. version; they build at their first draw instead, another driver version builds them again, and so does deleting the file. + What this step leaves for play, and the shared pipeline list. The step translates every shader's default variant + and the shader list's variants and builds each one's stage library or compute pipeline. A whole graphics pipeline + is built ahead only at the start of a game folder that has drawn it before (`pipeline-keys.bin`); a pipeline met + for the first time is linked from its stage libraries in 2 to 6 ms, except a mesh pipeline, which has no libraries + and compiles whole (0.2 to 1.1 s, its draws left out of the picture meanwhile). `tools/pipeline_list.py merge` + folds the `pipeline-keys.bin` and `shader-answers.bin` of game folders that were played into a pipeline list; + `prepare_shaders.py` reads it from `tools\data\pipeline_list.bin` (`data\pipeline_list.bin` in a release folder, or + `--pipeline-list`), adds the shader variants it names to the seed and writes its records next to the game as + `seed-pipeline-keys.bin`, and `--prepare-shaders` builds the mesh pipelines among them into the pipeline cache and + the driver's (`runtime/vk_pipeline.c` `seed_pipelines`; `NX2NAVI_PREPARE_PIPELINES=all` builds the vertex pipelines + too). `--warm-shaders` builds them again, and `nx2navi.exe --prepare-pipelines` does that step alone. The list is + derived from game data: it is not in the repository (`tools/check_content.py` refuses it), and a kit, a launcher + release or an installer release takes one with `--pipeline-list` (`tools/make_kit.py`, `make_launcher.py`, + `make_release.py`). A list belongs to the record layout of the runtime that wrote it; another runtime says so and + builds nothing from it. `python tests\pipeline_list_test.py` checks the tool on made-up folders. + 7. Run. The game keeps running when its window loses the focus (`assume_focus = 1` in `nx2navi.ini`, the default; the Game tab switches it). With 0 it pauses, as it does on the Switch when Home is pressed. diff --git a/docs/installer.md b/docs/installer.md index 7587c906..fe0d8f1e 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -148,7 +148,8 @@ llvm/ llvm-mingw, x86_64-only subset (Apache-2.0 with LLVM excep runtime/ libnx2navirt.a, libnx2navirt-v2.a and the headers the generated code and modules.c/romfs.c/patches.c include bin/ nx2navi_xlate.exe, hactool.exe, SDL3.dll, libopus-0.dll, ninja.exe -data/ shader_list.tsv (MIT, THIRD_PARTY_NOTICES.md); pgo-.profdata + .json, the execution profile an +data/ shader_list.tsv (MIT, THIRD_PARTY_NOTICES.md); pipeline_list.bin when the release was made with + one (building.md, step 6); pgo-.profdata + .json, the execution profile an update's code and the runtime libraries compile with (building.md, "Profile-guided build") THIRD_PARTY_NOTICES.md nx2navi.ini.example, VERSION the default ini; the release string (the repository's VERSION plus the commit) diff --git a/runtime/cfg.c b/runtime/cfg.c index c2a1d3c7..9c36b92d 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -53,7 +53,7 @@ static const char *const known[] = { "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC", - "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", + "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", "SCENE_SHARE", @@ -234,6 +234,7 @@ static const CfgMeta metas[] = { {"PIPE_LINK_DRAW_MS", "3", "1..50", RESTART}, // vk_exec_draw.c link_draw_ms {"PIPE_FRAME_WAIT_MS", "50", "0..500", RESTART}, // vk_exec_draw.c frame_wait {"PIPE_SKIP_STEADY", "1", "0|1", RESTART}, // vk_exec.c steady_on + {"PREPARE_PIPELINES", "mesh", "0|mesh|all", RESTART}, // vk_pipeline.c seed_pipelines {"PIPE_AHEAD", "1", "0|1", RESTART}, // vk_exec.c vk_exec_ahead, vk_pipeline.c vk_pipeline_hint {"GPL_FEEDBACK", "1", "0|1", RESTART}, // vk_pipeline.c gpl_feedback {"PIPE_FAST_WORKERS", "1", "0|1", RESTART}, // vk_pipeline.c lib_worker, fast_take diff --git a/runtime/host.h b/runtime/host.h index e8af6dde..2199afd3 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -389,6 +389,7 @@ int probe_gamepad(void); // input.c int probe_movie(void); // movie_probe.c: host VP9 movie::Decoder on one RomFS movie int vk_pipeline_prepare_shaders(void); // vk_pipeline.c: --prepare-shaders (tools/prepare_shaders.py) int vk_pipeline_warm_shaders(void); // vk_pipeline.c: --warm-shaders (tools/build_game.py) +int vk_pipeline_prepare_pipelines(void); // vk_pipeline.c: --prepare-pipelines (the shared pipeline list) int vk_pipeline_probe_prewarm(void); // vk_pipeline.c: --probe-prewarm // vk_device.c: the drivers' own shader caches beside the exe (MESA_SHADER_CACHE_DIR, on NVIDIA's Linux driver // __GL_SHADER_DISK_CACHE_PATH and _SIZE) in the environment, once; before any thread can load a Vulkan driver diff --git a/runtime/main.c b/runtime/main.c index 0462cedd..ad9af950 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -529,7 +529,7 @@ static int usage(const char *argv0) { "--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " "--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | " - "--prepare-shaders | --warm-shaders | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0); + "--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0); return 2; } @@ -586,7 +586,8 @@ int main(int argc, char **argv) { // threads, so narrowing them to one die would only make the preparation take longer. int prepare = 0; for (int i = 1; i < argc; i++) - if (!strcmp(argv[i], "--prepare-shaders") || !strcmp(argv[i], "--warm-shaders")) prepare = 1; + if (!strcmp(argv[i], "--prepare-shaders") || !strcmp(argv[i], "--warm-shaders") || + !strcmp(argv[i], "--prepare-pipelines")) prepare = 1; if (!prepare) perf_cpu_start(); { // NX2NAVI_POWER_FULL=0 leaves Windows' power throttling of an unfocused window in place (A/B runs) const char *pf = cfg_get("NX2NAVI_POWER_FULL"); @@ -679,6 +680,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--probe-movie")) return probe_movie(); if (!strcmp(mode, "--prepare-shaders")) return vk_pipeline_prepare_shaders(); if (!strcmp(mode, "--warm-shaders")) return vk_pipeline_warm_shaders(); + if (!strcmp(mode, "--prepare-pipelines")) return vk_pipeline_prepare_pipelines(); if (!strcmp(mode, "--probe-prewarm")) return vk_pipeline_probe_prewarm(); if (!strcmp(mode, "--boot")) { cfg.boot = 1; rt_boot(); return 0; } return usage(argv[0]); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index d1a26deb..52865614 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -5823,6 +5823,163 @@ static void lb_warm_prepared(const char *dir) { free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0, LB.skipped = 0; } +// ---- the shared pipeline list (tools/pipeline_list.py; seed-pipeline-keys.bin next to the game) +// tools/prepare_shaders.py writes the list's key records there, in the layout of pipeline-keys.bin, after putting the +// shader variants they name into the seed. The preparation builds them once, after the seed is translated: each +// pipeline goes through the pipeline cache and so into pipeline-cache.bin and the driver's own disk cache, and is +// dropped with the process. Nothing is added to this folder's pipeline-keys.bin: the boot keeps building only what +// this install drew, and a listed pipeline costs no start-up time or video memory until a draw meets it. That draw +// then finds its pipeline in the caches: a few milliseconds for a mesh pipeline, which has no stage libraries and +// otherwise leaves its draws out of the picture for its whole compile (0.2 to 1.1 s each on a first visit of a forest, +// 18 of them). +// NX2NAVI_PREPARE_PIPELINES: mesh (default) builds the list's mesh pipelines only; all, every record; 0, none. A +// vertex pipeline is linked from its prepared stage libraries in 2 to 6 ms when a draw misses it, and its module +// depends on the vertex output union of the pairs this install has recorded (vs_outs), so one built here is not +// certain to be the one a draw asks for. +// A driver whose compiler crashes on one of these would crash at every preparation: seed-pipelines-running.txt marks +// the step, and a run that finds the marker of a run that died skips the step for good on this folder (the pipelines +// are then built at their first draw, as without a list) until the marker is deleted. +static int cache_save(uint32_t wait_ms); + +static int seed_pipelines(const char *dir) { + const char *mode = cfg_value("NX2NAVI_PREPARE_PIPELINES"); + int all = !strcmp(mode, "all"); + char path[PLAT_PATH_MAX * 2 + 48], mark[PLAT_PATH_MAX * 2 + 48]; + snprintf(path, sizeof path, "%s/seed-pipeline-keys.bin", dir); + snprintf(mark, sizeof mark, "%s/seed-pipelines-running.txt", dir); + FILE *f = fopen(path, "rb"); + if (!f) return 0; // no list: nothing to say + if (!strcmp(mode, "0")) { + fclose(f); + printf("prepare-pipelines: off (NX2NAVI_PREPARE_PIPELINES=0)\n"); + return 0; + } + if (plat_file_exists(mark)) { + fclose(f); + printf("prepare-pipelines: skipped: an earlier preparation ended while it built the listed pipelines (%s); they " + "are built at their first draw\n", mark); + return 0; + } + KeysHeader h; + fseek(f, 0, SEEK_END); + long len = ftell(f); + fseek(f, 0, SEEK_SET); + size_t n = 0; + KeyRec *r = NULL; + if (len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8) && + h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec)) { + n = ((size_t)len - sizeof h) / sizeof(KeyRec); + r = malloc(n * sizeof *r + 1); + if (r && fread(r, sizeof *r, n, f) != n) n = 0; + } else { + printf("prepare-pipelines: %s is a list for another version of the game's runtime (its record layout differs); " + "no pipeline is built from it\n", path); + } + fclose(f); + uint8_t (*ids)[32] = n ? malloc(3 * n * 32) : NULL; + const GfxShader **sh = n ? calloc(3 * n, sizeof *sh) : NULL; + if (!r || !ids || !sh) { + free(r), free(ids), free(sh); + return 0; + } + double t0 = now_ms(); + size_t m = 0, u = 0; + for (size_t i = 0; i < n; i++) { + memcpy(ids[m++], r[i].vs, 32); + if (r[i].has_fs) memcpy(ids[m++], r[i].fs, 32); + if (r[i].has_ts) memcpy(ids[m++], r[i].ts, 32); + } + qsort(ids, m, 32, id_cmp); + for (size_t i = 0; i < m; i++) + if (!u || memcmp(ids[u - 1], ids[i], 32)) memcpy(ids[u++], ids[i], 32); + int cpus = (int)plat_cpu_usable(); + const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); + int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2; + gfx_shader_prewarm((const uint8_t(*)[32])ids, u, sh, cpus, procs < 1 ? 1 : procs); + static const char note[] = "the listed pipelines are being built; a run that finds this file skips them\n"; + plat_file_write_atomic(mark, note, sizeof note - 1); + uint32_t queued = 0, dropped = 0, known = 0, other = 0, mesh = 0; + plat_mutex_lock(&P.lock); + uint32_t built0 = K.built, failed0 = K.failed; + K.loading = 1; + for (size_t i = 0; i < n; i++) { + const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp); + const uint8_t (*fi)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL; + const uint8_t (*t)[32] = r[i].has_ts ? bsearch(r[i].ts, ids, u, 32, id_cmp) : NULL; + const GfxShader *vs = v ? sh[v - ids] : NULL, *fs = fi ? sh[fi - ids] : NULL, *ts = t ? sh[t - ids] : NULL; + if (!vs || (r[i].has_fs && !fs) || (r[i].has_ts && !ts)) { + dropped++; + continue; + } + VkPipeState s = r[i].s; + s.vs = vs, s.fs = fs, s.ts = ts; + if (!is_mesh(&s) && !all) { + other++; + continue; + } + if (!is_mesh(&s) && P.indep && fs && gfx_shader_stage(vs) == GFX_STAGE_VS) { // its vertex output union, as the boot + GfxShaderIo io[32]; + vs_outs(vs, fs, io); + } + GKey k; + gkey_fill(&s, &k); + uint64_t hk = key_hash(&k, sizeof k); + int q = 0; + if (pipe_find(&k, sizeof k, hk)) known++; + else pipe_miss(&s, &k, hk, 2, &q), queued += (uint32_t)q, mesh += q && is_mesh(&s); + } + K.loading = 0; + K.until = 0; // every one of them: no boot waits for this + plat_mutex_unlock(&P.lock); + free(ids), free(sh), free(r); + PlatThread *th[64]; + int nt = 0; + for (int i = 0; i < cpus && i < 64 && queued; i++) + if ((th[nt] = plat_thread_new(warm_builder, NULL, "pipe list"))) nt++; + if (!nt && queued) warm_builder(NULL); + for (int i = 0; i < nt; i++) + while (!plat_thread_join_ns(th[i], 2000000000u)) { + printf("prepare-pipelines: progress %u %u\n", + __atomic_load_n(&K.built, __ATOMIC_RELAXED) + __atomic_load_n(&K.failed, __ATOMIC_RELAXED) - built0 - failed0, + queued); + fflush(stdout); + } + plat_mutex_lock(&P.lock); + while (K.left) plat_cond_wait(&done_cv, &P.lock); // those a pool worker took + uint32_t built = K.built - built0, failed = K.failed - failed0; + plat_mutex_unlock(&P.lock); + int saved = cache_save(5000); // as a save at exit: it waits that long for a moment with no build running + plat_file_delete(mark); + char msg[640]; + snprintf(msg, sizeof msg, "prepare-pipelines: %zu listed pipelines: %u built (%u mesh), %u failed, %u the same as one " + "built before, %u without a shader this game version or its shader cache has, %u vertex pipelines left to " + "their first draw (NX2NAVI_PREPARE_PIPELINES=%s); on %d threads in %.0f ms; pipeline cache %s", n, built, + mesh, failed, known, dropped, other, all ? "all" : "mesh", nt ? nt : 1, now_ms() - t0, + saved ? "saved" : "not saved (builds kept running)"); + printf("%s\n", msg); + fflush(stdout); + rt_log("vk_pipeline: %s", msg); + fb_log("after the listed pipelines"); + return 0; +} + +// nx2navi.exe --prepare-pipelines: the list's pipelines alone, for a folder whose shaders are prepared already. +int vk_pipeline_prepare_pipelines(void) { + if (!ensure_init()) { + fprintf(stderr, "prepare-pipelines: no Vulkan device\n"); + return 1; + } + char dir[PLAT_PATH_MAX * 2]; + if (!cache_dir(dir, sizeof dir)) return 1; + char path[PLAT_PATH_MAX * 2 + 48]; + snprintf(path, sizeof path, "%s/seed-pipeline-keys.bin", dir); + if (!plat_file_exists(path)) { + fprintf(stderr, "prepare-pipelines: no %s (tools/prepare_shaders.py writes it from the pipeline list)\n", path); + return 1; + } + return seed_pipelines(dir); +} + // nx2navi.exe --prepare-shaders (tools/prepare_shaders.py): the device sets the translator's host answers (they are in every // cache key), the seed next to nx2navi.exe is translated on every core, then every prepared variant's stage library or // compute pipeline is built once into the driver's disk cache (lb_warm_prepared). No game runs. @@ -5837,6 +5994,9 @@ int vk_pipeline_prepare_shaders(void) { const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2; int rc = gfx_shader_prepare_seed(dir, cpus, procs < 1 ? 1 : procs); + // The listed pipelines before the stage libraries: the pipeline cache is saved with them in it and without the + // tens of thousands of libraries the next step builds, which would take the file past its size limit. + if (!rc) seed_pipelines(dir); if (!rc) lb_warm_prepared(dir); return rc; } @@ -5858,6 +6018,7 @@ int vk_pipeline_warm_shaders(void) { fprintf(stderr, "warm-shaders: no %s (tools/prepare_shaders.py writes it)\n", prp); return 1; } + seed_pipelines(dir); // before the libraries, as in vk_pipeline_prepare_shaders lb_warm_prepared(dir); return 0; } diff --git a/tests/pipeline_list_test.py b/tests/pipeline_list_test.py new file mode 100644 index 00000000..6c110dae --- /dev/null +++ b/tests/pipeline_list_test.py @@ -0,0 +1,132 @@ +"""tools/pipeline_list.py on made-up game folders: records folded once, a key whose shader the folder's answers do not +name left out, --mesh-only, a folder of another record layout left out, the file read back, and what +tools/prepare_shaders.py takes from a list for its seed. Also tools/check_content.py refusing such files in a commit. +No game files; stdlib only. + +Run: python tests/pipeline_list_test.py +""" +import hashlib +import struct +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +import check_content # noqa: E402 +import pipeline_list as pl # noqa: E402 + +REC = 464 # a KeyRec of the runtime that wrote the test's files: three ids, two flags, the state +VS, FS, TS, MS = 0, 4, 6, 7 + + +def h(seed): + return hashlib.sha256(seed.encode()).digest() + + +def answer(seed, stage, salt=0): + """A shader-answers.bin record: stage, code hash, input id, 147 bytes of answers, the pad byte.""" + return struct.pack(' bool: return path.suffix.lower() == ".spv" or path.name.lower().endswith("_spv.h") +# What a game folder records in play and what tools/pipeline_list.py folds from it: the game's shader code, the +# answers it was translated for, the pipelines it drew. All of it is derived from game data. +RECORD_MAGICS = (b"NX2NSPRG", b"NX2NSANS", b"NX2NPKEY", b"NX2NPLST") + + +def recorded_in_play(path: Path, data: bytes | None) -> bool: + """A shader-programs.bin, shader-answers.bin, pipeline-keys.bin or a pipeline list, by its first bytes. A build kit + or a release folder may carry a pipeline list (data/pipeline_list.bin), so make_kit.py and make_release.py, which + call problems_for, are not held to this.""" + if data is None: + if not path.is_file(): + return False + with open(path, "rb") as f: + data = f.read(8) + return data[:8] in RECORD_MAGICS + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--staged", action="store_true", help="check staged files only") @@ -100,9 +117,12 @@ def main() -> int: for path in list_files(args.staged): # --staged checks what the commit takes: a marker staged and then removed from the working copy alone # passed when the file on disk was read - problems = problems_for(path, data=staged_bytes(path) if args.staged else None) + data = staged_bytes(path) if args.staged else None + problems = problems_for(path, data=data) if compiled_shader(path): problems.append("compiled shader: the repository holds the source, tools/build_shaders.py builds this") + if recorded_in_play(path, data): + problems.append("recorded from a game folder (shaders, answers, pipeline keys or a pipeline list)") for problem in problems: print(f"{path}: {problem}") failed = True diff --git a/tools/make_game.py b/tools/make_game.py index 542b5b67..b931f777 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1672,7 +1672,8 @@ def step_shaders(out, update=False): if rc: fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it ' 'meets them. Run the build again to retry this step') - for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them + # translated now: the shader cache holds them. seed-pipeline-keys.bin stays: --warm-shaders builds from it again + for seed in ('seed-programs.bin', 'seed-answers.bin'): (out / seed).unlink(missing_ok=True) pack_shader_cache(out) diff --git a/tools/make_kit.py b/tools/make_kit.py index 6c77b9cb..588c1e9d 100644 --- a/tools/make_kit.py +++ b/tools/make_kit.py @@ -28,7 +28,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] NAME = 'NX2navi-BuildKit' TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', - 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', + 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', 'build_game.py', 'check_isa.py', 'build_shaders.py', 'make_dlaa_mask.py', 'make_mvobj_fs.py', 'pack_shader_cache.py', 'vulkan_pin.py') # make_game.py imports build_game's vma_include and vulkan_include on # Linux for the VMA header and for Vulkan headers newer than the system's (which build_game takes from vulkan_pin), and @@ -60,6 +60,8 @@ def main(): help='the Streamline SDK folder instead of downloading the SDK release with --dlssg') ap.add_argument('--prompt-art', type=Path, nargs='+', metavar='FOLDER', help='the button art: folders holding XGamepad, P5Gamepad and Keyboard_Mouse, or those folders') + ap.add_argument('--pipeline-list', type=Path, metavar='FILE', + help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)') ap.add_argument('--out', type=Path, default=ROOT / 'build' / 'kit') ap.add_argument('--zip', action='store_true') a = ap.parse_args() @@ -100,6 +102,12 @@ def main(): if a.prompt_art: # make_game.py draws the button prompts from it (its prompts step) n, size = prompt_art.copy_art(a.prompt_art, kit / prompt_art.FOLDER) print(f'button art: {n} pictures, {size / 1e6:.1f} MB') + if a.pipeline_list: # prepare_shaders.py reads it there (BUNDLED_PIPELINES) + import pipeline_list + if pipeline_list.unpack(a.pipeline_list.read_bytes()) is None: + sys.exit(f'{a.pipeline_list} is not a pipeline list (tools/pipeline_list.py merge writes one)') + (kit / 'tools' / 'data').mkdir(exist_ok=True) + shutil.copy2(a.pipeline_list, kit / 'tools' / 'data' / 'pipeline_list.bin') import check_content bad = [f'{p.relative_to(kit)}: {m}' for p in sorted(kit.rglob('*')) if p.is_file() for m in check_content.problems_for(p, allow_large=p.suffix in ('.exe', '.dll'))] # the translator, the DLLs diff --git a/tools/make_launcher.py b/tools/make_launcher.py index 373a0ed5..f320e542 100644 --- a/tools/make_launcher.py +++ b/tools/make_launcher.py @@ -77,7 +77,7 @@ PINS = { 'ac87609db3f715404dd41a989c2f1706ea1512f3d86217f938d11c63af8eb505'), } TOOLS = ('make_game.py', 'nup.py', 'romfs_index.py', 'check_content.py', 'hostmem.py', 'prepare_shaders.py', - 'shader_census.py', 'shader_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', + 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'zstd_compat.py', 'pad_prompts.py', 'prompt_art.py', 'button_icons.py', 'romfs_check.py', 'pack_shader_cache.py') TOP = ('LICENSE', 'LICENSE-ADDITIONS', 'THIRD_PARTY_NOTICES.md', 'nx2navi.ini.example') SKIP_DIRS = {'__pycache__', '.pytest_cache', 'native'} @@ -177,7 +177,7 @@ def build_python(py): # ---------------------------------------------------------------- app// -def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None): +def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, dlssg=None, pipelines=None): # The runtime's own shaders are source in the repository and SPIR-V headers in a release: a player's PC compiles # runtime/ (make_game.py) and has no shader compiler. Build the missing and the stale ones before the copy. build_shaders.build(ROOT / 'runtime') @@ -205,6 +205,10 @@ def build_app(app, ver, xlate, feed_repo, lock, ffx=None, dlss=None, art=None, d if art: # make_game.py draws the button prompts from it (its prompts step) n, size = prompt_art.copy_art(art, app / prompt_art.FOLDER) say(f'button art: {n} pictures, {size / 1e6:.1f} MB') + if pipelines: # prepare_shaders.py reads it there (BUNDLED_PIPELINES) + (app / 'tools' / 'data').mkdir(parents=True, exist_ok=True) + shutil.copy2(pipelines, app / 'tools' / 'data' / 'pipeline_list.bin') + say(f'pipeline list: {pipelines.stat().st_size / 1e6:.1f} MB') (app / 'toolchain.lock.json').write_text(json.dumps(lock, indent=1), encoding='utf-8') (app / 'feed.json').write_text(json.dumps({'type': 'github', 'repo': feed_repo}, indent=1), encoding='utf-8') @@ -324,6 +328,8 @@ def main(): 'nvngx_dlssg.dll files, instead of downloading the SDK release with --dlssg') ap.add_argument('--prompt-art', type=Path, nargs='+', metavar='FOLDER', help='the button art: folders holding XGamepad, P5Gamepad and Keyboard_Mouse, or those folders') + ap.add_argument('--pipeline-list', type=Path, metavar='FILE', + help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)') ap.add_argument('--out', type=Path, default=MAIN_ROOT / 'build' / 'launcher-release') ap.add_argument('--feed', default=DEFAULT_FEED, help='the GitHub repository the launcher looks for releases in') ap.add_argument('--notes', type=Path, help='release notes (markdown) for the manifest and the GitHub release') @@ -356,7 +362,11 @@ def main(): dlssg = (a.dlssg, streamline_pin.resolve(a.streamline, DL)) if a.dlssg else None say(f'DLSS Frame Generation: {a.dlssg}, Streamline {streamline_pin.VERSION}: {dlssg[1]}' if dlssg else 'DLSS Frame Generation left out (no --dlssg)') - build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx, dlss, a.prompt_art, dlssg) + if a.pipeline_list: + import pipeline_list + if pipeline_list.unpack(a.pipeline_list.read_bytes()) is None: + sys.exit(f'{a.pipeline_list} is not a pipeline list (tools/pipeline_list.py merge writes one)') + build_app(folder / 'app' / ver, ver, a.xlate, a.feed, lock, a.ffx, dlss, a.prompt_art, dlssg, a.pipeline_list) (folder / 'app' / 'current.txt').write_text(ver + '\n', encoding='utf-8') build_python(folder / 'python') cc, windres = native_compiler(lock) diff --git a/tools/make_release.py b/tools/make_release.py index 2a3692aa..0b36ee0b 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -113,7 +113,7 @@ LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'button_icons.py', 'check_content.py', 'check_isa.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py', 'ncz.py', 'pack_shader_cache.py', 'prepare_shaders.py', - 'romfs_index.py', 'shader_census.py', 'shader_list.py', 'ultrawide_layout.py', 'cheats.py', + 'romfs_index.py', 'shader_census.py', 'shader_list.py', 'pipeline_list.py', 'ultrawide_layout.py', 'cheats.py', 'exefs_patches.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon @@ -427,7 +427,7 @@ def runtime_lib(out, llvm, pgo): shutil.copy2(ROOT / 'runtime' / name, rt / name) -def build_sources(out, pgo): +def build_sources(out, pgo, pipelines=None): copy_tree(ROOT / 'recomp', out / 'recomp', lambda r: not r.endswith(('.py', '.txt')) or '__pycache__' in r) for f in TOOLS_FILES: (out / 'tools').mkdir(exist_ok=True) @@ -437,6 +437,8 @@ def build_sources(out, pgo): (out / 'data').mkdir(exist_ok=True) text = shader_list.history(MAIN_ROOT / 'refs' / 'MacNativeGames', MNG_COMMIT) (out / 'data' / 'shader_list.tsv').write_text(text, encoding='utf-8', newline='\n') + if pipelines: # prepare_shaders.py reads it there (BUNDLED_PIPELINES) + shutil.copy2(pipelines, out / 'data' / 'pipeline_list.bin') for p in pgo.values(): for f in (p, p.with_suffix('.json')): if f.is_file(): @@ -586,6 +588,8 @@ def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)') ap.add_argument('--zip', type=Path, help='also write the folder as this zip') + ap.add_argument('--pipeline-list', type=Path, metavar='FILE', + help='the shared pipeline list to carry (tools/pipeline_list.py merge writes it; it is derived from game data and stays out of the repository)') ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder') ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3', help='--prove: build and run the check for this CPU level (default v3)') @@ -604,7 +608,7 @@ def main(): build_deps(out) pgo = profiles() runtime_lib(out, llvm, pgo) - build_sources(out, pgo) + build_sources(out, pgo, args.pipeline_list) build_launcher(out, llvm) collect_licenses(out) check_folder(out) diff --git a/tools/pipeline_list.py b/tools/pipeline_list.py new file mode 100644 index 00000000..c86d2f94 --- /dev/null +++ b/tools/pipeline_list.py @@ -0,0 +1,196 @@ +"""The shared pipeline list: graphics pipelines players met, for an install to build before play. + +A game folder records every graphics pipeline a run builds in pipeline-keys.bin (runtime/vk_pipeline.c "pipeline +keys": the pipeline state with its shaders named by input id) and every shader variant in shader-answers.bin +(runtime/gfx_shader.c "answers met in play": stage, code hash, input id, translator answers). The next start of that +folder builds its recorded pipelines ahead of play; another install, or the same one after its caches were removed, +meets each of them in play first. A mesh pipeline met that way has no stage libraries to be linked from, so its draws +are left out of the picture for its whole compile, 0.2 to 1.1 s. + +This tool folds the two files of any number of game folders into one list file. tools/prepare_shaders.py reads the +list beside the shared shader list: the shader variants it names go into the seed, so the preparation translates them, +and its pipeline records go next to the game as seed-pipeline-keys.bin, which `nx2navi.exe --prepare-shaders` (and +`--prepare-pipelines` on a folder that is prepared already) builds once into the pipeline cache and the graphics +driver's own. + +What a record depends on. Its shaders are named by input id, a hash of the game's shader code and the answers, which +no translator, GPU or driver changes: a record made on one machine names the same shaders on another with the same +game version. Its state is the game's (formats, blend, depth and stencil, topology, sample count) and a few choices of +the runtime's settings (a culled foliage shadow pass, the depth range); everything that depends on the GPU is derived +from it when the pipeline is built. A record whose state this install never reaches costs one pipeline in the caches, +and one whose shader this game version lacks is left out by the preparation. The record layout belongs to the runtime +that wrote it (KEYS_VERSION and the record size, in the file's header): a list is made again for a runtime that +changes either, and the runtime ignores one that does not match. + +The list is derived from game data: it stays out of the repository (tools/check_content.py refuses it) and travels +with the installer archive as data/pipeline_list.bin, as the shader list does. + + python tools/pipeline_list.py merge OUT.bin GAME_DIR [GAME_DIR ...] [--mesh-only] + python tools/pipeline_list.py info FILE_OR_GAME_DIR [...] + +File: "NX2NPLST", version u32, answers bytes u32, keys bytes u32, then a shader-answers.bin image (header and the +records the keys name, input id kept) and a pipeline-keys.bin image (header and records). +""" +import argparse +import struct +import sys +from pathlib import Path + +MAGIC = b'NX2NPLST' +VERSION = 1 +ANS_MAGIC, ANS_REC = b'NX2NSANS', 4 + 32 + 32 + 147 + 1 # gfx_shader.c: stage, code hash, input id, answers, pad +KEYS_MAGIC = b'NX2NPKEY' +ID = 32 +STAGE_NAMES = {0: 'vertex', 4: 'fragment', 5: 'compute', 6: 'task', 7: 'mesh'} # runtime/gfx.h GFX_STAGE_* +STAGE_MESH = 7 + + +def read_answers(data): + """shader-answers.bin image -> its records (bytes each); [] for another magic or record size.""" + if len(data) < 16 or data[:8] != ANS_MAGIC: + return [] + _, rec = struct.unpack_from(' (magic, version, record size, records); records [] for a file too short.""" + if len(data) < 16: + return b'', 0, 0, [] + magic = data[:8] + version, rec = struct.unpack_from(' (answers records, pipeline-keys.bin image); None when it is not one.""" + if len(data) < 20 or data[:8] != MAGIC: + return None + version, na, nk = struct.unpack_from(' Date: Tue, 10 Nov 2026 14:56:56 -0500 Subject: [PATCH 3/3] An update that carries the shaders over builds the kit's current pipeline list, and a GPU without mesh shaders skips the list's mesh pipelines. The carry path of an update does not run the shader preparation: it runs --warm-shaders, which builds the listed pipelines from the seed-pipeline-keys.bin the game folder already has. make_game.py now writes that file from the kit's list first, so a newer list reaches an updated install. Its shader variants are not translated on this path; a record whose variant the folder has neither prepared nor met is left out and counted. On a GPU that has no mesh shaders every mesh record failed with a log line each. They are counted in the step's line instead. Co-Authored-By: Claude Fable 5.2 --- runtime/vk_pipeline.c | 12 ++++++++---- tools/make_game.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 52865614..2cfa06f2 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -5898,7 +5898,7 @@ static int seed_pipelines(const char *dir) { gfx_shader_prewarm((const uint8_t(*)[32])ids, u, sh, cpus, procs < 1 ? 1 : procs); static const char note[] = "the listed pipelines are being built; a run that finds this file skips them\n"; plat_file_write_atomic(mark, note, sizeof note - 1); - uint32_t queued = 0, dropped = 0, known = 0, other = 0, mesh = 0; + uint32_t queued = 0, dropped = 0, known = 0, other = 0, mesh = 0, no_mesh = 0; plat_mutex_lock(&P.lock); uint32_t built0 = K.built, failed0 = K.failed; K.loading = 1; @@ -5917,6 +5917,10 @@ static int seed_pipelines(const char *dir) { other++; continue; } + if (is_mesh(&s) && !vk_ctx()->has_mesh_shader) { // this GPU draws no mesh pipeline + no_mesh++; + continue; + } if (!is_mesh(&s) && P.indep && fs && gfx_shader_stage(vs) == GFX_STAGE_VS) { // its vertex output union, as the boot GfxShaderIo io[32]; vs_outs(vs, fs, io); @@ -5953,9 +5957,9 @@ static int seed_pipelines(const char *dir) { char msg[640]; snprintf(msg, sizeof msg, "prepare-pipelines: %zu listed pipelines: %u built (%u mesh), %u failed, %u the same as one " "built before, %u without a shader this game version or its shader cache has, %u vertex pipelines left to " - "their first draw (NX2NAVI_PREPARE_PIPELINES=%s); on %d threads in %.0f ms; pipeline cache %s", n, built, - mesh, failed, known, dropped, other, all ? "all" : "mesh", nt ? nt : 1, now_ms() - t0, - saved ? "saved" : "not saved (builds kept running)"); + "their first draw (NX2NAVI_PREPARE_PIPELINES=%s), %u mesh pipelines this GPU cannot draw; on %d threads in " + "%.0f ms; pipeline cache %s", n, built, mesh, failed, known, dropped, other, all ? "all" : "mesh", no_mesh, + nt ? nt : 1, now_ms() - t0, saved ? "saved" : "not saved (builds kept running)"); printf("%s\n", msg); fflush(stdout); rt_log("vk_pipeline: %s", msg); diff --git a/tools/make_game.py b/tools/make_game.py index b931f777..39767037 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1622,6 +1622,20 @@ def shader_run(cmd, cwd=None): return rc +def refresh_pipeline_seed(out): + """The kit's pipeline list (tools/pipeline_list.py) as the game folder's seed-pipeline-keys.bin, so that an update + which carries the shaders over builds a newer list's pipelines too: `--warm-shaders` builds from that file. The + list's shader variants are not translated on this path, so a record whose variant this folder has neither prepared + nor met is left out by the game, which counts those in its line.""" + lst = KIT / 'tools' / 'data' / 'pipeline_list.bin' + if not lst.is_file(): + return + import pipeline_list + got = pipeline_list.unpack(lst.read_bytes()) + if got: + (out / 'seed-pipeline-keys.bin').write_bytes(got[1]) + + def warm_driver_cache(out): """After an update that carries the shaders over: every prepared stage library is built into the graphics driver's cache again (`nx2navi.exe --warm-shaders`). It takes seconds while the driver still has them. It takes minutes when @@ -1633,6 +1647,7 @@ def warm_driver_cache(out): exe = out / EXE_NAME if not exe.is_file() or not (out / 'shader-prepared.bin').is_file(): return + refresh_pipeline_seed(out) say(" the prepared shaders are built into the graphics driver's cache again") t0 = time.monotonic() try: