diff --git a/docs/building.md b/docs/building.md index 126feecc..382530ae 100644 --- a/docs/building.md +++ b/docs/building.md @@ -394,6 +394,11 @@ A change to how `pipeline-cache.bin` is read (`runtime/vk_cache_file.h`, `runtim through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU; about 2 GB of memory for a few seconds. +A change to the layout of `shader-prepared.bin` (`runtime/prp_file.h`, `runtime/gfx_shader.c` +`gfx_shader_prepare_seed`, `tools/make_game.py` `prepared_from`) passes `python tests\prp_file_check.py`, which builds +the header alone, checks the parse of both versions and has `make_game.py` read the list the C side wrote. No game +files, no GPU. + A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice.h`, `runtime/vk_memory.c` `vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and its reason by platform, by GPU driver and by what the player set. No game files, no GPU. diff --git a/docs/launcher.md b/docs/launcher.md index a5570ba1..65eece2b 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -263,7 +263,11 @@ The stub swaps `python.next` in while no Python runs, then starts the new versio game files are already unpacked builds on from them as well (`jobs.InstallJob.args`). 7. **Prepare the Shaders** is skipped once shaders were prepared before (`shader-prepared.bin`): the game keeps using them and translates them again with the new translator in the background (`NX2NAVI_SHADER_CARRY`). With - `shader_carry = 0`, or nothing prepared yet, the step prepares them all as at the install. The step still builds + `shader_carry = 0`, or nothing prepared yet, the step prepares them all as at the install. So it does when the + list was prepared from another shader seed than the one this kit writes: `shader-prepared.bin` carries the digests + of the two seed files and the translator identity (`runtime/prp_file.h`), the step writes the seed first + (`prepare_shaders.py --no-run`, seconds) and compares (`make_game.py` `seed_changed`); a list from before it + recorded its seed is prepared again once. Another translator alone does not prepare again. The step still builds the prepared shaders into the graphics driver's cache again (`make_game.py` `warm_driver_cache`): seconds while the driver has them, 20 minutes on a player's PC when the update changed what they are built with. Its bar follows that count from 0 to 100 %, and the Updates row says an update takes from a few minutes to half an hour diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index f35468cf..132c395e 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -80,6 +80,7 @@ #include "gfx.h" #include "host.h" #include "platform/platform.h" +#include "prp_file.h" #define CONTROL_SIZE 0x880 #define CODE_SKIP 0x30 // graphics code starts after the first 0x30 bytes of the program @@ -2820,10 +2821,40 @@ void gfx_shader_prepare_recorded(int threads, int procs) { } // ---- prepared variants (shader-prepared.bin next to pipeline-keys.bin) -// "NX2NSPRP", version, record size; records: stage, priority, 2 zero bytes, input id. gfx_shader_prepare_seed writes +// The layout is prp_file.h's: a head that says what the list was prepared from (the two seed files and the +// translator), then one record per variant: stage, priority, 2 zero bytes, input id. gfx_shader_prepare_seed writes // them in priority order, one per input id; gfx_shader_warm_list reads them at boot. -#define PRP_VERSION 1 -#define PRP_REC 36 + +// The sha256 of the whole file; 0 when it cannot be read. +static int file_sha256(const char *path, uint8_t out[32]) { + FILE *f = fopen(path, "rb"); + if (!f) return 0; + fseek(f, 0, SEEK_END); + long len = ftell(f); // a seed past 2 GB is not read by rec_read either + fseek(f, 0, SEEK_SET); + uint8_t *b = len > 0 ? malloc((size_t)len) : NULL; + int ok = b && fread(b, 1, (size_t)len, f) == (size_t)len; + fclose(f); + if (ok) rt_sha256(b, (size_t)len, out); + free(b); + return ok; +} + +// The records of the prepared list at path (PRP_REC bytes each, malloc'd; their count in *n), NULL without any. +// *from, *has_from (both optional): what a version 2 file says it was prepared from. +uint8_t *gfx_shader_prepared_read(const char *path, size_t *n, PrpFrom *from, int *has_from) { + uint64_t size = 0; + uint8_t *f = plat_file_read(path, &size), *out = NULL; + PrpFrom fr = {{0}}; + int has = 0; + const uint8_t *recs = prp_parse(f, f ? size : 0, n, &fr, &has); + if (recs && (out = malloc(*n * PRP_REC))) memcpy(out, recs, *n * PRP_REC); + if (!out) *n = 0; + if (from) *from = fr; + if (has_from) *has_from = has; + free(f); + return out; +} int gfx_shader_prepare_seed(const char *dir, int threads, int procs) { char ans[PATH_CAP + 32], prg[PATH_CAP + 32], out[PATH_CAP + 32]; @@ -2833,6 +2864,23 @@ int gfx_shader_prepare_seed(const char *dir, int threads, int procs) { double t0 = (double)plat_now_ms(); uint64_t fail0 = (uint64_t)n_failures; uint8_t *ab = NULL, (*iid)[32] = NULL; + // What this list is prepared from goes into its head (prp_file.h): the seed as it is on the disk now, and the + // translator before the host options are mixed in, which is what nx2navi_xlate.id holds. The list there before + // is replaced whole at the end; the log says whether it came from this seed. + paths(); + PrpFrom from = {{0}}, was; + int seeded = file_sha256(prg, from.programs) && file_sha256(ans, from.answers), had = 0; + memcpy(from.translator, xlate_base, 32); + size_t n_was = 0; + free(gfx_shader_prepared_read(out, &n_was, &was, &had)); + if (seeded && n_was) { + int same_seed = had && !memcmp(was.programs, from.programs, 32) && !memcmp(was.answers, from.answers, 32); + int same_xl = had && !memcmp(was.translator, from.translator, 32); + rt_log("gfx_shader: prepare: %s holds %zu variants %s; it is replaced by this preparation", out, n_was, + !had ? "and does not say what they were prepared from" + : !same_seed ? "prepared from another seed" + : !same_xl ? "prepared from this seed by another translator" : "prepared from this seed and translator"); + } // For the whole run: the variants built into the driver's cache after it (vk_pipeline.c lb_warm_prepared) are the // current translator's, as before carrying existed. plat_inc32(&C.suspended); @@ -2852,18 +2900,21 @@ int gfx_shader_prepare_seed(const char *dir, int threads, int procs) { by[r[1]]++, stages[r[0]]++, u++; } free(seen.v); - RecHeader h = {{'N', 'X', '2', 'N', 'S', 'P', 'R', 'P'}, PRP_VERSION, PRP_REC}; - uint8_t *file = recs ? malloc(sizeof h + u * PRP_REC + 1) : NULL; + uint8_t *file = recs ? malloc(PRP_HEAD + PRP_FROM_SIZE + u * PRP_REC + 1) : NULL; int ok = file != NULL; if (ok) { - memcpy(file, &h, sizeof h), memcpy(file + sizeof h, recs, u * PRP_REC); - ok = plat_file_write_atomic(out, file, sizeof h + u * PRP_REC); + size_t head = prp_head_write(file, &from); + memcpy(file + head, recs, u * PRP_REC); + ok = plat_file_write_atomic(out, file, head + u * PRP_REC); } - char msg[640]; + char msg[768]; snprintf(msg, sizeof msg, "gfx_shader: prepared %zu shader variants (vertex %zu, fragment %zu, compute %zu; title %zu, " - "warm-up %zu, other %zu) from %lld seed records, %llu translations failed, in %.0f ms; %s %s", u, + "warm-up %zu, other %zu) from %lld seed records (seed programs %02x%02x%02x%02x, answers %02x%02x%02x%02x, " + "translator %02x%02x%02x%02x), %llu translations failed, in %.0f ms; %s %s", u, stages[GFX_STAGE_VS], stages[GFX_STAGE_FS], stages[GFX_STAGE_CS], by[0], by[1], by[2], n, - (unsigned long long)((uint64_t)n_failures - fail0), (double)plat_now_ms() - t0, + from.programs[0], from.programs[1], from.programs[2], from.programs[3], from.answers[0], from.answers[1], + from.answers[2], from.answers[3], from.translator[0], from.translator[1], from.translator[2], + from.translator[3], (unsigned long long)((uint64_t)n_failures - fail0), (double)plat_now_ms() - t0, ok ? "wrote" : "cannot write", out); rt_log("%s", msg); printf("%s\n", msg); @@ -2884,7 +2935,17 @@ size_t gfx_shader_warm_list(GfxWarmEnt **out) { plat_mutex_lock(&rec_io); // no append meanwhile rec_read(RC.ans_path, "NX2NSANS", ANS_VERSION, ANS_REC, &ab, &an); plat_mutex_unlock(&rec_io); - rec_read(prp, "NX2NSPRP", PRP_VERSION, PRP_REC, &pb, &pn); + PrpFrom from; + int has_from = 0; + pb = gfx_shader_prepared_read(prp, &pn, &from, &has_from), pn *= PRP_REC; + if (pb && has_from) + rt_log("gfx_shader: %s was prepared from seed programs %02x%02x%02x%02x, answers %02x%02x%02x%02x, by translator " + "%02x%02x%02x%02x%s", prp, from.programs[0], from.programs[1], from.programs[2], from.programs[3], + from.answers[0], from.answers[1], from.answers[2], from.answers[3], from.translator[0], from.translator[1], + from.translator[2], from.translator[3], + memcmp(from.translator, xlate_base, 32) ? " (not this one: its translations are carried over)" : ""); + else if (pb) + rt_log("gfx_shader: %s does not say what it was prepared from (written before the list recorded it)", prp); size_t na = an / ANS_REC, np = pn / PRP_REC, n = 0; GfxWarmEnt *e = calloc(na + np + 1, sizeof *e); Set64 seen = {0}; diff --git a/runtime/prp_file.h b/runtime/prp_file.h new file mode 100644 index 00000000..7d2ca525 --- /dev/null +++ b/runtime/prp_file.h @@ -0,0 +1,66 @@ +// The layout of shader-prepared.bin, the list of shader variants a preparation translated and built ahead of play +// (runtime/gfx_shader.c gfx_shader_prepare_seed writes it; gfx_shader_warm_list and runtime/vk_pipeline.c lw_start, +// lb_warm_prepared read it; tools/make_game.py prepared_from reads its head). +// +// "NX2NSPRP", version (u32), record size (u32, 36) +// version 2 only: what the list was prepared from, 96 bytes: the sha256 of seed-programs.bin, the sha256 of +// seed-answers.bin (each over the whole file), and the translator identity (the 32 bytes of nx2navi_xlate.id, +// before the host options are mixed in) +// records: stage, priority, 2 zero bytes, input id (32 bytes); in priority order, one per input id +// +// A version 1 file has no such block: it does not say what it was prepared from. The seed decides which variants are in +// the list, and nothing else recorded it: a build that came with another seed (more shaders listed, another pipeline +// list) kept the old list, and the variants only the new seed names were translated when play first drew them. With +// the block the build compares it with the seed it has just written and prepares again when they differ +// (tools/make_game.py step_shaders). The translator identity is recorded for the same comparison; a list prepared by +// another translator is not discarded for that alone, since input ids do not depend on the translator and the game +// carries the translations over (gfx_shader.c, "Carry"). +// tests/prp_file_check.py checks this header against the reader in tools/make_game.py. +#pragma once +#include +#include +#include + +#define PRP_MAGIC "NX2NSPRP" +#define PRP_VERSION 2 +#define PRP_REC 36 +#define PRP_HEAD 16 // magic, version, record size +#define PRP_FROM_SIZE 96 + +typedef struct { + uint8_t programs[32]; // sha256 of seed-programs.bin + uint8_t answers[32]; // sha256 of seed-answers.bin + uint8_t translator[32]; // the translator identity +} PrpFrom; + +// The head of a version 2 file into out (PRP_HEAD + PRP_FROM_SIZE bytes); returns that size. +static inline size_t prp_head_write(uint8_t *out, const PrpFrom *from) { + uint32_t v = PRP_VERSION, r = PRP_REC; + memcpy(out, PRP_MAGIC, 8), memcpy(out + 8, &v, 4), memcpy(out + 12, &r, 4); + memcpy(out + PRP_HEAD, from->programs, 32), memcpy(out + PRP_HEAD + 32, from->answers, 32); + memcpy(out + PRP_HEAD + 64, from->translator, 32); + return PRP_HEAD + PRP_FROM_SIZE; +} + +// The records of the file f of size bytes: a pointer into f and their count in *n; NULL for a file that is no prepared +// list or has no whole record. *has_from: 1 with *from filled for a version 2 file, 0 for a version 1 file. +static inline const uint8_t *prp_parse(const uint8_t *f, uint64_t size, size_t *n, PrpFrom *from, int *has_from) { + uint32_t v = 0, r = 0; + *n = 0, *has_from = 0; + if (!f || size < PRP_HEAD || memcmp(f, PRP_MAGIC, 8)) return NULL; + memcpy(&v, f + 8, 4), memcpy(&r, f + 12, 4); + if (r != PRP_REC || (v != 1 && v != 2)) return NULL; + uint64_t at = PRP_HEAD; + if (v == 2) { + if (size < PRP_HEAD + PRP_FROM_SIZE) return NULL; + memcpy(from->programs, f + PRP_HEAD, 32), memcpy(from->answers, f + PRP_HEAD + 32, 32); + memcpy(from->translator, f + PRP_HEAD + 64, 32); + *has_from = 1, at += PRP_FROM_SIZE; + } + *n = (size_t)((size - at) / PRP_REC); + return *n ? f + at : NULL; +} + +// runtime/gfx_shader.c: the records of the prepared list at path (PRP_REC bytes each, malloc'd; their count in *n), +// NULL without any. *from, *has_from (both optional): what a version 2 file says it was prepared from. +uint8_t *gfx_shader_prepared_read(const char *path, size_t *n, PrpFrom *from, int *has_from); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index b6741050..fa348c0e 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -47,6 +47,7 @@ #include "host.h" #include "nvn_hmap.h" #include "platform/platform.h" +#include "prp_file.h" #include "tex.h" #include "vk_cache_file.h" #include "vk_gpa.h" @@ -5637,12 +5638,11 @@ static void lw_start(void) { if (!cache_dir(dir, sizeof dir)) return; snprintf(path, sizeof path, "%s/shader-prepared.bin", dir); snprintf(LW.stamp, sizeof LW.stamp, "%s/shader-warm.txt", dir); - uint64_t size = 0; - uint8_t *f = plat_file_read(path, &size); - size_t n = f && size >= 16 ? (size_t)(size - 16) / 36 : 0; + size_t n = 0; + uint8_t *f = gfx_shader_prepared_read(path, &n, NULL, NULL); // the records alone, whichever version the file is LW.e = n ? calloc(n, sizeof *LW.e) : NULL; for (size_t i = 0; LW.e && i < n; i++) - memcpy(LW.e[i].iid, f + 16 + 36 * i + 4, 32), LW.e[i].stage = f[16 + 36 * i], LW.e[i].prio = f[16 + 36 * i + 1]; + memcpy(LW.e[i].iid, f + PRP_REC * i + 4, 32), LW.e[i].stage = f[PRP_REC * i], LW.e[i].prio = f[PRP_REC * i + 1]; free(f); if (!LW.e) return; // nothing prepared (a build without the preparation step) LW.n = n; @@ -5776,12 +5776,11 @@ static void lb_warm_worker(void *arg) { static void lb_warm_prepared(const char *dir) { char path[PLAT_PATH_MAX * 2 + 32]; snprintf(path, sizeof path, "%s/shader-prepared.bin", dir); - uint64_t size = 0; - uint8_t *f = plat_file_read(path, &size); - size_t n = f && size >= 16 ? (size_t)(size - 16) / 36 : 0; + size_t n = 0; + uint8_t *f = gfx_shader_prepared_read(path, &n, NULL, NULL); // the records alone, whichever version the file is LB.e = n ? calloc(n, sizeof *LB.e) : NULL; for (size_t i = 0; LB.e && i < n; i++) - memcpy(LB.e[i].iid, f + 16 + 36 * i + 4, 32), LB.e[i].stage = f[16 + 36 * i], LB.e[i].prio = f[16 + 36 * i + 1]; + memcpy(LB.e[i].iid, f + PRP_REC * i + 4, 32), LB.e[i].stage = f[PRP_REC * i], LB.e[i].prio = f[PRP_REC * i + 1]; free(f); if (!LB.e) return; skip_init(dir); diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 47aaefea..69e86d4f 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -1438,6 +1438,11 @@ print(json.dumps({'status': 'pass', 'folders': []})) ''' +def prepared_list(programs, answers, translator, records=1): + """A version 2 shader-prepared.bin (runtime/prp_file.h): the head, what it was prepared from, then records.""" + return b'NX2NSPRP' + struct.pack(' +#include +#include + +#include "prp_file.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +int main(int argc, char **argv) { + if (argc < 2) return printf("usage: prp_file_check \n"), 2; + uint8_t file[PRP_HEAD + PRP_FROM_SIZE + 3 * PRP_REC + 8]; + PrpFrom from, got; + for (int i = 0; i < 32; i++) from.programs[i] = (uint8_t)i, from.answers[i] = (uint8_t)(0x40 + i), from.translator[i] = (uint8_t)(0x80 + i); + size_t n = 99; + int has = 9; + + // version 2: the head, then three records + size_t head = prp_head_write(file, &from); + CHECK(head == PRP_HEAD + PRP_FROM_SIZE && head == 112, "the head is %zu bytes", head); + for (int r = 0; r < 3; r++) { + uint8_t *rec = file + head + r * PRP_REC; + memset(rec, 0, PRP_REC); + rec[0] = (uint8_t)(r + 1), rec[1] = (uint8_t)r, memset(rec + 4, 0xa0 + r, 32); + } + const uint8_t *recs = prp_parse(file, head + 3 * PRP_REC, &n, &got, &has); + CHECK(recs == file + head && n == 3 && has == 1, "version 2: %zu records, has_from %d", n, has); + CHECK(!memcmp(&got, &from, sizeof from), "version 2: what it was prepared from comes back"); + CHECK(recs && recs[PRP_REC * 2] == 3 && recs[PRP_REC * 2 + 4] == 0xa2, "version 2: the third record"); + // a cut last record is not counted + recs = prp_parse(file, head + 2 * PRP_REC + 35, &n, &got, &has); + CHECK(recs && n == 2, "a cut record: %zu", n); + // the head alone, or less: no list + CHECK(!prp_parse(file, head, &n, &got, &has) && n == 0, "no records"); + CHECK(!prp_parse(file, head - 1, &n, &got, &has) && n == 0, "a cut head"); + CHECK(!prp_parse(NULL, 0, &n, &got, &has), "no file"); + + // version 1: records right after the 16 bytes, and nothing about where they came from + uint8_t v1[PRP_HEAD + 2 * PRP_REC]; + uint32_t one = 1, rec = PRP_REC; + memcpy(v1, PRP_MAGIC, 8), memcpy(v1 + 8, &one, 4), memcpy(v1 + 12, &rec, 4); + memset(v1 + PRP_HEAD, 0x11, 2 * PRP_REC); + recs = prp_parse(v1, sizeof v1, &n, &got, &has); + CHECK(recs == v1 + PRP_HEAD && n == 2 && has == 0, "version 1: %zu records, has_from %d", n, has); + + // another version, record size or magic: not read + uint32_t three = 3, other = 40; + memcpy(v1 + 8, &three, 4); + CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "version 3"); + memcpy(v1 + 8, &one, 4), memcpy(v1 + 12, &other, 4); + CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "another record size"); + memcpy(v1 + 12, &rec, 4), v1[0] = 'X'; + CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "another magic"); + + char path[1024]; + snprintf(path, sizeof path, "%s/shader-prepared.bin", argv[1]); + FILE *f = fopen(path, "wb"); + CHECK(f && fwrite(file, 1, head + 3 * PRP_REC, f) == head + 3 * PRP_REC, "cannot write %s", path); + if (f) fclose(f); + + if (!fails) printf("prepared list file: all C checks passed\n"); + return fails != 0; +} diff --git a/tests/prp_file_check.py b/tests/prp_file_check.py new file mode 100644 index 00000000..53e915d4 --- /dev/null +++ b/tests/prp_file_check.py @@ -0,0 +1,53 @@ +"""Build and run tests/prp_file_check.c: runtime/prp_file.h, the layout of shader-prepared.bin with what the list was +prepared from. The list the C side writes is then read by tools/make_game.py prepared_from, so the game's writer and +the build's reader cannot drift apart. No game files, no GPU. + +Usage: prp_file_check.py +""" +import importlib.util +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'prp_file' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('prp_file_check.exe' if os.name == 'nt' else 'prp_file_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'prp_file_check.c'), '-o', str(exe)], env=env) + rc = subprocess.run([str(exe), str(work)], env=env).returncode + if rc: + return rc + sys.path.insert(0, str(ROOT / 'tools')) + spec = importlib.util.spec_from_file_location('make_game', ROOT / 'tools' / 'make_game.py') + mg = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mg) + want = {'programs': bytes(range(32)).hex(), 'answers': bytes(range(0x40, 0x60)).hex(), + 'translator': bytes(range(0x80, 0xa0)).hex()} + got = mg.prepared_from(work) + if got != want: + print(f'FAIL: tools/make_game.py prepared_from reads {got}, the game wrote {want}') + return 1 + print('prepared list file: tools/make_game.py reads what the game writes') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/make_game.py b/tools/make_game.py index 8b121eeb..f5ec3d77 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1830,6 +1830,47 @@ def shader_carry(out): return (out / 'shader-prepared.bin').is_file() and bool(cache_identities(out)) +# The head of shader-prepared.bin (runtime/prp_file.h): magic, version, record size, and from version 2 on what the +# list was prepared from: sha256 of seed-programs.bin, sha256 of seed-answers.bin, the translator identity. +PREPARED_HEAD = struct.Struct('<8sII') +SEED_FILES = ('seed-programs.bin', 'seed-answers.bin') + + +def prepared_from(out): + """What the game folder's prepared shader list says it was prepared from: {'programs', 'answers', 'translator'} + as hex digests; {} for a list that does not say (written before the list recorded it); None without a list.""" + try: + with open(out / 'shader-prepared.bin', 'rb') as f: + data = f.read(PREPARED_HEAD.size + 96) + except OSError: + return None + if len(data) < PREPARED_HEAD.size: + return {} + magic, version, _rec = PREPARED_HEAD.unpack_from(data) + if magic != b'NX2NSPRP' or version != 2 or len(data) < PREPARED_HEAD.size + 96: + return {} + body = data[PREPARED_HEAD.size:] + return {'programs': body[:32].hex(), 'answers': body[32:64].hex(), 'translator': body[64:96].hex()} + + +def seed_changed(out): + """Why the shaders prepared before cannot stay in use, given the seed just written into the game folder + (SEED_FILES): a sentence, or '' when the list was prepared from this very seed. A list that does not say what it + was prepared from counts as changed: nothing tells whether it holds what this seed names. + The translator is not compared: input ids do not depend on it and the game carries the translations of the one + before over (shader_carry).""" + was = prepared_from(out) + if not was: + return 'the shaders prepared before do not say which shader seed they were prepared from' + try: + now = {'programs': sha256_file(out / SEED_FILES[0]), 'answers': sha256_file(out / SEED_FILES[1])} + except OSError: + return '' # no seed to compare with: as before the list recorded it + if (was['programs'], was['answers']) != (now['programs'], now['answers']): + return 'the shader seed changed since the shaders were prepared' + return '' + + def shader_run(cmd, cwd=None, build_only=False): """Runs a shader preparation command and returns its exit code. Its output is read as it comes: the counts nx2navi.exe prints become progress events, the other lines go on to the log, and while nothing moves an event @@ -1921,7 +1962,8 @@ def step_shaders(out, update=False): (shader_run reports its progress). update: after a kit update the shaders prepared before stay in use and the game translates them again in the background (shader_carry), so the update does not wait the minutes a full preparation takes; the driver's cache - gets their stage libraries again (warm_driver_cache).""" + gets their stage libraries again (warm_driver_cache). Only a list prepared from the seed this kit writes stays + (seed_changed): with another seed, or a list that does not say, everything is prepared again as at the install.""" # The game translates with the copy beside it. The compile step refreshes that copy, but a kit that changes only # the translator runs this step again alone (step_key), which would then prepare with the old one. Only a kit that # has one: the compile step, which always runs first, stops a build without it. @@ -1931,7 +1973,22 @@ def step_shaders(out, update=False): for name in (XLATE_NAME, 'nx2navi_xlate.id'): copy_if_changed(d / name, out / name) break - if update and shader_carry(out): + cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), + '--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME] + why = '' + carry = update and shader_carry(out) + if carry and prepared_from(out) is not None: + # The seed this kit makes from the game files, written without preparing anything (seconds): the list + # prepared before stays in use only when it was prepared from this seed. Another seed names other shader + # variants (a newer pipeline list, a change to which variants are prepared), and those the old list lacks + # would be translated when play first draws them. A seed that cannot be written leaves things as they were. + if shader_run(cmd + ['--no-run']) == 0: + why = seed_changed(out) + if why: + say(f' {why}: preparing them again') + if carry and not why: + for seed in SEED_FILES: + (out / seed).unlink(missing_ok=True) say(' the shaders prepared before this update stay in use; the game translates them again in the background') if loose_shaders(out) >= PACK_LOOSE_MIN: # minutes without a count (4 on a player's 328,000 files): the row says what runs meanwhile @@ -1939,14 +1996,12 @@ def step_shaders(out, update=False): pack_shader_cache(out) warm_driver_cache(out) return - cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), - '--exefs', str(out / 'exefs'), '--no-list', '--exe', EXE_NAME] rc = shader_run(cmd) if rc: fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it ' 'meets them. Run the build again to retry this step') # translated now: the shader cache holds them. seed-pipeline-keys.bin stays: --warm-shaders builds from it again - for seed in ('seed-programs.bin', 'seed-answers.bin'): + for seed in SEED_FILES: (out / seed).unlink(missing_ok=True) pack_shader_cache(out)