Record in shader-prepared.bin which seed it was prepared from, and prepare again at an update when the seed changed.

The prepared list named its shader variants and nothing about where they came from, so an update that keeps the prepared shaders (shader_carry) kept the old list whatever seed the new kit writes: variants only the new seed names, from a newer pipeline list or a change to which variants are prepared, were translated when play first drew them. The existing identities do not cover this: nx2navi_xlate.id and identities.txt name translators, which decide cache keys and what is carried over, not which variants are listed.

The list is now version 2 (runtime/prp_file.h): after the head come the sha256 of seed-programs.bin, the sha256 of seed-answers.bin and the translator identity, written by gfx_shader_prepare_seed; the three readers go through gfx_shader_prepared_read and still read a version 1 list. At an update step_shaders writes the seed first (prepare_shaders.py --no-run, about 4 s) and keeps the list only when its digests are that seed's (make_game.py seed_changed); another seed, or a list that does not say, is prepared again. Another translator alone still carries over: input ids do not depend on the translator, and preparing everything again for it is what carrying was made to avoid.

Checked with tests/prp_file_check.py (the C layout against make_game's reader), three launcher tests, and the exe's --prepare-shaders on a seed without records in a folder of its own: the digests it wrote are those of the seed files, one byte more in the seed reads as changed, and after preparing again it does not. A version 1 list of 42,142 variants and the same list as version 2 are both read whole by --probe-prewarm.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 04:36:33 -05:00
parent 871b06e2e7
commit cba95f8d0a
9 changed files with 412 additions and 28 deletions

View File

@@ -80,6 +80,7 @@
#include "gfx.h"
#include "host.h"
#include "platform/platform.h"
#include "prp_file.h"
#define CONTROL_SIZE 0x880
#define CODE_SKIP 0x30 // graphics code starts after the first 0x30 bytes of the program
@@ -2820,10 +2821,40 @@ void gfx_shader_prepare_recorded(int threads, int procs) {
}
// ---- prepared variants (shader-prepared.bin next to pipeline-keys.bin)
// "NX2NSPRP", version, record size; records: stage, priority, 2 zero bytes, input id. gfx_shader_prepare_seed writes
// The layout is prp_file.h's: a head that says what the list was prepared from (the two seed files and the
// translator), then one record per variant: stage, priority, 2 zero bytes, input id. gfx_shader_prepare_seed writes
// them in priority order, one per input id; gfx_shader_warm_list reads them at boot.
#define PRP_VERSION 1
#define PRP_REC 36
// The sha256 of the whole file; 0 when it cannot be read.
static int file_sha256(const char *path, uint8_t out[32]) {
FILE *f = fopen(path, "rb");
if (!f) return 0;
fseek(f, 0, SEEK_END);
long len = ftell(f); // a seed past 2 GB is not read by rec_read either
fseek(f, 0, SEEK_SET);
uint8_t *b = len > 0 ? malloc((size_t)len) : NULL;
int ok = b && fread(b, 1, (size_t)len, f) == (size_t)len;
fclose(f);
if (ok) rt_sha256(b, (size_t)len, out);
free(b);
return ok;
}
// The records of the prepared list at path (PRP_REC bytes each, malloc'd; their count in *n), NULL without any.
// *from, *has_from (both optional): what a version 2 file says it was prepared from.
uint8_t *gfx_shader_prepared_read(const char *path, size_t *n, PrpFrom *from, int *has_from) {
uint64_t size = 0;
uint8_t *f = plat_file_read(path, &size), *out = NULL;
PrpFrom fr = {{0}};
int has = 0;
const uint8_t *recs = prp_parse(f, f ? size : 0, n, &fr, &has);
if (recs && (out = malloc(*n * PRP_REC))) memcpy(out, recs, *n * PRP_REC);
if (!out) *n = 0;
if (from) *from = fr;
if (has_from) *has_from = has;
free(f);
return out;
}
int gfx_shader_prepare_seed(const char *dir, int threads, int procs) {
char ans[PATH_CAP + 32], prg[PATH_CAP + 32], out[PATH_CAP + 32];
@@ -2833,6 +2864,23 @@ int gfx_shader_prepare_seed(const char *dir, int threads, int procs) {
double t0 = (double)plat_now_ms();
uint64_t fail0 = (uint64_t)n_failures;
uint8_t *ab = NULL, (*iid)[32] = NULL;
// What this list is prepared from goes into its head (prp_file.h): the seed as it is on the disk now, and the
// translator before the host options are mixed in, which is what nx2navi_xlate.id holds. The list there before
// is replaced whole at the end; the log says whether it came from this seed.
paths();
PrpFrom from = {{0}}, was;
int seeded = file_sha256(prg, from.programs) && file_sha256(ans, from.answers), had = 0;
memcpy(from.translator, xlate_base, 32);
size_t n_was = 0;
free(gfx_shader_prepared_read(out, &n_was, &was, &had));
if (seeded && n_was) {
int same_seed = had && !memcmp(was.programs, from.programs, 32) && !memcmp(was.answers, from.answers, 32);
int same_xl = had && !memcmp(was.translator, from.translator, 32);
rt_log("gfx_shader: prepare: %s holds %zu variants %s; it is replaced by this preparation", out, n_was,
!had ? "and does not say what they were prepared from"
: !same_seed ? "prepared from another seed"
: !same_xl ? "prepared from this seed by another translator" : "prepared from this seed and translator");
}
// For the whole run: the variants built into the driver's cache after it (vk_pipeline.c lb_warm_prepared) are the
// current translator's, as before carrying existed.
plat_inc32(&C.suspended);
@@ -2852,18 +2900,21 @@ int gfx_shader_prepare_seed(const char *dir, int threads, int procs) {
by[r[1]]++, stages[r[0]]++, u++;
}
free(seen.v);
RecHeader h = {{'N', 'X', '2', 'N', 'S', 'P', 'R', 'P'}, PRP_VERSION, PRP_REC};
uint8_t *file = recs ? malloc(sizeof h + u * PRP_REC + 1) : NULL;
uint8_t *file = recs ? malloc(PRP_HEAD + PRP_FROM_SIZE + u * PRP_REC + 1) : NULL;
int ok = file != NULL;
if (ok) {
memcpy(file, &h, sizeof h), memcpy(file + sizeof h, recs, u * PRP_REC);
ok = plat_file_write_atomic(out, file, sizeof h + u * PRP_REC);
size_t head = prp_head_write(file, &from);
memcpy(file + head, recs, u * PRP_REC);
ok = plat_file_write_atomic(out, file, head + u * PRP_REC);
}
char msg[640];
char msg[768];
snprintf(msg, sizeof msg, "gfx_shader: prepared %zu shader variants (vertex %zu, fragment %zu, compute %zu; title %zu, "
"warm-up %zu, other %zu) from %lld seed records, %llu translations failed, in %.0f ms; %s %s", u,
"warm-up %zu, other %zu) from %lld seed records (seed programs %02x%02x%02x%02x, answers %02x%02x%02x%02x, "
"translator %02x%02x%02x%02x), %llu translations failed, in %.0f ms; %s %s", u,
stages[GFX_STAGE_VS], stages[GFX_STAGE_FS], stages[GFX_STAGE_CS], by[0], by[1], by[2], n,
(unsigned long long)((uint64_t)n_failures - fail0), (double)plat_now_ms() - t0,
from.programs[0], from.programs[1], from.programs[2], from.programs[3], from.answers[0], from.answers[1],
from.answers[2], from.answers[3], from.translator[0], from.translator[1], from.translator[2],
from.translator[3], (unsigned long long)((uint64_t)n_failures - fail0), (double)plat_now_ms() - t0,
ok ? "wrote" : "cannot write", out);
rt_log("%s", msg);
printf("%s\n", msg);
@@ -2884,7 +2935,17 @@ size_t gfx_shader_warm_list(GfxWarmEnt **out) {
plat_mutex_lock(&rec_io); // no append meanwhile
rec_read(RC.ans_path, "NX2NSANS", ANS_VERSION, ANS_REC, &ab, &an);
plat_mutex_unlock(&rec_io);
rec_read(prp, "NX2NSPRP", PRP_VERSION, PRP_REC, &pb, &pn);
PrpFrom from;
int has_from = 0;
pb = gfx_shader_prepared_read(prp, &pn, &from, &has_from), pn *= PRP_REC;
if (pb && has_from)
rt_log("gfx_shader: %s was prepared from seed programs %02x%02x%02x%02x, answers %02x%02x%02x%02x, by translator "
"%02x%02x%02x%02x%s", prp, from.programs[0], from.programs[1], from.programs[2], from.programs[3],
from.answers[0], from.answers[1], from.answers[2], from.answers[3], from.translator[0], from.translator[1],
from.translator[2], from.translator[3],
memcmp(from.translator, xlate_base, 32) ? " (not this one: its translations are carried over)" : "");
else if (pb)
rt_log("gfx_shader: %s does not say what it was prepared from (written before the list recorded it)", prp);
size_t na = an / ANS_REC, np = pn / PRP_REC, n = 0;
GfxWarmEnt *e = calloc(na + np + 1, sizeof *e);
Set64 seen = {0};

66
runtime/prp_file.h Normal file
View File

@@ -0,0 +1,66 @@
// The layout of shader-prepared.bin, the list of shader variants a preparation translated and built ahead of play
// (runtime/gfx_shader.c gfx_shader_prepare_seed writes it; gfx_shader_warm_list and runtime/vk_pipeline.c lw_start,
// lb_warm_prepared read it; tools/make_game.py prepared_from reads its head).
//
// "NX2NSPRP", version (u32), record size (u32, 36)
// version 2 only: what the list was prepared from, 96 bytes: the sha256 of seed-programs.bin, the sha256 of
// seed-answers.bin (each over the whole file), and the translator identity (the 32 bytes of nx2navi_xlate.id,
// before the host options are mixed in)
// records: stage, priority, 2 zero bytes, input id (32 bytes); in priority order, one per input id
//
// A version 1 file has no such block: it does not say what it was prepared from. The seed decides which variants are in
// the list, and nothing else recorded it: a build that came with another seed (more shaders listed, another pipeline
// list) kept the old list, and the variants only the new seed names were translated when play first drew them. With
// the block the build compares it with the seed it has just written and prepares again when they differ
// (tools/make_game.py step_shaders). The translator identity is recorded for the same comparison; a list prepared by
// another translator is not discarded for that alone, since input ids do not depend on the translator and the game
// carries the translations over (gfx_shader.c, "Carry").
// tests/prp_file_check.py checks this header against the reader in tools/make_game.py.
#pragma once
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#define PRP_MAGIC "NX2NSPRP"
#define PRP_VERSION 2
#define PRP_REC 36
#define PRP_HEAD 16 // magic, version, record size
#define PRP_FROM_SIZE 96
typedef struct {
uint8_t programs[32]; // sha256 of seed-programs.bin
uint8_t answers[32]; // sha256 of seed-answers.bin
uint8_t translator[32]; // the translator identity
} PrpFrom;
// The head of a version 2 file into out (PRP_HEAD + PRP_FROM_SIZE bytes); returns that size.
static inline size_t prp_head_write(uint8_t *out, const PrpFrom *from) {
uint32_t v = PRP_VERSION, r = PRP_REC;
memcpy(out, PRP_MAGIC, 8), memcpy(out + 8, &v, 4), memcpy(out + 12, &r, 4);
memcpy(out + PRP_HEAD, from->programs, 32), memcpy(out + PRP_HEAD + 32, from->answers, 32);
memcpy(out + PRP_HEAD + 64, from->translator, 32);
return PRP_HEAD + PRP_FROM_SIZE;
}
// The records of the file f of size bytes: a pointer into f and their count in *n; NULL for a file that is no prepared
// list or has no whole record. *has_from: 1 with *from filled for a version 2 file, 0 for a version 1 file.
static inline const uint8_t *prp_parse(const uint8_t *f, uint64_t size, size_t *n, PrpFrom *from, int *has_from) {
uint32_t v = 0, r = 0;
*n = 0, *has_from = 0;
if (!f || size < PRP_HEAD || memcmp(f, PRP_MAGIC, 8)) return NULL;
memcpy(&v, f + 8, 4), memcpy(&r, f + 12, 4);
if (r != PRP_REC || (v != 1 && v != 2)) return NULL;
uint64_t at = PRP_HEAD;
if (v == 2) {
if (size < PRP_HEAD + PRP_FROM_SIZE) return NULL;
memcpy(from->programs, f + PRP_HEAD, 32), memcpy(from->answers, f + PRP_HEAD + 32, 32);
memcpy(from->translator, f + PRP_HEAD + 64, 32);
*has_from = 1, at += PRP_FROM_SIZE;
}
*n = (size_t)((size - at) / PRP_REC);
return *n ? f + at : NULL;
}
// runtime/gfx_shader.c: the records of the prepared list at path (PRP_REC bytes each, malloc'd; their count in *n),
// NULL without any. *from, *has_from (both optional): what a version 2 file says it was prepared from.
uint8_t *gfx_shader_prepared_read(const char *path, size_t *n, PrpFrom *from, int *has_from);

View File

@@ -47,6 +47,7 @@
#include "host.h"
#include "nvn_hmap.h"
#include "platform/platform.h"
#include "prp_file.h"
#include "tex.h"
#include "vk_cache_file.h"
#include "vk_gpa.h"
@@ -5637,12 +5638,11 @@ static void lw_start(void) {
if (!cache_dir(dir, sizeof dir)) return;
snprintf(path, sizeof path, "%s/shader-prepared.bin", dir);
snprintf(LW.stamp, sizeof LW.stamp, "%s/shader-warm.txt", dir);
uint64_t size = 0;
uint8_t *f = plat_file_read(path, &size);
size_t n = f && size >= 16 ? (size_t)(size - 16) / 36 : 0;
size_t n = 0;
uint8_t *f = gfx_shader_prepared_read(path, &n, NULL, NULL); // the records alone, whichever version the file is
LW.e = n ? calloc(n, sizeof *LW.e) : NULL;
for (size_t i = 0; LW.e && i < n; i++)
memcpy(LW.e[i].iid, f + 16 + 36 * i + 4, 32), LW.e[i].stage = f[16 + 36 * i], LW.e[i].prio = f[16 + 36 * i + 1];
memcpy(LW.e[i].iid, f + PRP_REC * i + 4, 32), LW.e[i].stage = f[PRP_REC * i], LW.e[i].prio = f[PRP_REC * i + 1];
free(f);
if (!LW.e) return; // nothing prepared (a build without the preparation step)
LW.n = n;
@@ -5776,12 +5776,11 @@ static void lb_warm_worker(void *arg) {
static void lb_warm_prepared(const char *dir) {
char path[PLAT_PATH_MAX * 2 + 32];
snprintf(path, sizeof path, "%s/shader-prepared.bin", dir);
uint64_t size = 0;
uint8_t *f = plat_file_read(path, &size);
size_t n = f && size >= 16 ? (size_t)(size - 16) / 36 : 0;
size_t n = 0;
uint8_t *f = gfx_shader_prepared_read(path, &n, NULL, NULL); // the records alone, whichever version the file is
LB.e = n ? calloc(n, sizeof *LB.e) : NULL;
for (size_t i = 0; LB.e && i < n; i++)
memcpy(LB.e[i].iid, f + 16 + 36 * i + 4, 32), LB.e[i].stage = f[16 + 36 * i], LB.e[i].prio = f[16 + 36 * i + 1];
memcpy(LB.e[i].iid, f + PRP_REC * i + 4, 32), LB.e[i].stage = f[PRP_REC * i], LB.e[i].prio = f[PRP_REC * i + 1];
free(f);
if (!LB.e) return;
skip_init(dir);