mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 11:29:58 -04:00
Record in shader-prepared.bin which seed it was prepared from, and prepare again at an update when the seed changed.
The prepared list named its shader variants and nothing about where they came from, so an update that keeps the prepared shaders (shader_carry) kept the old list whatever seed the new kit writes: variants only the new seed names, from a newer pipeline list or a change to which variants are prepared, were translated when play first drew them. The existing identities do not cover this: nx2navi_xlate.id and identities.txt name translators, which decide cache keys and what is carried over, not which variants are listed. The list is now version 2 (runtime/prp_file.h): after the head come the sha256 of seed-programs.bin, the sha256 of seed-answers.bin and the translator identity, written by gfx_shader_prepare_seed; the three readers go through gfx_shader_prepared_read and still read a version 1 list. At an update step_shaders writes the seed first (prepare_shaders.py --no-run, about 4 s) and keeps the list only when its digests are that seed's (make_game.py seed_changed); another seed, or a list that does not say, is prepared again. Another translator alone still carries over: input ids do not depend on the translator, and preparing everything again for it is what carrying was made to avoid. Checked with tests/prp_file_check.py (the C layout against make_game's reader), three launcher tests, and the exe's --prepare-shaders on a seed without records in a folder of its own: the digests it wrote are those of the seed files, one byte more in the seed reads as changed, and after preparing again it does not. A version 1 list of 42,142 variants and the same list as version 2 are both read whole by --probe-prewarm. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -1438,6 +1438,11 @@ print(json.dumps({'status': 'pass', 'folders': []}))
|
||||
'''
|
||||
|
||||
|
||||
def prepared_list(programs, answers, translator, records=1):
|
||||
"""A version 2 shader-prepared.bin (runtime/prp_file.h): the head, what it was prepared from, then records."""
|
||||
return b'NX2NSPRP' + struct.pack('<II', 2, 36) + programs + answers + translator + bytes(36 * records)
|
||||
|
||||
|
||||
class MakeGameTest(Tmp):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
@@ -1750,18 +1755,88 @@ class MakeGameTest(Tmp):
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
self.mg.step_shaders(self.out, update=True)
|
||||
self.assertEqual(calls, []) # never prepared: nothing to build again
|
||||
(self.out / 'shader-prepared.bin').write_bytes(bytes(16))
|
||||
# a list that says what it was prepared from; the stand-in writes no seed, so there is none to differ from
|
||||
(self.out / 'shader-prepared.bin').write_bytes(prepared_list(b'p' * 32, b'a' * 32, b't' * 32))
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
self.mg.step_shaders(self.out, update=True)
|
||||
self.assertEqual(calls, [([str(self.out / self.mg.EXE_NAME), '--warm-shaders'], str(self.out))])
|
||||
self.assertEqual(calls[0][0][-1], '--no-run') # the seed is written first, to compare the list with
|
||||
self.assertEqual(calls[1:], [([str(self.out / self.mg.EXE_NAME), '--warm-shaders'], str(self.out))])
|
||||
self.assertIn("driver's cache warm", buf.getvalue())
|
||||
self.rc = 3
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
self.mg.step_shaders(self.out, update=True) # no exception: the game warms it itself
|
||||
self.assertIn('did not finish (3)', buf.getvalue())
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(len(calls), 4)
|
||||
|
||||
def test_prepared_list_says_what_it_was_prepared_from(self):
|
||||
"""shader-prepared.bin carries the digests of the two seed files and the translator from version 2 on
|
||||
(runtime/prp_file.h); a version 1 list, or a file cut short, says nothing, and no file is None."""
|
||||
self.out.mkdir(parents=True, exist_ok=True)
|
||||
self.assertIsNone(self.mg.prepared_from(self.out))
|
||||
lst = self.out / 'shader-prepared.bin'
|
||||
lst.write_bytes(b'NX2NSPRP' + struct.pack('<II', 1, 36) + bytes(36 * 3))
|
||||
self.assertEqual(self.mg.prepared_from(self.out), {})
|
||||
lst.write_bytes(prepared_list(b'p' * 32, b'a' * 32, b't' * 32)[:60])
|
||||
self.assertEqual(self.mg.prepared_from(self.out), {})
|
||||
lst.write_bytes(prepared_list(b'p' * 32, b'a' * 32, b't' * 32, records=2))
|
||||
self.assertEqual(self.mg.prepared_from(self.out),
|
||||
{'programs': '70' * 32, 'answers': '61' * 32, 'translator': '74' * 32})
|
||||
|
||||
def test_update_prepares_again_when_the_seed_changed(self):
|
||||
"""An update keeps the shaders prepared before only when their list was prepared from the seed this kit
|
||||
writes: the seed is written first (--no-run) and compared with the list's head. Another seed, or a list that
|
||||
does not say what it came from, runs the whole preparation; the same seed with another translator carries
|
||||
over as before, and the seed files do not stay in the game folder either way."""
|
||||
calls = []
|
||||
|
||||
def run(cmd, cwd=None, build_only=False):
|
||||
calls.append(cmd[-1])
|
||||
if cmd[-1] == '--no-run': # the kit's seed, as tools/prepare_shaders.py writes it
|
||||
(self.out / 'seed-programs.bin').write_bytes(self.programs)
|
||||
(self.out / 'seed-answers.bin').write_bytes(self.answers)
|
||||
return 0
|
||||
for name, value in (('shader_carry', lambda out: True), ('shader_run', run),
|
||||
('pack_shader_cache', lambda out: None)):
|
||||
self.addCleanup(setattr, self.mg, name, getattr(self.mg, name))
|
||||
setattr(self.mg, name, value)
|
||||
self.out.mkdir(parents=True, exist_ok=True)
|
||||
(self.out / self.mg.EXE_NAME).write_bytes(b'MZ')
|
||||
self.programs, self.answers = b'NX2NSPRG programs 1', b'NX2NSANS answers 1'
|
||||
digest = lambda b: hashlib.sha256(b).digest() # noqa: E731
|
||||
|
||||
def update():
|
||||
calls.clear()
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
self.mg.step_shaders(self.out, update=True)
|
||||
self.assertFalse((self.out / 'seed-programs.bin').exists() or (self.out / 'seed-answers.bin').exists())
|
||||
return buf.getvalue()
|
||||
lst = self.out / 'shader-prepared.bin'
|
||||
# prepared from this seed: kept, and the driver's cache is warmed
|
||||
lst.write_bytes(prepared_list(digest(self.programs), digest(self.answers), b't' * 32))
|
||||
self.assertIn('stay in use', update())
|
||||
self.assertEqual(calls, ['--no-run', '--warm-shaders'])
|
||||
# the same seed, another translator: still kept, the game carries the translations over
|
||||
lst.write_bytes(prepared_list(digest(self.programs), digest(self.answers), b'u' * 32))
|
||||
self.assertIn('stay in use', update())
|
||||
# one byte of the seed's programs differs: everything is prepared again
|
||||
self.programs = b'NX2NSPRG programs 2'
|
||||
text = update()
|
||||
self.assertIn('the shader seed changed since the shaders were prepared: preparing them again', text)
|
||||
self.assertNotIn('stay in use', text)
|
||||
self.assertEqual(calls, ['--no-run', self.mg.EXE_NAME]) # the seed, then the preparation (its last argument)
|
||||
# so does a change to the answers alone
|
||||
lst.write_bytes(prepared_list(digest(self.programs), digest(b'other answers'), b't' * 32))
|
||||
self.assertIn('preparing them again', update())
|
||||
# a list from before it recorded its seed
|
||||
lst.write_bytes(b'NX2NSPRP' + struct.pack('<II', 1, 36) + bytes(36))
|
||||
self.assertIn('do not say which shader seed', update())
|
||||
self.assertEqual(calls, ['--no-run', self.mg.EXE_NAME])
|
||||
# a preparation that then records this seed is kept by the next update
|
||||
lst.write_bytes(prepared_list(digest(self.programs), digest(self.answers), b't' * 32))
|
||||
self.assertIn('stay in use', update())
|
||||
|
||||
def test_romfs_failure_stops_the_build(self):
|
||||
ran, lifting, failed = [], threading.Event(), threading.Event()
|
||||
|
||||
66
tests/prp_file_check.c
Normal file
66
tests/prp_file_check.c
Normal file
@@ -0,0 +1,66 @@
|
||||
// runtime/prp_file.h: the layout of shader-prepared.bin. Checks the parse of a version 1 list, of a version 2 list and
|
||||
// of damaged ones, then writes a version 2 list into the folder given as argv[1] for tests/prp_file_check.py, which
|
||||
// reads its head with tools/make_game.py prepared_from. Prints each failed check and exits 1 on any.
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "prp_file.h"
|
||||
|
||||
static int fails;
|
||||
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 2) return printf("usage: prp_file_check <folder>\n"), 2;
|
||||
uint8_t file[PRP_HEAD + PRP_FROM_SIZE + 3 * PRP_REC + 8];
|
||||
PrpFrom from, got;
|
||||
for (int i = 0; i < 32; i++) from.programs[i] = (uint8_t)i, from.answers[i] = (uint8_t)(0x40 + i), from.translator[i] = (uint8_t)(0x80 + i);
|
||||
size_t n = 99;
|
||||
int has = 9;
|
||||
|
||||
// version 2: the head, then three records
|
||||
size_t head = prp_head_write(file, &from);
|
||||
CHECK(head == PRP_HEAD + PRP_FROM_SIZE && head == 112, "the head is %zu bytes", head);
|
||||
for (int r = 0; r < 3; r++) {
|
||||
uint8_t *rec = file + head + r * PRP_REC;
|
||||
memset(rec, 0, PRP_REC);
|
||||
rec[0] = (uint8_t)(r + 1), rec[1] = (uint8_t)r, memset(rec + 4, 0xa0 + r, 32);
|
||||
}
|
||||
const uint8_t *recs = prp_parse(file, head + 3 * PRP_REC, &n, &got, &has);
|
||||
CHECK(recs == file + head && n == 3 && has == 1, "version 2: %zu records, has_from %d", n, has);
|
||||
CHECK(!memcmp(&got, &from, sizeof from), "version 2: what it was prepared from comes back");
|
||||
CHECK(recs && recs[PRP_REC * 2] == 3 && recs[PRP_REC * 2 + 4] == 0xa2, "version 2: the third record");
|
||||
// a cut last record is not counted
|
||||
recs = prp_parse(file, head + 2 * PRP_REC + 35, &n, &got, &has);
|
||||
CHECK(recs && n == 2, "a cut record: %zu", n);
|
||||
// the head alone, or less: no list
|
||||
CHECK(!prp_parse(file, head, &n, &got, &has) && n == 0, "no records");
|
||||
CHECK(!prp_parse(file, head - 1, &n, &got, &has) && n == 0, "a cut head");
|
||||
CHECK(!prp_parse(NULL, 0, &n, &got, &has), "no file");
|
||||
|
||||
// version 1: records right after the 16 bytes, and nothing about where they came from
|
||||
uint8_t v1[PRP_HEAD + 2 * PRP_REC];
|
||||
uint32_t one = 1, rec = PRP_REC;
|
||||
memcpy(v1, PRP_MAGIC, 8), memcpy(v1 + 8, &one, 4), memcpy(v1 + 12, &rec, 4);
|
||||
memset(v1 + PRP_HEAD, 0x11, 2 * PRP_REC);
|
||||
recs = prp_parse(v1, sizeof v1, &n, &got, &has);
|
||||
CHECK(recs == v1 + PRP_HEAD && n == 2 && has == 0, "version 1: %zu records, has_from %d", n, has);
|
||||
|
||||
// another version, record size or magic: not read
|
||||
uint32_t three = 3, other = 40;
|
||||
memcpy(v1 + 8, &three, 4);
|
||||
CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "version 3");
|
||||
memcpy(v1 + 8, &one, 4), memcpy(v1 + 12, &other, 4);
|
||||
CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "another record size");
|
||||
memcpy(v1 + 12, &rec, 4), v1[0] = 'X';
|
||||
CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "another magic");
|
||||
|
||||
char path[1024];
|
||||
snprintf(path, sizeof path, "%s/shader-prepared.bin", argv[1]);
|
||||
FILE *f = fopen(path, "wb");
|
||||
CHECK(f && fwrite(file, 1, head + 3 * PRP_REC, f) == head + 3 * PRP_REC, "cannot write %s", path);
|
||||
if (f) fclose(f);
|
||||
|
||||
if (!fails) printf("prepared list file: all C checks passed\n");
|
||||
return fails != 0;
|
||||
}
|
||||
53
tests/prp_file_check.py
Normal file
53
tests/prp_file_check.py
Normal file
@@ -0,0 +1,53 @@
|
||||
"""Build and run tests/prp_file_check.c: runtime/prp_file.h, the layout of shader-prepared.bin with what the list was
|
||||
prepared from. The list the C side writes is then read by tools/make_game.py prepared_from, so the game's writer and
|
||||
the build's reader cannot drift apart. No game files, no GPU.
|
||||
|
||||
Usage: prp_file_check.py
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||
|
||||
|
||||
def compiler():
|
||||
if os.environ.get('CC'):
|
||||
return os.environ['CC']
|
||||
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
|
||||
return str(MSYS_BIN / 'clang.exe')
|
||||
return shutil.which('clang') or 'clang'
|
||||
|
||||
|
||||
def main():
|
||||
work = ROOT / 'build' / 'test' / 'prp_file'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
exe = work / ('prp_file_check.exe' if os.name == 'nt' else 'prp_file_check')
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
|
||||
str(ROOT / 'tests' / 'prp_file_check.c'), '-o', str(exe)], env=env)
|
||||
rc = subprocess.run([str(exe), str(work)], env=env).returncode
|
||||
if rc:
|
||||
return rc
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
spec = importlib.util.spec_from_file_location('make_game', ROOT / 'tools' / 'make_game.py')
|
||||
mg = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mg)
|
||||
want = {'programs': bytes(range(32)).hex(), 'answers': bytes(range(0x40, 0x60)).hex(),
|
||||
'translator': bytes(range(0x80, 0xa0)).hex()}
|
||||
got = mg.prepared_from(work)
|
||||
if got != want:
|
||||
print(f'FAIL: tools/make_game.py prepared_from reads {got}, the game wrote {want}')
|
||||
return 1
|
||||
print('prepared list file: tools/make_game.py reads what the game writes')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user