Merge the stutter fixes: a pipeline cache past 2 GB is read, the prepared shader list knows its seed, the pipeline cache is written only when it changed with a small file for what play adds, and a queued build a draw waits for goes first.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 04:59:05 -05:00
17 changed files with 982 additions and 140 deletions

View File

@@ -1438,6 +1438,11 @@ print(json.dumps({'status': 'pass', 'folders': []}))
'''
def prepared_list(programs, answers, translator, records=1):
"""A version 2 shader-prepared.bin (runtime/prp_file.h): the head, what it was prepared from, then records."""
return b'NX2NSPRP' + struct.pack('<II', 2, 36) + programs + answers + translator + bytes(36 * records)
class MakeGameTest(Tmp):
def setUp(self):
super().setUp()
@@ -1750,18 +1755,88 @@ class MakeGameTest(Tmp):
with contextlib.redirect_stdout(io.StringIO()):
self.mg.step_shaders(self.out, update=True)
self.assertEqual(calls, []) # never prepared: nothing to build again
(self.out / 'shader-prepared.bin').write_bytes(bytes(16))
# a list that says what it was prepared from; the stand-in writes no seed, so there is none to differ from
(self.out / 'shader-prepared.bin').write_bytes(prepared_list(b'p' * 32, b'a' * 32, b't' * 32))
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
self.mg.step_shaders(self.out, update=True)
self.assertEqual(calls, [([str(self.out / self.mg.EXE_NAME), '--warm-shaders'], str(self.out))])
self.assertEqual(calls[0][0][-1], '--no-run') # the seed is written first, to compare the list with
self.assertEqual(calls[1:], [([str(self.out / self.mg.EXE_NAME), '--warm-shaders'], str(self.out))])
self.assertIn("driver's cache warm", buf.getvalue())
self.rc = 3
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
self.mg.step_shaders(self.out, update=True) # no exception: the game warms it itself
self.assertIn('did not finish (3)', buf.getvalue())
self.assertEqual(len(calls), 2)
self.assertEqual(len(calls), 4)
def test_prepared_list_says_what_it_was_prepared_from(self):
"""shader-prepared.bin carries the digests of the two seed files and the translator from version 2 on
(runtime/prp_file.h); a version 1 list, or a file cut short, says nothing, and no file is None."""
self.out.mkdir(parents=True, exist_ok=True)
self.assertIsNone(self.mg.prepared_from(self.out))
lst = self.out / 'shader-prepared.bin'
lst.write_bytes(b'NX2NSPRP' + struct.pack('<II', 1, 36) + bytes(36 * 3))
self.assertEqual(self.mg.prepared_from(self.out), {})
lst.write_bytes(prepared_list(b'p' * 32, b'a' * 32, b't' * 32)[:60])
self.assertEqual(self.mg.prepared_from(self.out), {})
lst.write_bytes(prepared_list(b'p' * 32, b'a' * 32, b't' * 32, records=2))
self.assertEqual(self.mg.prepared_from(self.out),
{'programs': '70' * 32, 'answers': '61' * 32, 'translator': '74' * 32})
def test_update_prepares_again_when_the_seed_changed(self):
"""An update keeps the shaders prepared before only when their list was prepared from the seed this kit
writes: the seed is written first (--no-run) and compared with the list's head. Another seed, or a list that
does not say what it came from, runs the whole preparation; the same seed with another translator carries
over as before, and the seed files do not stay in the game folder either way."""
calls = []
def run(cmd, cwd=None, build_only=False):
calls.append(cmd[-1])
if cmd[-1] == '--no-run': # the kit's seed, as tools/prepare_shaders.py writes it
(self.out / 'seed-programs.bin').write_bytes(self.programs)
(self.out / 'seed-answers.bin').write_bytes(self.answers)
return 0
for name, value in (('shader_carry', lambda out: True), ('shader_run', run),
('pack_shader_cache', lambda out: None)):
self.addCleanup(setattr, self.mg, name, getattr(self.mg, name))
setattr(self.mg, name, value)
self.out.mkdir(parents=True, exist_ok=True)
(self.out / self.mg.EXE_NAME).write_bytes(b'MZ')
self.programs, self.answers = b'NX2NSPRG programs 1', b'NX2NSANS answers 1'
digest = lambda b: hashlib.sha256(b).digest() # noqa: E731
def update():
calls.clear()
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
self.mg.step_shaders(self.out, update=True)
self.assertFalse((self.out / 'seed-programs.bin').exists() or (self.out / 'seed-answers.bin').exists())
return buf.getvalue()
lst = self.out / 'shader-prepared.bin'
# prepared from this seed: kept, and the driver's cache is warmed
lst.write_bytes(prepared_list(digest(self.programs), digest(self.answers), b't' * 32))
self.assertIn('stay in use', update())
self.assertEqual(calls, ['--no-run', '--warm-shaders'])
# the same seed, another translator: still kept, the game carries the translations over
lst.write_bytes(prepared_list(digest(self.programs), digest(self.answers), b'u' * 32))
self.assertIn('stay in use', update())
# one byte of the seed's programs differs: everything is prepared again
self.programs = b'NX2NSPRG programs 2'
text = update()
self.assertIn('the shader seed changed since the shaders were prepared: preparing them again', text)
self.assertNotIn('stay in use', text)
self.assertEqual(calls, ['--no-run', self.mg.EXE_NAME]) # the seed, then the preparation (its last argument)
# so does a change to the answers alone
lst.write_bytes(prepared_list(digest(self.programs), digest(b'other answers'), b't' * 32))
self.assertIn('preparing them again', update())
# a list from before it recorded its seed
lst.write_bytes(b'NX2NSPRP' + struct.pack('<II', 1, 36) + bytes(36))
self.assertIn('do not say which shader seed', update())
self.assertEqual(calls, ['--no-run', self.mg.EXE_NAME])
# a preparation that then records this seed is kept by the next update
lst.write_bytes(prepared_list(digest(self.programs), digest(self.answers), b't' * 32))
self.assertIn('stay in use', update())
def test_romfs_failure_stops_the_build(self):
ran, lifting, failed = [], threading.Event(), threading.Event()

View File

@@ -0,0 +1,114 @@
// runtime/vk_cache_file.h: the read of pipeline-cache.bin (runtime/vk_pipeline.c cache_load) with 64-bit sizes. Makes a
// sparse file just over 2 GB in the folder given as argv[1], with a marker at its start and at its end, and checks that
// it is measured and read whole, that a size limit refuses it by its real size without reading it, and the small cases.
// Built and run by tests/pipeline_cache_file_check.py; prints each failed check and exits 1 on any.
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "vk_cache_file.h"
#ifdef _WIN32
#include <windows.h>
#include <winioctl.h>
#else
#include <fcntl.h>
#include <unistd.h>
#endif
static int fails;
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
#define HEADER 32 // the size of the header every pipeline cache starts with (VkPipelineCacheHeaderVersionOne)
// A file of len bytes: head at its start, tail as its last bytes, a hole between them that takes no disk space.
static int make_sparse(const char *path, uint64_t len, const char *head, const char *tail) {
size_t hn = strlen(head), tn = strlen(tail);
#ifdef _WIN32
HANDLE h = CreateFileA(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (h == INVALID_HANDLE_VALUE) return 0;
DWORD got = 0;
LARGE_INTEGER at = {.QuadPart = (LONGLONG)(len - tn)};
int ok = DeviceIoControl(h, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &got, NULL) &&
WriteFile(h, head, (DWORD)hn, &got, NULL) && got == hn && SetFilePointerEx(h, at, NULL, FILE_BEGIN) &&
WriteFile(h, tail, (DWORD)tn, &got, NULL) && got == tn;
CloseHandle(h);
return ok;
#else
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
if (fd < 0) return 0;
int ok = write(fd, head, hn) == (ssize_t)hn && pwrite(fd, tail, tn, (off_t)(len - tn)) == (ssize_t)tn;
close(fd);
return ok;
#endif
}
int main(int argc, char **argv) {
if (argc < 2) return printf("usage: pipeline_cache_file_check <folder>\n"), 2;
char path[1024];
uint64_t len = 0;
int why = -1;
// no file: a first run, which is not a refusal
snprintf(path, sizeof path, "%s/none.bin", argv[1]);
remove(path);
CHECK(!cache_file_read(path, HEADER, 0, &len, &why) && why == CACHE_FILE_NONE && len == 0, "no file: why %d", why);
// a file of a header alone holds no pipeline: refused with its size
snprintf(path, sizeof path, "%s/short.bin", argv[1]);
FILE *f = fopen(path, "wb");
for (int i = 0; f && i < HEADER; i++) fputc(i, f);
if (f) fclose(f);
CHECK(!cache_file_read(path, HEADER, 0, &len, &why) && why == CACHE_FILE_SHORT && len == HEADER,
"header only: why %d, len %llu", why, (unsigned long long)len);
remove(path);
// a small file: read whole
f = fopen(path, "wb");
for (int i = 0; f && i < 4096; i++) fputc(i & 0xff, f);
if (f) fclose(f);
uint8_t *d = cache_file_read(path, HEADER, 1 << 20, &len, &why);
CHECK(d && why == CACHE_FILE_OK && len == 4096 && d[4095] == 0xff && d[257] == 1, "small file: why %d, len %llu", why,
(unsigned long long)len);
free(d);
remove(path);
// just over 2 GB: a 32-bit tell measures it as negative
const uint64_t big = (2ull << 30) + 4096;
snprintf(path, sizeof path, "%s/big.bin", argv[1]);
if (!make_sparse(path, big, "HEAD-OF-THE-CACHE", "TAIL-OF-THE-CACHE")) {
printf("FAIL: cannot make the sparse file %s\n", path);
remove(path);
return 1;
}
f = fopen(path, "rb");
fseek(f, 0, SEEK_END);
long l32 = ftell(f);
fclose(f);
printf("pipeline cache file: %llu bytes; ftell into a long (%d bits here) gives %ld\n", (unsigned long long)big,
(int)sizeof(long) * 8, l32);
// under a limit: refused before any read, and its real size comes back for the log line
d = cache_file_read(path, HEADER, 512ull << 20, &len, &why);
CHECK(!d && why == CACHE_FILE_OVER_CAP && len == big, "over the limit: why %d, len %llu", why, (unsigned long long)len);
CHECK(strlen(cache_file_why(why)) > 0, "a refusal has words for the log");
free(d);
// with no limit, and with a limit above its size: read whole
for (int pass = 0; pass < 2; pass++) {
d = cache_file_read(path, HEADER, pass ? 4096ull << 20 : 0, &len, &why);
CHECK(d && why == CACHE_FILE_OK && len == big, "pass %d: over 2 GB: why %d (%s), len %llu", pass, why,
cache_file_why(why), (unsigned long long)len);
if (d) {
CHECK(!memcmp(d, "HEAD-OF-THE-CACHE", 17), "pass %d: the start of the file is the start of the block", pass);
CHECK(!memcmp(d + big - 17, "TAIL-OF-THE-CACHE", 17), "pass %d: the last bytes are past 2 GB in the block", pass);
CHECK(d[1ull << 31] == 0 && d[(1ull << 30) + 5] == 0, "pass %d: the hole reads as zeros", pass);
}
free(d);
}
remove(path);
if (!fails) printf("pipeline cache file: all checks passed\n");
return fails != 0;
}

View File

@@ -0,0 +1,38 @@
"""Build and run tests/pipeline_cache_file_check.c: runtime/vk_cache_file.h, the read of pipeline-cache.bin with 64-bit
sizes. The test makes a sparse file just over 2 GB in build/test/pipeline_cache_file (it takes no disk space, and about
2 GB of memory for the seconds the read lasts) and removes it.
Usage: pipeline_cache_file_check.py
"""
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
def compiler():
if os.environ.get('CC'):
return os.environ['CC']
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
def main():
work = ROOT / 'build' / 'test' / 'pipeline_cache_file'
work.mkdir(parents=True, exist_ok=True)
exe = work / ('pipeline_cache_file_check.exe' if os.name == 'nt' else 'pipeline_cache_file_check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
str(ROOT / 'tests' / 'pipeline_cache_file_check.c'), '-o', str(exe)], env=env)
return subprocess.run([str(exe), str(work)], env=env, timeout=600).returncode
if __name__ == '__main__':
sys.exit(main())

66
tests/prp_file_check.c Normal file
View File

@@ -0,0 +1,66 @@
// runtime/prp_file.h: the layout of shader-prepared.bin. Checks the parse of a version 1 list, of a version 2 list and
// of damaged ones, then writes a version 2 list into the folder given as argv[1] for tests/prp_file_check.py, which
// reads its head with tools/make_game.py prepared_from. Prints each failed check and exits 1 on any.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "prp_file.h"
static int fails;
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
int main(int argc, char **argv) {
if (argc < 2) return printf("usage: prp_file_check <folder>\n"), 2;
uint8_t file[PRP_HEAD + PRP_FROM_SIZE + 3 * PRP_REC + 8];
PrpFrom from, got;
for (int i = 0; i < 32; i++) from.programs[i] = (uint8_t)i, from.answers[i] = (uint8_t)(0x40 + i), from.translator[i] = (uint8_t)(0x80 + i);
size_t n = 99;
int has = 9;
// version 2: the head, then three records
size_t head = prp_head_write(file, &from);
CHECK(head == PRP_HEAD + PRP_FROM_SIZE && head == 112, "the head is %zu bytes", head);
for (int r = 0; r < 3; r++) {
uint8_t *rec = file + head + r * PRP_REC;
memset(rec, 0, PRP_REC);
rec[0] = (uint8_t)(r + 1), rec[1] = (uint8_t)r, memset(rec + 4, 0xa0 + r, 32);
}
const uint8_t *recs = prp_parse(file, head + 3 * PRP_REC, &n, &got, &has);
CHECK(recs == file + head && n == 3 && has == 1, "version 2: %zu records, has_from %d", n, has);
CHECK(!memcmp(&got, &from, sizeof from), "version 2: what it was prepared from comes back");
CHECK(recs && recs[PRP_REC * 2] == 3 && recs[PRP_REC * 2 + 4] == 0xa2, "version 2: the third record");
// a cut last record is not counted
recs = prp_parse(file, head + 2 * PRP_REC + 35, &n, &got, &has);
CHECK(recs && n == 2, "a cut record: %zu", n);
// the head alone, or less: no list
CHECK(!prp_parse(file, head, &n, &got, &has) && n == 0, "no records");
CHECK(!prp_parse(file, head - 1, &n, &got, &has) && n == 0, "a cut head");
CHECK(!prp_parse(NULL, 0, &n, &got, &has), "no file");
// version 1: records right after the 16 bytes, and nothing about where they came from
uint8_t v1[PRP_HEAD + 2 * PRP_REC];
uint32_t one = 1, rec = PRP_REC;
memcpy(v1, PRP_MAGIC, 8), memcpy(v1 + 8, &one, 4), memcpy(v1 + 12, &rec, 4);
memset(v1 + PRP_HEAD, 0x11, 2 * PRP_REC);
recs = prp_parse(v1, sizeof v1, &n, &got, &has);
CHECK(recs == v1 + PRP_HEAD && n == 2 && has == 0, "version 1: %zu records, has_from %d", n, has);
// another version, record size or magic: not read
uint32_t three = 3, other = 40;
memcpy(v1 + 8, &three, 4);
CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "version 3");
memcpy(v1 + 8, &one, 4), memcpy(v1 + 12, &other, 4);
CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "another record size");
memcpy(v1 + 12, &rec, 4), v1[0] = 'X';
CHECK(!prp_parse(v1, sizeof v1, &n, &got, &has), "another magic");
char path[1024];
snprintf(path, sizeof path, "%s/shader-prepared.bin", argv[1]);
FILE *f = fopen(path, "wb");
CHECK(f && fwrite(file, 1, head + 3 * PRP_REC, f) == head + 3 * PRP_REC, "cannot write %s", path);
if (f) fclose(f);
if (!fails) printf("prepared list file: all C checks passed\n");
return fails != 0;
}

53
tests/prp_file_check.py Normal file
View File

@@ -0,0 +1,53 @@
"""Build and run tests/prp_file_check.c: runtime/prp_file.h, the layout of shader-prepared.bin with what the list was
prepared from. The list the C side writes is then read by tools/make_game.py prepared_from, so the game's writer and
the build's reader cannot drift apart. No game files, no GPU.
Usage: prp_file_check.py
"""
import importlib.util
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
def compiler():
if os.environ.get('CC'):
return os.environ['CC']
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
def main():
work = ROOT / 'build' / 'test' / 'prp_file'
work.mkdir(parents=True, exist_ok=True)
exe = work / ('prp_file_check.exe' if os.name == 'nt' else 'prp_file_check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
str(ROOT / 'tests' / 'prp_file_check.c'), '-o', str(exe)], env=env)
rc = subprocess.run([str(exe), str(work)], env=env).returncode
if rc:
return rc
sys.path.insert(0, str(ROOT / 'tools'))
spec = importlib.util.spec_from_file_location('make_game', ROOT / 'tools' / 'make_game.py')
mg = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mg)
want = {'programs': bytes(range(32)).hex(), 'answers': bytes(range(0x40, 0x60)).hex(),
'translator': bytes(range(0x80, 0xa0)).hex()}
got = mg.prepared_from(work)
if got != want:
print(f'FAIL: tools/make_game.py prepared_from reads {got}, the game wrote {want}')
return 1
print('prepared list file: tools/make_game.py reads what the game writes')
return 0
if __name__ == '__main__':
sys.exit(main())