diff --git a/docs/building.md b/docs/building.md index 92da41be..632c3c81 100644 --- a/docs/building.md +++ b/docs/building.md @@ -643,6 +643,11 @@ A change to how `pipeline-cache.bin` is read (`runtime/vk_cache_file.h`, `runtim through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU; about 2 GB of memory for a few seconds. +A change to how `pipeline-binaries.bin` is indexed (`runtime/vk_pipeline.c` `pb_index`, `pb_init`) passes +`python tests\pb_torn_check.py`, which cuts the record layout and the index walk out of `vk_pipeline.c` and walks files +of whole records, with a last record cut short at many lengths and with a malformed record: the whole records before +it are indexed and the walk ends where they end, which is where `pb_init` cuts the file back to. No game files, no GPU. + A change to the layout of `shader-prepared.bin` (`runtime/prp_file.h`, `runtime/gfx_shader.c` `gfx_shader_prepare_seed`, `tools/make_game.py` `prepared_from`) passes `python tests\prp_file_check.py`, which builds the header alone, checks the parse of both versions and has `make_game.py` read the list the C side wrote. No game diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 10566d58..19b34197 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -315,6 +315,27 @@ static void pb_rewrite(const PbHeader *hd) { // an empty file with this driver PB.file_size = f ? sizeof *hd : 0; } +// Indexes the records of f (len bytes) after its header: per record its type, key and offset. Returns the offset +// after the last whole record, which is len unless the file ends inside a record (an append the process did not +// finish) or holds a malformed one. +static uint64_t pb_index(FILE *f, uint64_t len) { + uint64_t good = sizeof(PbHeader); + for (;;) { + uint32_t head[3]; + uint8_t key[PB_KEY]; + if (pb_seek(f, (int64_t)good, SEEK_SET) || fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || + head[0] < PB_HEAD || (head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || + good + 4 + head[0] > len) + break; + if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) { + if (head[1] == PB_PIPE) PB.pipes++; + else PB.blobs++; + } + good += 4 + (uint64_t)head[0]; + } + return good; +} + // Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache). static void pb_init(VkCtx *c) { if (!c->has_pipeline_binary || !P.cache_path[0]) return; @@ -348,25 +369,21 @@ static void pb_init(VkCtx *c) { pb_seek(f, 0, SEEK_SET); if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version"; else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB"; - else { - good = sizeof hd; - for (;;) { // index: per record its type, key and offset - uint32_t head[3]; - uint8_t key[PB_KEY]; - if (fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || head[0] < PB_HEAD || - (head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || good + 4 + head[0] > len) - break; - if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) { - if (head[1] == PB_PIPE) PB.pipes++; - else PB.blobs++; - } - good += 4 + (uint64_t)head[0]; - if (pb_seek(f, (int64_t)good, SEEK_SET)) break; - } - if (good != len) why = "cut short (a record half written)"; - } + else good = pb_index(f, len); fclose(f); } + // A file that ends inside a record (the process ended during an append, or a disk filled): the whole records + // before it stay, and the file is cut back to them so that appends go on from there. Only when it cannot be cut + // does it start empty. + if (f && !why && good != len) { + if (plat_file_truncate(PB.path, good)) { + rt_log("vk_pipeline: pipeline binaries: %s was cut short (a record half written); %llu bytes after the last " + "whole record dropped", PB.path, (unsigned long long)(len - good)); + len = good; + } else { + why = "cut short (a record half written), and it cannot be cut back to its whole records"; + } + } if (why || !f) { // start empty: the index is dropped and the file rewritten if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0); free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0; @@ -393,8 +410,8 @@ static uint8_t *pb_read(uint32_t type, const uint8_t *key, uint32_t ks, uint32_t size_t at = (size_t)(s->off - PB.file_size) + 4; ok = at + s->size <= PB.pend_n; if (ok) memcpy(head, PB.pend + at, s->size); - } else { - ok = !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1; + } else { // rf is NULL only after pb_flush could not open the file again + ok = PB.rf && !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1; } uint32_t t, k; if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4); @@ -527,8 +544,17 @@ static void pb_flush(void) { int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1; if (f) ok = !fclose(f) && ok; if (ok) PB.file_size += PB.pend_n; - else rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory", PB.path); if (ok) PB.pend_n = 0; + if (!ok) { + // Part of the append may be in the file (a full disk): the file is cut back to file_size, where the index + // places the pending records, or the next append would land after those bytes. The read handle is closed + // meanwhile, since plat_file_truncate opens the file alone. + if (PB.rf) fclose(PB.rf); + int cut = plat_file_truncate(PB.path, PB.file_size); + PB.rf = fopen(PB.path, "rb"); + rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory%s", PB.path, + cut ? "" : " (and the file cannot be cut back to its whole records)"); + } } plat_mutex_unlock(&PB.lock); } diff --git a/tests/pb_torn_check.py b/tests/pb_torn_check.py new file mode 100644 index 00000000..95263634 --- /dev/null +++ b/tests/pb_torn_check.py @@ -0,0 +1,155 @@ +"""Check the index walk of pipeline-binaries.bin (runtime/vk_pipeline.c pb_index, NX2NAVI_PIPE_BINARIES=1). pb_init +keeps the whole records of a file that ends inside one (the process ended during an append, or a disk filled) and +cuts the file back to them; the walk must say where the last whole record ends. The check cuts the file's record +layout and pb_hash, pb_find, pb_insert and pb_index out of vk_pipeline.c, writes files of whole records, one with a +torn last record and one with a malformed record in the middle, and checks the offset and the index it gives. + +No GPU, no game files, any Python. Needs the Vulkan headers of MSYS2 UCRT64 (vulkan/vulkan.h). + +Usage: pb_torn_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH.""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include "platform/platform.h" +#ifdef _WIN32 +#define pb_seek _fseeki64 +#define pb_tell _ftelli64 +#else +#define pb_seek fseeko +#define pb_tell ftello +#endif +static uint64_t fnv(uint64_t h, const void *p, size_t n) { + const uint8_t *b = p; + for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 0x100000001b3ull; + return h; +} +#define FNV0 0xcbf29ce484222325ull +''' + +MAIN = r''' +static void put(FILE *f, uint32_t type, uint8_t key0, uint32_t payload) { + uint32_t size = PB_HEAD + payload, ks = 16; + uint8_t key[PB_KEY] = {key0, 1, 2, 3}; + fwrite(&size, 4, 1, f), fwrite(&type, 4, 1, f), fwrite(&ks, 4, 1, f), fwrite(key, PB_KEY, 1, f); + for (uint32_t i = 0; i < payload; i++) fputc((int)(i * 7 + key0), f); +} + +static void reset(void) { + free(PB.slot); + PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0; +} + +static uint64_t walk(const char *path, uint64_t *len) { + FILE *f = fopen(path, "rb"); + assert(f); + pb_seek(f, 0, SEEK_END); + *len = (uint64_t)pb_tell(f); + reset(); + uint64_t good = pb_index(f, *len); + fclose(f); + return good; +} + +int main(int argc, char **argv) { + const char *path = argv[1]; + PbHeader hd = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, 2, 16, {0}}; + uint64_t len, whole; + + /* Three whole records: the walk reaches the end. */ + FILE *f = fopen(path, "wb"); + fwrite(&hd, sizeof hd, 1, f); + put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY); + fclose(f); + assert(walk(path, &len) == len && PB.n == 3 && PB.blobs == 2 && PB.pipes == 1); + whole = len; + assert(pb_find(pb_hash(PB_BLOB, (const uint8_t[PB_KEY]){1, 1, 2, 3}, 16))->off == sizeof hd); + + /* A fourth record cut short, at every length from its size field to one byte short of whole: the three stay and + the walk ends where the fourth begins. */ + uint32_t rec4 = 4 + PB_HEAD + 500; + for (uint32_t cut = 1; cut < rec4; cut += cut < 16 ? 1 : 37) { + f = fopen(path, "wb"); + fwrite(&hd, sizeof hd, 1, f); + put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY); + fclose(f); + f = fopen(path, "ab"); + uint8_t *rec = calloc(1, rec4); + uint32_t size = PB_HEAD + 500, type = PB_BLOB, ks = 16; + memcpy(rec, &size, 4), memcpy(rec + 4, &type, 4), memcpy(rec + 8, &ks, 4), rec[12] = 4; + fwrite(rec, cut, 1, f); + free(rec); + fclose(f); + uint64_t good = walk(path, &len); + assert(len == whole + cut && good == whole && PB.n == 3); + } + + /* A malformed record in the middle: the records before it stay, the walk stops at it. */ + f = fopen(path, "wb"); + fwrite(&hd, sizeof hd, 1, f); + put(f, PB_BLOB, 1, 1000); + long at = ftell(f); + put(f, 7, 2, 3); /* no such type */ + put(f, PB_PIPE, 3, 4 + 16 + PB_KEY); + fclose(f); + assert(walk(path, &len) == (uint64_t)at && len > (uint64_t)at && PB.n == 1); + + /* The header alone: nothing indexed, the walk ends at the header. */ + f = fopen(path, "wb"); + fwrite(&hd, sizeof hd, 1, f); + fclose(f); + assert(walk(path, &len) == sizeof hd && len == sizeof hd && PB.n == 0); + + reset(); + printf("pipeline binaries index: whole records kept, a torn last record and a malformed one end the walk there\n"); + return 0; +} +''' + + +def cut(source, start, end): + i = source.index(start) + return source[i:source.index(end, i)] + + +def main(): + src = (ROOT / 'runtime' / 'vk_pipeline.c').read_text(encoding='utf-8') + production = cut(src, '#define PB_MAX_BIN 8', '// Opens pipeline-binaries.bin') + work = ROOT / 'build' / 'test' / 'pb_torn' + work.mkdir(parents=True, exist_ok=True) + c = work / 'check.c' + c.write_text(HARNESS + production + MAIN, encoding='utf-8') + exe = work / ('check.exe' if os.name == 'nt' else 'check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-Wno-unused-variable', + '-march=x86-64-v3', '-I', str(ROOT / 'runtime'), str(c), '-o', str(exe)], env=env) + return subprocess.run([str(exe), str(work / 'pipeline-binaries.bin')], env=env, timeout=60).returncode + + +if __name__ == '__main__': + sys.exit(main())