Keep the whole records of a pipeline-binaries.bin that ends inside a record, and cut a failed append back out.

With NX2NAVI_PIPE_BINARIES=1, a process ended during the autosave's append (a kill, a crash, a fatal on another
thread) or a disk that filled left the file ending inside a record, and pb_init then started empty, dropping every
whole record (up to the 1 GB limit). After a failed append pb_flush also left the torn bytes in the file while
file_size stayed, so every later record's offset was off by them. The index walk moves into pb_index, which returns
where the last whole record ends; pb_init keeps the index of those records and cuts the file back to that offset with
plat_file_truncate after it closed its handle, and starts empty only when the cut fails. pb_flush closes the read
handle, cuts the file back to file_size and opens it again when an append fails (plat_file_truncate opens the file
alone, so it fails while another handle is open); pb_read refuses to read when that reopen failed.

Not reproduced before the change; the option is off by default. tests/pb_torn_check.py (new) cuts the record layout
and pb_index out of vk_pipeline.c and walks files of whole records, with a last record cut short at many lengths and
with a malformed record; vk_pipeline.c compiles with no new warnings.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/building.md

@@ -643,6 +643,11 @@ A change to how `pipeline-cache.bin` is read (`runtime/vk_cache_file.h`, `runtim
 through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU;
 about 2 GB of memory for a few seconds.
 
+A change to how `pipeline-binaries.bin` is indexed (`runtime/vk_pipeline.c` `pb_index`, `pb_init`) passes
+`python tests\pb_torn_check.py`, which cuts the record layout and the index walk out of `vk_pipeline.c` and walks files
+of whole records, with a last record cut short at many lengths and with a malformed record: the whole records before
+it are indexed and the walk ends where they end, which is where `pb_init` cuts the file back to. No game files, no GPU.
+
 A change to the layout of `shader-prepared.bin` (`runtime/prp_file.h`, `runtime/gfx_shader.c`
 `gfx_shader_prepare_seed`, `tools/make_game.py` `prepared_from`) passes `python tests\prp_file_check.py`, which builds
 the header alone, checks the parse of both versions and has `make_game.py` read the list the C side wrote. No game

runtime/vk_pipeline.c

@@ -315,6 +315,27 @@ static void pb_rewrite(const PbHeader *hd) {   // an empty file with this driver
     PB.file_size = f ? sizeof *hd : 0;
 }
 
+// Indexes the records of f (len bytes) after its header: per record its type, key and offset. Returns the offset
+// after the last whole record, which is len unless the file ends inside a record (an append the process did not
+// finish) or holds a malformed one.
+static uint64_t pb_index(FILE *f, uint64_t len) {
+    uint64_t good = sizeof(PbHeader);
+    for (;;) {
+        uint32_t head[3];
+        uint8_t key[PB_KEY];
+        if (pb_seek(f, (int64_t)good, SEEK_SET) || fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY ||
+            head[0] < PB_HEAD || (head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 ||
+            good + 4 + head[0] > len)
+            break;
+        if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) {
+            if (head[1] == PB_PIPE) PB.pipes++;
+            else PB.blobs++;
+        }
+        good += 4 + (uint64_t)head[0];
+    }
+    return good;
+}
+
 // Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache).
 static void pb_init(VkCtx *c) {
     if (!c->has_pipeline_binary || !P.cache_path[0]) return;
@@ -348,25 +369,21 @@ static void pb_init(VkCtx *c) {
         pb_seek(f, 0, SEEK_SET);
         if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version";
         else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB";
-        else {
-            good = sizeof hd;
-            for (;;) {   // index: per record its type, key and offset
-                uint32_t head[3];
-                uint8_t key[PB_KEY];
-                if (fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || head[0] < PB_HEAD ||
-                    (head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || good + 4 + head[0] > len)
-                    break;
-                if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) {
-                    if (head[1] == PB_PIPE) PB.pipes++;
-                    else PB.blobs++;
-                }
-                good += 4 + (uint64_t)head[0];
-                if (pb_seek(f, (int64_t)good, SEEK_SET)) break;
-            }
-            if (good != len) why = "cut short (a record half written)";
-        }
+        else good = pb_index(f, len);
         fclose(f);
     }
+    // A file that ends inside a record (the process ended during an append, or a disk filled): the whole records
+    // before it stay, and the file is cut back to them so that appends go on from there. Only when it cannot be cut
+    // does it start empty.
+    if (f && !why && good != len) {
+        if (plat_file_truncate(PB.path, good)) {
+            rt_log("vk_pipeline: pipeline binaries: %s was cut short (a record half written); %llu bytes after the last "
+                   "whole record dropped", PB.path, (unsigned long long)(len - good));
+            len = good;
+        } else {
+            why = "cut short (a record half written), and it cannot be cut back to its whole records";
+        }
+    }
     if (why || !f) {   // start empty: the index is dropped and the file rewritten
         if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0);
         free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
@@ -393,8 +410,8 @@ static uint8_t *pb_read(uint32_t type, const uint8_t *key, uint32_t ks, uint32_t
         size_t at = (size_t)(s->off - PB.file_size) + 4;
         ok = at + s->size <= PB.pend_n;
         if (ok) memcpy(head, PB.pend + at, s->size);
-    } else {
-        ok = !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1;
+    } else {   // rf is NULL only after pb_flush could not open the file again
+        ok = PB.rf && !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1;
     }
     uint32_t t, k;
     if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4);
@@ -527,8 +544,17 @@ static void pb_flush(void) {
         int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1;
         if (f) ok = !fclose(f) && ok;
         if (ok) PB.file_size += PB.pend_n;
-        else rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory", PB.path);
         if (ok) PB.pend_n = 0;
+        if (!ok) {
+            // Part of the append may be in the file (a full disk): the file is cut back to file_size, where the index
+            // places the pending records, or the next append would land after those bytes. The read handle is closed
+            // meanwhile, since plat_file_truncate opens the file alone.
+            if (PB.rf) fclose(PB.rf);
+            int cut = plat_file_truncate(PB.path, PB.file_size);
+            PB.rf = fopen(PB.path, "rb");
+            rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory%s", PB.path,
+                   cut ? "" : " (and the file cannot be cut back to its whole records)");
+        }
     }
     plat_mutex_unlock(&PB.lock);
 }

tests/pb_torn_check.py

@@ -0,0 +1,155 @@
+"""Check the index walk of pipeline-binaries.bin (runtime/vk_pipeline.c pb_index, NX2NAVI_PIPE_BINARIES=1). pb_init
+keeps the whole records of a file that ends inside one (the process ended during an append, or a disk filled) and
+cuts the file back to them; the walk must say where the last whole record ends. The check cuts the file's record
+layout and pb_hash, pb_find, pb_insert and pb_index out of vk_pipeline.c, writes files of whole records, one with a
+torn last record and one with a malformed record in the middle, and checks the offset and the index it gives.
+
+No GPU, no game files, any Python. Needs the Vulkan headers of MSYS2 UCRT64 (vulkan/vulkan.h).
+
+Usage: pb_torn_check.py
+"""
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
+
+
+def compiler():
+    """$CC, else MSYS2 UCRT64's clang, else clang on PATH."""
+    if os.environ.get('CC'):
+        return os.environ['CC']
+    if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
+        return str(MSYS_BIN / 'clang.exe')
+    return shutil.which('clang') or 'clang'
+
+
+HARNESS = r'''
+#include <vulkan/vulkan.h>
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "platform/platform.h"
+#ifdef _WIN32
+#define pb_seek _fseeki64
+#define pb_tell _ftelli64
+#else
+#define pb_seek fseeko
+#define pb_tell ftello
+#endif
+static uint64_t fnv(uint64_t h, const void *p, size_t n) {
+    const uint8_t *b = p;
+    for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 0x100000001b3ull;
+    return h;
+}
+#define FNV0 0xcbf29ce484222325ull
+'''
+
+MAIN = r'''
+static void put(FILE *f, uint32_t type, uint8_t key0, uint32_t payload) {
+    uint32_t size = PB_HEAD + payload, ks = 16;
+    uint8_t key[PB_KEY] = {key0, 1, 2, 3};
+    fwrite(&size, 4, 1, f), fwrite(&type, 4, 1, f), fwrite(&ks, 4, 1, f), fwrite(key, PB_KEY, 1, f);
+    for (uint32_t i = 0; i < payload; i++) fputc((int)(i * 7 + key0), f);
+}
+
+static void reset(void) {
+    free(PB.slot);
+    PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
+}
+
+static uint64_t walk(const char *path, uint64_t *len) {
+    FILE *f = fopen(path, "rb");
+    assert(f);
+    pb_seek(f, 0, SEEK_END);
+    *len = (uint64_t)pb_tell(f);
+    reset();
+    uint64_t good = pb_index(f, *len);
+    fclose(f);
+    return good;
+}
+
+int main(int argc, char **argv) {
+    const char *path = argv[1];
+    PbHeader hd = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, 2, 16, {0}};
+    uint64_t len, whole;
+
+    /* Three whole records: the walk reaches the end. */
+    FILE *f = fopen(path, "wb");
+    fwrite(&hd, sizeof hd, 1, f);
+    put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
+    fclose(f);
+    assert(walk(path, &len) == len && PB.n == 3 && PB.blobs == 2 && PB.pipes == 1);
+    whole = len;
+    assert(pb_find(pb_hash(PB_BLOB, (const uint8_t[PB_KEY]){1, 1, 2, 3}, 16))->off == sizeof hd);
+
+    /* A fourth record cut short, at every length from its size field to one byte short of whole: the three stay and
+       the walk ends where the fourth begins. */
+    uint32_t rec4 = 4 + PB_HEAD + 500;
+    for (uint32_t cut = 1; cut < rec4; cut += cut < 16 ? 1 : 37) {
+        f = fopen(path, "wb");
+        fwrite(&hd, sizeof hd, 1, f);
+        put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
+        fclose(f);
+        f = fopen(path, "ab");
+        uint8_t *rec = calloc(1, rec4);
+        uint32_t size = PB_HEAD + 500, type = PB_BLOB, ks = 16;
+        memcpy(rec, &size, 4), memcpy(rec + 4, &type, 4), memcpy(rec + 8, &ks, 4), rec[12] = 4;
+        fwrite(rec, cut, 1, f);
+        free(rec);
+        fclose(f);
+        uint64_t good = walk(path, &len);
+        assert(len == whole + cut && good == whole && PB.n == 3);
+    }
+
+    /* A malformed record in the middle: the records before it stay, the walk stops at it. */
+    f = fopen(path, "wb");
+    fwrite(&hd, sizeof hd, 1, f);
+    put(f, PB_BLOB, 1, 1000);
+    long at = ftell(f);
+    put(f, 7, 2, 3);   /* no such type */
+    put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
+    fclose(f);
+    assert(walk(path, &len) == (uint64_t)at && len > (uint64_t)at && PB.n == 1);
+
+    /* The header alone: nothing indexed, the walk ends at the header. */
+    f = fopen(path, "wb");
+    fwrite(&hd, sizeof hd, 1, f);
+    fclose(f);
+    assert(walk(path, &len) == sizeof hd && len == sizeof hd && PB.n == 0);
+
+    reset();
+    printf("pipeline binaries index: whole records kept, a torn last record and a malformed one end the walk there\n");
+    return 0;
+}
+'''
+
+
+def cut(source, start, end):
+    i = source.index(start)
+    return source[i:source.index(end, i)]
+
+
+def main():
+    src = (ROOT / 'runtime' / 'vk_pipeline.c').read_text(encoding='utf-8')
+    production = cut(src, '#define PB_MAX_BIN 8', '// Opens pipeline-binaries.bin')
+    work = ROOT / 'build' / 'test' / 'pb_torn'
+    work.mkdir(parents=True, exist_ok=True)
+    c = work / 'check.c'
+    c.write_text(HARNESS + production + MAIN, encoding='utf-8')
+    exe = work / ('check.exe' if os.name == 'nt' else 'check')
+    env = dict(os.environ)
+    if os.name == 'nt' and MSYS_BIN.exists():
+        env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
+    subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-Wno-unused-variable',
+                           '-march=x86-64-v3', '-I', str(ROOT / 'runtime'), str(c), '-o', str(exe)], env=env)
+    return subprocess.run([str(exe), str(work / 'pipeline-binaries.bin')], env=env, timeout=60).returncode
+
+
+if __name__ == '__main__':
+    sys.exit(main())