Files
NX2navi/tests/pb_torn_check.py
chrg6 79c41ca76a Keep the whole records of a pipeline-binaries.bin that ends inside a record, and cut a failed append back out.
With NX2NAVI_PIPE_BINARIES=1, a process ended during the autosave's append (a kill, a crash, a fatal on another
thread) or a disk that filled left the file ending inside a record, and pb_init then started empty, dropping every
whole record (up to the 1 GB limit). After a failed append pb_flush also left the torn bytes in the file while
file_size stayed, so every later record's offset was off by them. The index walk moves into pb_index, which returns
where the last whole record ends; pb_init keeps the index of those records and cuts the file back to that offset with
plat_file_truncate after it closed its handle, and starts empty only when the cut fails. pb_flush closes the read
handle, cuts the file back to file_size and opens it again when an append fails (plat_file_truncate opens the file
alone, so it fails while another handle is open); pb_read refuses to read when that reopen failed.

Not reproduced before the change; the option is off by default. tests/pb_torn_check.py (new) cuts the record layout
and pb_index out of vk_pipeline.c and walks files of whole records, with a last record cut short at many lengths and
with a malformed record; vk_pipeline.c compiles with no new warnings.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 18:41:45 -05:00

156 lines
5.5 KiB
Python

"""Check the index walk of pipeline-binaries.bin (runtime/vk_pipeline.c pb_index, NX2NAVI_PIPE_BINARIES=1). pb_init
keeps the whole records of a file that ends inside one (the process ended during an append, or a disk filled) and
cuts the file back to them; the walk must say where the last whole record ends. The check cuts the file's record
layout and pb_hash, pb_find, pb_insert and pb_index out of vk_pipeline.c, writes files of whole records, one with a
torn last record and one with a malformed record in the middle, and checks the offset and the index it gives.
No GPU, no game files, any Python. Needs the Vulkan headers of MSYS2 UCRT64 (vulkan/vulkan.h).
Usage: pb_torn_check.py
"""
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
def compiler():
"""$CC, else MSYS2 UCRT64's clang, else clang on PATH."""
if os.environ.get('CC'):
return os.environ['CC']
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
HARNESS = r'''
#include <vulkan/vulkan.h>
#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "platform/platform.h"
#ifdef _WIN32
#define pb_seek _fseeki64
#define pb_tell _ftelli64
#else
#define pb_seek fseeko
#define pb_tell ftello
#endif
static uint64_t fnv(uint64_t h, const void *p, size_t n) {
const uint8_t *b = p;
for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 0x100000001b3ull;
return h;
}
#define FNV0 0xcbf29ce484222325ull
'''
MAIN = r'''
static void put(FILE *f, uint32_t type, uint8_t key0, uint32_t payload) {
uint32_t size = PB_HEAD + payload, ks = 16;
uint8_t key[PB_KEY] = {key0, 1, 2, 3};
fwrite(&size, 4, 1, f), fwrite(&type, 4, 1, f), fwrite(&ks, 4, 1, f), fwrite(key, PB_KEY, 1, f);
for (uint32_t i = 0; i < payload; i++) fputc((int)(i * 7 + key0), f);
}
static void reset(void) {
free(PB.slot);
PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
}
static uint64_t walk(const char *path, uint64_t *len) {
FILE *f = fopen(path, "rb");
assert(f);
pb_seek(f, 0, SEEK_END);
*len = (uint64_t)pb_tell(f);
reset();
uint64_t good = pb_index(f, *len);
fclose(f);
return good;
}
int main(int argc, char **argv) {
const char *path = argv[1];
PbHeader hd = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, 2, 16, {0}};
uint64_t len, whole;
/* Three whole records: the walk reaches the end. */
FILE *f = fopen(path, "wb");
fwrite(&hd, sizeof hd, 1, f);
put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
fclose(f);
assert(walk(path, &len) == len && PB.n == 3 && PB.blobs == 2 && PB.pipes == 1);
whole = len;
assert(pb_find(pb_hash(PB_BLOB, (const uint8_t[PB_KEY]){1, 1, 2, 3}, 16))->off == sizeof hd);
/* A fourth record cut short, at every length from its size field to one byte short of whole: the three stay and
the walk ends where the fourth begins. */
uint32_t rec4 = 4 + PB_HEAD + 500;
for (uint32_t cut = 1; cut < rec4; cut += cut < 16 ? 1 : 37) {
f = fopen(path, "wb");
fwrite(&hd, sizeof hd, 1, f);
put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
fclose(f);
f = fopen(path, "ab");
uint8_t *rec = calloc(1, rec4);
uint32_t size = PB_HEAD + 500, type = PB_BLOB, ks = 16;
memcpy(rec, &size, 4), memcpy(rec + 4, &type, 4), memcpy(rec + 8, &ks, 4), rec[12] = 4;
fwrite(rec, cut, 1, f);
free(rec);
fclose(f);
uint64_t good = walk(path, &len);
assert(len == whole + cut && good == whole && PB.n == 3);
}
/* A malformed record in the middle: the records before it stay, the walk stops at it. */
f = fopen(path, "wb");
fwrite(&hd, sizeof hd, 1, f);
put(f, PB_BLOB, 1, 1000);
long at = ftell(f);
put(f, 7, 2, 3); /* no such type */
put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
fclose(f);
assert(walk(path, &len) == (uint64_t)at && len > (uint64_t)at && PB.n == 1);
/* The header alone: nothing indexed, the walk ends at the header. */
f = fopen(path, "wb");
fwrite(&hd, sizeof hd, 1, f);
fclose(f);
assert(walk(path, &len) == sizeof hd && len == sizeof hd && PB.n == 0);
reset();
printf("pipeline binaries index: whole records kept, a torn last record and a malformed one end the walk there\n");
return 0;
}
'''
def cut(source, start, end):
i = source.index(start)
return source[i:source.index(end, i)]
def main():
src = (ROOT / 'runtime' / 'vk_pipeline.c').read_text(encoding='utf-8')
production = cut(src, '#define PB_MAX_BIN 8', '// Opens pipeline-binaries.bin')
work = ROOT / 'build' / 'test' / 'pb_torn'
work.mkdir(parents=True, exist_ok=True)
c = work / 'check.c'
c.write_text(HARNESS + production + MAIN, encoding='utf-8')
exe = work / ('check.exe' if os.name == 'nt' else 'check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-Wno-unused-variable',
'-march=x86-64-v3', '-I', str(ROOT / 'runtime'), str(c), '-o', str(exe)], env=env)
return subprocess.run([str(exe), str(work / 'pipeline-binaries.bin')], env=env, timeout=60).returncode
if __name__ == '__main__':
sys.exit(main())