mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-20 06:09:58 -04:00
Keep the whole records of a pipeline-binaries.bin that ends inside a record, and cut a failed append back out.
With NX2NAVI_PIPE_BINARIES=1, a process ended during the autosave's append (a kill, a crash, a fatal on another thread) or a disk that filled left the file ending inside a record, and pb_init then started empty, dropping every whole record (up to the 1 GB limit). After a failed append pb_flush also left the torn bytes in the file while file_size stayed, so every later record's offset was off by them. The index walk moves into pb_index, which returns where the last whole record ends; pb_init keeps the index of those records and cuts the file back to that offset with plat_file_truncate after it closed its handle, and starts empty only when the cut fails. pb_flush closes the read handle, cuts the file back to file_size and opens it again when an append fails (plat_file_truncate opens the file alone, so it fails while another handle is open); pb_read refuses to read when that reopen failed. Not reproduced before the change; the option is off by default. tests/pb_torn_check.py (new) cuts the record layout and pb_index out of vk_pipeline.c and walks files of whole records, with a last record cut short at many lengths and with a malformed record; vk_pipeline.c compiles with no new warnings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -643,6 +643,11 @@ A change to how `pipeline-cache.bin` is read (`runtime/vk_cache_file.h`, `runtim
|
|||||||
through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU;
|
through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU;
|
||||||
about 2 GB of memory for a few seconds.
|
about 2 GB of memory for a few seconds.
|
||||||
|
|
||||||
|
A change to how `pipeline-binaries.bin` is indexed (`runtime/vk_pipeline.c` `pb_index`, `pb_init`) passes
|
||||||
|
`python tests\pb_torn_check.py`, which cuts the record layout and the index walk out of `vk_pipeline.c` and walks files
|
||||||
|
of whole records, with a last record cut short at many lengths and with a malformed record: the whole records before
|
||||||
|
it are indexed and the walk ends where they end, which is where `pb_init` cuts the file back to. No game files, no GPU.
|
||||||
|
|
||||||
A change to the layout of `shader-prepared.bin` (`runtime/prp_file.h`, `runtime/gfx_shader.c`
|
A change to the layout of `shader-prepared.bin` (`runtime/prp_file.h`, `runtime/gfx_shader.c`
|
||||||
`gfx_shader_prepare_seed`, `tools/make_game.py` `prepared_from`) passes `python tests\prp_file_check.py`, which builds
|
`gfx_shader_prepare_seed`, `tools/make_game.py` `prepared_from`) passes `python tests\prp_file_check.py`, which builds
|
||||||
the header alone, checks the parse of both versions and has `make_game.py` read the list the C side wrote. No game
|
the header alone, checks the parse of both versions and has `make_game.py` read the list the C side wrote. No game
|
||||||
|
|||||||
@@ -315,6 +315,27 @@ static void pb_rewrite(const PbHeader *hd) { // an empty file with this driver
|
|||||||
PB.file_size = f ? sizeof *hd : 0;
|
PB.file_size = f ? sizeof *hd : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Indexes the records of f (len bytes) after its header: per record its type, key and offset. Returns the offset
|
||||||
|
// after the last whole record, which is len unless the file ends inside a record (an append the process did not
|
||||||
|
// finish) or holds a malformed one.
|
||||||
|
static uint64_t pb_index(FILE *f, uint64_t len) {
|
||||||
|
uint64_t good = sizeof(PbHeader);
|
||||||
|
for (;;) {
|
||||||
|
uint32_t head[3];
|
||||||
|
uint8_t key[PB_KEY];
|
||||||
|
if (pb_seek(f, (int64_t)good, SEEK_SET) || fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY ||
|
||||||
|
head[0] < PB_HEAD || (head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 ||
|
||||||
|
good + 4 + head[0] > len)
|
||||||
|
break;
|
||||||
|
if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) {
|
||||||
|
if (head[1] == PB_PIPE) PB.pipes++;
|
||||||
|
else PB.blobs++;
|
||||||
|
}
|
||||||
|
good += 4 + (uint64_t)head[0];
|
||||||
|
}
|
||||||
|
return good;
|
||||||
|
}
|
||||||
|
|
||||||
// Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache).
|
// Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache).
|
||||||
static void pb_init(VkCtx *c) {
|
static void pb_init(VkCtx *c) {
|
||||||
if (!c->has_pipeline_binary || !P.cache_path[0]) return;
|
if (!c->has_pipeline_binary || !P.cache_path[0]) return;
|
||||||
@@ -348,25 +369,21 @@ static void pb_init(VkCtx *c) {
|
|||||||
pb_seek(f, 0, SEEK_SET);
|
pb_seek(f, 0, SEEK_SET);
|
||||||
if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version";
|
if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version";
|
||||||
else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB";
|
else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB";
|
||||||
else {
|
else good = pb_index(f, len);
|
||||||
good = sizeof hd;
|
|
||||||
for (;;) { // index: per record its type, key and offset
|
|
||||||
uint32_t head[3];
|
|
||||||
uint8_t key[PB_KEY];
|
|
||||||
if (fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || head[0] < PB_HEAD ||
|
|
||||||
(head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || good + 4 + head[0] > len)
|
|
||||||
break;
|
|
||||||
if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) {
|
|
||||||
if (head[1] == PB_PIPE) PB.pipes++;
|
|
||||||
else PB.blobs++;
|
|
||||||
}
|
|
||||||
good += 4 + (uint64_t)head[0];
|
|
||||||
if (pb_seek(f, (int64_t)good, SEEK_SET)) break;
|
|
||||||
}
|
|
||||||
if (good != len) why = "cut short (a record half written)";
|
|
||||||
}
|
|
||||||
fclose(f);
|
fclose(f);
|
||||||
}
|
}
|
||||||
|
// A file that ends inside a record (the process ended during an append, or a disk filled): the whole records
|
||||||
|
// before it stay, and the file is cut back to them so that appends go on from there. Only when it cannot be cut
|
||||||
|
// does it start empty.
|
||||||
|
if (f && !why && good != len) {
|
||||||
|
if (plat_file_truncate(PB.path, good)) {
|
||||||
|
rt_log("vk_pipeline: pipeline binaries: %s was cut short (a record half written); %llu bytes after the last "
|
||||||
|
"whole record dropped", PB.path, (unsigned long long)(len - good));
|
||||||
|
len = good;
|
||||||
|
} else {
|
||||||
|
why = "cut short (a record half written), and it cannot be cut back to its whole records";
|
||||||
|
}
|
||||||
|
}
|
||||||
if (why || !f) { // start empty: the index is dropped and the file rewritten
|
if (why || !f) { // start empty: the index is dropped and the file rewritten
|
||||||
if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0);
|
if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0);
|
||||||
free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
|
free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
|
||||||
@@ -393,8 +410,8 @@ static uint8_t *pb_read(uint32_t type, const uint8_t *key, uint32_t ks, uint32_t
|
|||||||
size_t at = (size_t)(s->off - PB.file_size) + 4;
|
size_t at = (size_t)(s->off - PB.file_size) + 4;
|
||||||
ok = at + s->size <= PB.pend_n;
|
ok = at + s->size <= PB.pend_n;
|
||||||
if (ok) memcpy(head, PB.pend + at, s->size);
|
if (ok) memcpy(head, PB.pend + at, s->size);
|
||||||
} else {
|
} else { // rf is NULL only after pb_flush could not open the file again
|
||||||
ok = !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1;
|
ok = PB.rf && !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1;
|
||||||
}
|
}
|
||||||
uint32_t t, k;
|
uint32_t t, k;
|
||||||
if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4);
|
if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4);
|
||||||
@@ -527,8 +544,17 @@ static void pb_flush(void) {
|
|||||||
int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1;
|
int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1;
|
||||||
if (f) ok = !fclose(f) && ok;
|
if (f) ok = !fclose(f) && ok;
|
||||||
if (ok) PB.file_size += PB.pend_n;
|
if (ok) PB.file_size += PB.pend_n;
|
||||||
else rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory", PB.path);
|
|
||||||
if (ok) PB.pend_n = 0;
|
if (ok) PB.pend_n = 0;
|
||||||
|
if (!ok) {
|
||||||
|
// Part of the append may be in the file (a full disk): the file is cut back to file_size, where the index
|
||||||
|
// places the pending records, or the next append would land after those bytes. The read handle is closed
|
||||||
|
// meanwhile, since plat_file_truncate opens the file alone.
|
||||||
|
if (PB.rf) fclose(PB.rf);
|
||||||
|
int cut = plat_file_truncate(PB.path, PB.file_size);
|
||||||
|
PB.rf = fopen(PB.path, "rb");
|
||||||
|
rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory%s", PB.path,
|
||||||
|
cut ? "" : " (and the file cannot be cut back to its whole records)");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
plat_mutex_unlock(&PB.lock);
|
plat_mutex_unlock(&PB.lock);
|
||||||
}
|
}
|
||||||
|
|||||||
155
tests/pb_torn_check.py
Normal file
155
tests/pb_torn_check.py
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
"""Check the index walk of pipeline-binaries.bin (runtime/vk_pipeline.c pb_index, NX2NAVI_PIPE_BINARIES=1). pb_init
|
||||||
|
keeps the whole records of a file that ends inside one (the process ended during an append, or a disk filled) and
|
||||||
|
cuts the file back to them; the walk must say where the last whole record ends. The check cuts the file's record
|
||||||
|
layout and pb_hash, pb_find, pb_insert and pb_index out of vk_pipeline.c, writes files of whole records, one with a
|
||||||
|
torn last record and one with a malformed record in the middle, and checks the offset and the index it gives.
|
||||||
|
|
||||||
|
No GPU, no game files, any Python. Needs the Vulkan headers of MSYS2 UCRT64 (vulkan/vulkan.h).
|
||||||
|
|
||||||
|
Usage: pb_torn_check.py
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||||
|
|
||||||
|
|
||||||
|
def compiler():
|
||||||
|
"""$CC, else MSYS2 UCRT64's clang, else clang on PATH."""
|
||||||
|
if os.environ.get('CC'):
|
||||||
|
return os.environ['CC']
|
||||||
|
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
|
||||||
|
return str(MSYS_BIN / 'clang.exe')
|
||||||
|
return shutil.which('clang') or 'clang'
|
||||||
|
|
||||||
|
|
||||||
|
HARNESS = r'''
|
||||||
|
#include <vulkan/vulkan.h>
|
||||||
|
#include <assert.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include "platform/platform.h"
|
||||||
|
#ifdef _WIN32
|
||||||
|
#define pb_seek _fseeki64
|
||||||
|
#define pb_tell _ftelli64
|
||||||
|
#else
|
||||||
|
#define pb_seek fseeko
|
||||||
|
#define pb_tell ftello
|
||||||
|
#endif
|
||||||
|
static uint64_t fnv(uint64_t h, const void *p, size_t n) {
|
||||||
|
const uint8_t *b = p;
|
||||||
|
for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 0x100000001b3ull;
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
#define FNV0 0xcbf29ce484222325ull
|
||||||
|
'''
|
||||||
|
|
||||||
|
MAIN = r'''
|
||||||
|
static void put(FILE *f, uint32_t type, uint8_t key0, uint32_t payload) {
|
||||||
|
uint32_t size = PB_HEAD + payload, ks = 16;
|
||||||
|
uint8_t key[PB_KEY] = {key0, 1, 2, 3};
|
||||||
|
fwrite(&size, 4, 1, f), fwrite(&type, 4, 1, f), fwrite(&ks, 4, 1, f), fwrite(key, PB_KEY, 1, f);
|
||||||
|
for (uint32_t i = 0; i < payload; i++) fputc((int)(i * 7 + key0), f);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void reset(void) {
|
||||||
|
free(PB.slot);
|
||||||
|
PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint64_t walk(const char *path, uint64_t *len) {
|
||||||
|
FILE *f = fopen(path, "rb");
|
||||||
|
assert(f);
|
||||||
|
pb_seek(f, 0, SEEK_END);
|
||||||
|
*len = (uint64_t)pb_tell(f);
|
||||||
|
reset();
|
||||||
|
uint64_t good = pb_index(f, *len);
|
||||||
|
fclose(f);
|
||||||
|
return good;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv) {
|
||||||
|
const char *path = argv[1];
|
||||||
|
PbHeader hd = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, 2, 16, {0}};
|
||||||
|
uint64_t len, whole;
|
||||||
|
|
||||||
|
/* Three whole records: the walk reaches the end. */
|
||||||
|
FILE *f = fopen(path, "wb");
|
||||||
|
fwrite(&hd, sizeof hd, 1, f);
|
||||||
|
put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
|
||||||
|
fclose(f);
|
||||||
|
assert(walk(path, &len) == len && PB.n == 3 && PB.blobs == 2 && PB.pipes == 1);
|
||||||
|
whole = len;
|
||||||
|
assert(pb_find(pb_hash(PB_BLOB, (const uint8_t[PB_KEY]){1, 1, 2, 3}, 16))->off == sizeof hd);
|
||||||
|
|
||||||
|
/* A fourth record cut short, at every length from its size field to one byte short of whole: the three stay and
|
||||||
|
the walk ends where the fourth begins. */
|
||||||
|
uint32_t rec4 = 4 + PB_HEAD + 500;
|
||||||
|
for (uint32_t cut = 1; cut < rec4; cut += cut < 16 ? 1 : 37) {
|
||||||
|
f = fopen(path, "wb");
|
||||||
|
fwrite(&hd, sizeof hd, 1, f);
|
||||||
|
put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
|
||||||
|
fclose(f);
|
||||||
|
f = fopen(path, "ab");
|
||||||
|
uint8_t *rec = calloc(1, rec4);
|
||||||
|
uint32_t size = PB_HEAD + 500, type = PB_BLOB, ks = 16;
|
||||||
|
memcpy(rec, &size, 4), memcpy(rec + 4, &type, 4), memcpy(rec + 8, &ks, 4), rec[12] = 4;
|
||||||
|
fwrite(rec, cut, 1, f);
|
||||||
|
free(rec);
|
||||||
|
fclose(f);
|
||||||
|
uint64_t good = walk(path, &len);
|
||||||
|
assert(len == whole + cut && good == whole && PB.n == 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A malformed record in the middle: the records before it stay, the walk stops at it. */
|
||||||
|
f = fopen(path, "wb");
|
||||||
|
fwrite(&hd, sizeof hd, 1, f);
|
||||||
|
put(f, PB_BLOB, 1, 1000);
|
||||||
|
long at = ftell(f);
|
||||||
|
put(f, 7, 2, 3); /* no such type */
|
||||||
|
put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
|
||||||
|
fclose(f);
|
||||||
|
assert(walk(path, &len) == (uint64_t)at && len > (uint64_t)at && PB.n == 1);
|
||||||
|
|
||||||
|
/* The header alone: nothing indexed, the walk ends at the header. */
|
||||||
|
f = fopen(path, "wb");
|
||||||
|
fwrite(&hd, sizeof hd, 1, f);
|
||||||
|
fclose(f);
|
||||||
|
assert(walk(path, &len) == sizeof hd && len == sizeof hd && PB.n == 0);
|
||||||
|
|
||||||
|
reset();
|
||||||
|
printf("pipeline binaries index: whole records kept, a torn last record and a malformed one end the walk there\n");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def cut(source, start, end):
|
||||||
|
i = source.index(start)
|
||||||
|
return source[i:source.index(end, i)]
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
src = (ROOT / 'runtime' / 'vk_pipeline.c').read_text(encoding='utf-8')
|
||||||
|
production = cut(src, '#define PB_MAX_BIN 8', '// Opens pipeline-binaries.bin')
|
||||||
|
work = ROOT / 'build' / 'test' / 'pb_torn'
|
||||||
|
work.mkdir(parents=True, exist_ok=True)
|
||||||
|
c = work / 'check.c'
|
||||||
|
c.write_text(HARNESS + production + MAIN, encoding='utf-8')
|
||||||
|
exe = work / ('check.exe' if os.name == 'nt' else 'check')
|
||||||
|
env = dict(os.environ)
|
||||||
|
if os.name == 'nt' and MSYS_BIN.exists():
|
||||||
|
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||||
|
subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-Wno-unused-variable',
|
||||||
|
'-march=x86-64-v3', '-I', str(ROOT / 'runtime'), str(c), '-o', str(exe)], env=env)
|
||||||
|
return subprocess.run([str(exe), str(work / 'pipeline-binaries.bin')], env=env, timeout=60).returncode
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main())
|
||||||
Reference in New Issue
Block a user