mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-20 08:19:56 -04:00
Say that pipeline-binaries.bin is cut back also at a malformed record, and check the cut itself.
The review of the torn-file fix (43ac5ae3) found that pb_index also stops at a malformed record in the middle of the file, so pb_init cuts the file back to it and drops the whole records after it, while its log line and the comment above the cut spoke only of a record half written. It also found that tests/pb_torn_check.py covered only the index walk, not the keep-and-cut that is the fix. The decision after the walk moves from pb_init into pb_keep(path, good, len), which returns the length to go on with (len when the records fill the file, good after cutting it back) or 0 when the file cannot be cut and starts empty; its log line now reads "<path> ends inside a record or holds a malformed one at byte <good>; <n> bytes from there dropped", and the comment and the start-empty reason say the same. Behaviour is unchanged. pb_torn_check.py now also cuts pb_keep out of vk_pipeline.c and runs it on real files, with plat_file_truncate a stand-in that cuts with the C library: a torn file is cut to its whole records and logged, a second walk reaches the end of what is left, a record appended after the cut is read back with the others; a malformed record in the middle is cut off with what follows; a whole file is left as it is, unlogged; and a file that cannot be cut gives 0 and stays as it was. The check passes, vk_pipeline.c compiles, and its -Wall -Wextra warnings are the four it had, none in pb_keep or pb_init. docs/building.md names pb_keep and what the check now covers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -643,10 +643,12 @@ A change to how `pipeline-cache.bin` is read (`runtime/vk_cache_file.h`, `runtim
|
|||||||
through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU;
|
through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU;
|
||||||
about 2 GB of memory for a few seconds.
|
about 2 GB of memory for a few seconds.
|
||||||
|
|
||||||
A change to how `pipeline-binaries.bin` is indexed (`runtime/vk_pipeline.c` `pb_index`, `pb_init`) passes
|
A change to how `pipeline-binaries.bin` is indexed or cut back (`runtime/vk_pipeline.c` `pb_index`, `pb_keep`,
|
||||||
`python tests\pb_torn_check.py`, which cuts the record layout and the index walk out of `vk_pipeline.c` and walks files
|
`pb_init`) passes `python tests\pb_torn_check.py`, which cuts the record layout, the index walk and `pb_keep` out of
|
||||||
of whole records, with a last record cut short at many lengths and with a malformed record: the whole records before
|
`vk_pipeline.c` and walks files of whole records, with a last record cut short at many lengths and with a malformed
|
||||||
it are indexed and the walk ends where they end, which is where `pb_init` cuts the file back to. No game files, no GPU.
|
record: the whole records before it are indexed and the walk ends where they end, `pb_keep` cuts the file back to
|
||||||
|
there, a second walk reaches the end of what is left, and a record appended after the cut is read back. No game files,
|
||||||
|
no GPU.
|
||||||
|
|
||||||
A change to the layout of `shader-prepared.bin` (`runtime/prp_file.h`, `runtime/gfx_shader.c`
|
A change to the layout of `shader-prepared.bin` (`runtime/prp_file.h`, `runtime/gfx_shader.c`
|
||||||
`gfx_shader_prepare_seed`, `tools/make_game.py` `prepared_from`) passes `python tests\prp_file_check.py`, which builds
|
`gfx_shader_prepare_seed`, `tools/make_game.py` `prepared_from`) passes `python tests\prp_file_check.py`, which builds
|
||||||
|
|||||||
@@ -336,6 +336,18 @@ static uint64_t pb_index(FILE *f, uint64_t len) {
|
|||||||
return good;
|
return good;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The length of the file at path (len bytes) to go on with after pb_index found its whole records ending at good: len
|
||||||
|
// when they fill it. When it ends inside a record (the process ended during an append, or a disk filled) or holds a
|
||||||
|
// malformed one, the records from there on are dropped and the file is cut back to good, so that appends go on after
|
||||||
|
// the whole records before it. 0 when it cannot be cut, and the file starts empty.
|
||||||
|
static uint64_t pb_keep(const char *path, uint64_t good, uint64_t len) {
|
||||||
|
if (good == len) return len;
|
||||||
|
if (!plat_file_truncate(path, good)) return 0;
|
||||||
|
rt_log("vk_pipeline: pipeline binaries: %s ends inside a record or holds a malformed one at byte %llu; %llu bytes "
|
||||||
|
"from there dropped", path, (unsigned long long)good, (unsigned long long)(len - good));
|
||||||
|
return good;
|
||||||
|
}
|
||||||
|
|
||||||
// Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache).
|
// Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache).
|
||||||
static void pb_init(VkCtx *c) {
|
static void pb_init(VkCtx *c) {
|
||||||
if (!c->has_pipeline_binary || !P.cache_path[0]) return;
|
if (!c->has_pipeline_binary || !P.cache_path[0]) return;
|
||||||
@@ -372,17 +384,13 @@ static void pb_init(VkCtx *c) {
|
|||||||
else good = pb_index(f, len);
|
else good = pb_index(f, len);
|
||||||
fclose(f);
|
fclose(f);
|
||||||
}
|
}
|
||||||
// A file that ends inside a record (the process ended during an append, or a disk filled): the whole records
|
// A file that ends inside a record (the process ended during an append, or a disk filled) or holds a malformed
|
||||||
// before it stay, and the file is cut back to them so that appends go on from there. Only when it cannot be cut
|
// record: the whole records before it stay, and the file is cut back to them so that appends go on from there.
|
||||||
// does it start empty.
|
// Only when it cannot be cut does it start empty.
|
||||||
if (f && !why && good != len) {
|
if (f && !why) {
|
||||||
if (plat_file_truncate(PB.path, good)) {
|
uint64_t kept = pb_keep(PB.path, good, len);
|
||||||
rt_log("vk_pipeline: pipeline binaries: %s was cut short (a record half written); %llu bytes after the last "
|
if (kept) len = kept;
|
||||||
"whole record dropped", PB.path, (unsigned long long)(len - good));
|
else why = "ends inside a record or holds a malformed one, and it cannot be cut back to its whole records";
|
||||||
len = good;
|
|
||||||
} else {
|
|
||||||
why = "cut short (a record half written), and it cannot be cut back to its whole records";
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (why || !f) { // start empty: the index is dropped and the file rewritten
|
if (why || !f) { // start empty: the index is dropped and the file rewritten
|
||||||
if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0);
|
if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0);
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
"""Check the index walk of pipeline-binaries.bin (runtime/vk_pipeline.c pb_index, NX2NAVI_PIPE_BINARIES=1). pb_init
|
"""Check how pipeline-binaries.bin keeps its whole records (runtime/vk_pipeline.c pb_index and pb_keep,
|
||||||
keeps the whole records of a file that ends inside one (the process ended during an append, or a disk filled) and
|
NX2NAVI_PIPE_BINARIES=1). pb_init keeps the whole records of a file that ends inside one (the process ended during an
|
||||||
cuts the file back to them; the walk must say where the last whole record ends. The check cuts the file's record
|
append, or a disk filled) or holds a malformed one, and cuts the file back to them; the walk must say where the last
|
||||||
layout and pb_hash, pb_find, pb_insert and pb_index out of vk_pipeline.c, writes files of whole records, one with a
|
whole record ends, and pb_keep must cut the file there. The check cuts the file's record layout and pb_hash, pb_find,
|
||||||
torn last record and one with a malformed record in the middle, and checks the offset and the index it gives.
|
pb_insert, pb_index and pb_keep out of vk_pipeline.c, writes files of whole records, one with a torn last record and
|
||||||
|
one with a malformed record in the middle, and checks the offset and the index the walk gives, the length pb_keep
|
||||||
|
leaves on the disk, that a second walk reaches the end of the cut file, and that a record appended after the cut is
|
||||||
|
read back. plat_file_truncate is a stand-in that cuts the file with the C library.
|
||||||
|
|
||||||
No GPU, no game files, any Python. Needs the Vulkan headers of MSYS2 UCRT64 (vulkan/vulkan.h).
|
No GPU, no game files, any Python. Needs the Vulkan headers of MSYS2 UCRT64 (vulkan/vulkan.h).
|
||||||
|
|
||||||
@@ -48,6 +51,30 @@ static uint64_t fnv(uint64_t h, const void *p, size_t n) {
|
|||||||
return h;
|
return h;
|
||||||
}
|
}
|
||||||
#define FNV0 0xcbf29ce484222325ull
|
#define FNV0 0xcbf29ce484222325ull
|
||||||
|
#ifdef _WIN32
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <io.h>
|
||||||
|
#else
|
||||||
|
#include <unistd.h>
|
||||||
|
#endif
|
||||||
|
static int logs, truncate_fails;
|
||||||
|
static char last_log[512];
|
||||||
|
void rt_log(const char *fmt, ...) {
|
||||||
|
logs++;
|
||||||
|
snprintf(last_log, sizeof last_log, "%s", fmt);
|
||||||
|
}
|
||||||
|
int plat_file_truncate(const char *path, uint64_t size) { /* platform.h's, with the C library */
|
||||||
|
if (truncate_fails) return 0;
|
||||||
|
#ifdef _WIN32
|
||||||
|
int fd = _open(path, _O_RDWR | _O_BINARY);
|
||||||
|
if (fd < 0) return 0;
|
||||||
|
int ok = _chsize_s(fd, (__int64)size) == 0;
|
||||||
|
_close(fd);
|
||||||
|
return ok;
|
||||||
|
#else
|
||||||
|
return truncate(path, (off_t)size) == 0;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
'''
|
'''
|
||||||
|
|
||||||
MAIN = r'''
|
MAIN = r'''
|
||||||
@@ -117,6 +144,56 @@ int main(int argc, char **argv) {
|
|||||||
fclose(f);
|
fclose(f);
|
||||||
assert(walk(path, &len) == (uint64_t)at && len > (uint64_t)at && PB.n == 1);
|
assert(walk(path, &len) == (uint64_t)at && len > (uint64_t)at && PB.n == 1);
|
||||||
|
|
||||||
|
/* pb_keep on a torn file: cut back to its whole records and logged; a second walk reaches the end of what is left,
|
||||||
|
and a record appended there is read back with the three. */
|
||||||
|
f = fopen(path, "wb");
|
||||||
|
fwrite(&hd, sizeof hd, 1, f);
|
||||||
|
put(f, PB_BLOB, 1, 1000), put(f, PB_BLOB, 2, 3), put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
|
||||||
|
put(f, PB_BLOB, 4, 500);
|
||||||
|
fclose(f);
|
||||||
|
uint64_t torn = whole + 4 + PB_HEAD + 500 - 77;
|
||||||
|
assert(plat_file_truncate(path, torn));
|
||||||
|
uint64_t good = walk(path, &len);
|
||||||
|
assert(len == torn && good == whole);
|
||||||
|
logs = 0;
|
||||||
|
assert(pb_keep(path, good, len) == whole && logs == 1 && strstr(last_log, "ends inside a record"));
|
||||||
|
assert(walk(path, &len) == len && len == whole && PB.n == 3);
|
||||||
|
f = fopen(path, "ab");
|
||||||
|
put(f, PB_BLOB, 5, 64);
|
||||||
|
fclose(f);
|
||||||
|
assert(walk(path, &len) == len && len == whole + 4 + PB_HEAD + 64 && PB.n == 4);
|
||||||
|
|
||||||
|
/* pb_keep on a malformed record in the middle: cut back to the record before it; the walk then reaches the end. */
|
||||||
|
f = fopen(path, "wb");
|
||||||
|
fwrite(&hd, sizeof hd, 1, f);
|
||||||
|
put(f, PB_BLOB, 1, 1000);
|
||||||
|
at = ftell(f);
|
||||||
|
put(f, 7, 2, 3);
|
||||||
|
put(f, PB_PIPE, 3, 4 + 16 + PB_KEY);
|
||||||
|
fclose(f);
|
||||||
|
good = walk(path, &len);
|
||||||
|
logs = 0;
|
||||||
|
assert(good == (uint64_t)at && pb_keep(path, good, len) == (uint64_t)at && logs == 1);
|
||||||
|
assert(walk(path, &len) == len && len == (uint64_t)at && PB.n == 1);
|
||||||
|
|
||||||
|
/* pb_keep on whole records: the file as it is, nothing logged. When the file cannot be cut: 0 (start empty), the
|
||||||
|
file untouched. */
|
||||||
|
f = fopen(path, "ab");
|
||||||
|
put(f, PB_BLOB, 2, 3);
|
||||||
|
fclose(f);
|
||||||
|
good = walk(path, &len);
|
||||||
|
logs = 0;
|
||||||
|
assert(good == len && pb_keep(path, good, len) == len && logs == 0);
|
||||||
|
f = fopen(path, "ab");
|
||||||
|
fputc(1, f);
|
||||||
|
fclose(f);
|
||||||
|
good = walk(path, &len);
|
||||||
|
truncate_fails = 1;
|
||||||
|
assert(good == len - 1 && pb_keep(path, good, len) == 0 && logs == 0);
|
||||||
|
truncate_fails = 0;
|
||||||
|
uint64_t len2;
|
||||||
|
assert(walk(path, &len2) == good && len2 == len);
|
||||||
|
|
||||||
/* The header alone: nothing indexed, the walk ends at the header. */
|
/* The header alone: nothing indexed, the walk ends at the header. */
|
||||||
f = fopen(path, "wb");
|
f = fopen(path, "wb");
|
||||||
fwrite(&hd, sizeof hd, 1, f);
|
fwrite(&hd, sizeof hd, 1, f);
|
||||||
@@ -124,7 +201,8 @@ int main(int argc, char **argv) {
|
|||||||
assert(walk(path, &len) == sizeof hd && len == sizeof hd && PB.n == 0);
|
assert(walk(path, &len) == sizeof hd && len == sizeof hd && PB.n == 0);
|
||||||
|
|
||||||
reset();
|
reset();
|
||||||
printf("pipeline binaries index: whole records kept, a torn last record and a malformed one end the walk there\n");
|
printf("pipeline binaries index: whole records kept, a torn last record and a malformed one end the walk there, "
|
||||||
|
"and the file is cut back to its whole records\n");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
'''
|
'''
|
||||||
|
|||||||
Reference in New Issue
Block a user