From c5cb67645ed9979fe99533b347a55a888785a5a6 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 22:14:32 -0500 Subject: [PATCH] Write the flight dump and the memory map through a temporary file and a rename. The window's close ends the process under the detached threads, so a dump file the flight recorder's thread was writing stayed cut short under its own name. Its three files are now written as .part and renamed when complete (runtime/part_file.h, with a check that kills a writer in the middle), and the allocator's map goes through plat_file_write_cached. The shader identities and the other cache files already went through an atomic write. Co-Authored-By: Claude Fable 5.2 --- docs/building.md | 8 ++++ runtime/flight.c | 16 ++++--- runtime/part_file.h | 41 ++++++++++++++++ runtime/vk_memory.c | 6 +-- tests/part_file_check.c | 100 +++++++++++++++++++++++++++++++++++++++ tests/part_file_check.py | 76 +++++++++++++++++++++++++++++ 6 files changed, 237 insertions(+), 10 deletions(-) create mode 100644 runtime/part_file.h create mode 100644 tests/part_file_check.c create mode 100644 tests/part_file_check.py diff --git a/docs/building.md b/docs/building.md index cbef4d2f..c78ab1b4 100644 --- a/docs/building.md +++ b/docs/building.md @@ -501,6 +501,14 @@ A change to what a host thread gives back when it ends (`runtime/platform/win32. sleep, read a file and end, half joined and half detached; the end hook runs once on each, and on Windows the process holds as many handles afterwards as before (each thread's sleep timer and read event are closed). No game files. +A file a detached thread writes piece by piece (the flight recorder's dump, `runtime/flight.c` `dump_main`) goes +through `runtime/part_file.h`: it is written as `.part` and renamed when complete, because the window's close +ends the process under such a thread and a file cut short there would read as a whole one. A file made in memory +first uses `plat_file_write_cached` or `plat_file_write_atomic`, which do the same in one call. A change to the +helper passes `python tests\part_file_check.py`, which builds it with the platform layer: nothing stands under the +name while a file is written, an older file stays until the new one is complete and after a write that was given up, +and a writer killed in the middle leaves the older file whole and the half as `.part`. No game files. + A change to the log queue (`runtime/logq.c`) passes `python tests\logq_check.py`, which builds it with the platform layer and a destination of its own: 160,000 lines from eight threads arrive whole and in each thread's order, a line logged while the queue's lock is taken waits and does not pass the lines queued before it, a destination that takes diff --git a/runtime/flight.c b/runtime/flight.c index 5f6a0f14..18de09bc 100644 --- a/runtime/flight.c +++ b/runtime/flight.c @@ -20,6 +20,7 @@ #include "gfx.h" #include "host.h" #include "input.h" +#include "part_file.h" #include "vk.h" #include "vk_exec.h" #include "platform/platform.h" @@ -200,11 +201,12 @@ static void copy_events(DumpJob *j, uint64_t first) { static const char *const tex_kind_names[FLIGHT_TEX_N] = {"new", "up", "reup", "release", "takeback", "drop", "bytes", "reg", "blank", "toblank", "blanksub", "direct", "unreg"}; -// /events.tsv: one line per texture event of the dump's frames (flight_tex). +// /events.tsv: one line per texture event of the dump's frames (flight_tex). Like the dump's other files it is +// written as a part file and renamed when complete (part_file.h): the game may close under the dump thread. static void write_events(const DumpJob *j) { char path[PLAT_PATH_MAX + 32]; snprintf(path, sizeof path, "%s/events.tsv", j->dir); - FILE *f = fopen(path, "w"); + FILE *f = part_file_open(path, "w"); if (!f) { rt_log("flight: cannot write %s", path); return; @@ -253,7 +255,7 @@ static void write_events(const DumpJob *j) { e->kind < FLIGHT_TEX_N ? tex_kind_names[e->kind] : "?", (unsigned long long)e->addr, e->format, e->w, e->h, e->layers, fl, detail, e->us); } - fclose(f); + if (!part_file_close(f, path, 1)) rt_log("flight: cannot write %s", path); } // ---------------------------------------------------------------- PNG (RGB, Paeth rows, fixed-Huffman deflate) @@ -369,7 +371,7 @@ static int write_png(const char *path, const uint8_t *rgba, PngBufs *pb) { } } size_t zn = deflate_fixed(pb->raw, (size_t)row * h, pb->z, pb->head); - FILE *f = fopen(path, "wb"); + FILE *f = part_file_open(path, "wb"); if (!f) return 0; static const uint8_t sig[8] = {0x89, 'P', 'N', 'G', 13, 10, 26, 10}; uint8_t ihdr[13] = {0, 0, (uint8_t)(w >> 8), (uint8_t)w, 0, 0, (uint8_t)(h >> 8), (uint8_t)h, 8, 2, 0, 0, 0}; @@ -377,7 +379,7 @@ static int write_png(const char *path, const uint8_t *rgba, PngBufs *pb) { chunk(f, "IHDR", ihdr, 13); chunk(f, "IDAT", pb->z, (uint32_t)zn); chunk(f, "IEND", NULL, 0); - return fclose(f) == 0; + return part_file_close(f, path, 1); } // ---------------------------------------------------------------- dump thread @@ -419,7 +421,7 @@ static void dump_main(void *arg) { } char path[PLAT_PATH_MAX + 32]; snprintf(path, sizeof path, "%s/frames.tsv", j->dir); - FILE *f = fopen(path, "w"); + FILE *f = part_file_open(path, "w"); if (!f) { rt_log("flight: cannot write %s", path); free(have), free(j->ev), free(j); @@ -439,7 +441,7 @@ static void dump_main(void *arg) { fprintf(f, "\t%.4f\t%.4f\t%llu", r->step_page, r->step_game, (unsigned long long)r->guest_presents); fprintf(f, "\t%d\n", have ? have[i] : 0); } - fclose(f); + if (!part_file_close(f, path, 1)) rt_log("flight: cannot write %s", path); rt_log("flight: dump %s: %s at frame %llu (%.1f ms), frames %llu-%llu, %d thumbnails, %zu texture events, written " "in %llu ms; recorder %.1f us per frame (max %.0f us)", j->dir, j->why, (unsigned long long)j->trig_frame, j->trig_interval_us / 1000.0, (unsigned long long)j->r[0].frame, diff --git a/runtime/part_file.h b/runtime/part_file.h new file mode 100644 index 00000000..f6c72e7b --- /dev/null +++ b/runtime/part_file.h @@ -0,0 +1,41 @@ +// A file written piece by piece on a thread nobody waits for when the process ends. +// +// The window's close ends the process with plat_exit while the detached threads still run (gfx_present.c +// gfx_window_closed), and a kill or a crash ends it anywhere. A file such a thread was in the middle of then stays +// cut short under its own name, and whoever opens it later takes it for a whole one. So the file is written as +// .part and gets its name with one rename once it is complete: a reader finds the whole file or none, and a +// .part file says what it is. A file made in memory first goes through plat_file_write_cached or +// plat_file_write_atomic instead, which do the same in one call. +#pragma once + +#include + +#include "platform/platform.h" + +#define PART_FILE_SUFFIX ".part" + +// .part into out; 0 when it does not fit. +static inline int part_file_name(const char *path, char *out, size_t cap) { + int n = snprintf(out, cap, "%s" PART_FILE_SUFFIX, path); + return n > 0 && (size_t)n < cap; +} + +// Opens .part for writing (mode: "w" or "wb"). NULL when it cannot be made. +static inline FILE *part_file_open(const char *path, const char *mode) { + char part[PLAT_PATH_MAX + 64]; + return part_file_name(path, part, sizeof part) ? fopen(part, mode) : NULL; +} + +// Closes f, the file part_file_open(path) gave, and renames it to path, over an older file of that name. With ok 0, +// after a write error on f or when the close or the rename fails, the part is deleted and path stays as it was. +// 1 when path now holds the whole new file. +static inline int part_file_close(FILE *f, const char *path, int ok) { + char part[PLAT_PATH_MAX + 64]; + if (!f) return 0; + ok = ok && !ferror(f); + ok = !fclose(f) && ok; + if (!part_file_name(path, part, sizeof part)) return 0; + if (ok && plat_file_replace(part, path, 0)) return 1; + plat_file_delete(part); + return 0; +} diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 48bb91ab..164a7b55 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -4497,9 +4497,9 @@ static void census_main(void *arg) { snprintf(path, sizeof path, "%.*svma-frame%llu.json", sl ? (int)(sl - lp) + 1 : 0, lp, (unsigned long long)j.frame); char *json = NULL; vmaBuildStatsString(j.c->vma, &json, VK_TRUE); - FILE *f = json ? fopen(path, "wb") : NULL; - int ok = f && fputs(json, f) >= 0; - if (f && fclose(f)) ok = 0; + // Through a temporary file and a rename: nobody waits for this thread when the game closes, and a map cut short + // there would read as a whole one + int ok = json && plat_file_write_cached(path, json, strlen(json)); rt_log("mem: vma detailed map %s %s", ok ? "written to" : "not written:", path); if (json) vmaFreeStatsString(j.c->vma, json); } diff --git a/tests/part_file_check.c b/tests/part_file_check.c new file mode 100644 index 00000000..f76c5942 --- /dev/null +++ b/tests/part_file_check.c @@ -0,0 +1,100 @@ +// runtime/part_file.h: a file written piece by piece gets its name only when it is complete. Built with the platform +// layer and run by tests/part_file_check.py. +// +// part_file_check the checks below, in +// part_file_check hang writes half of /killed.tsv, prints "writing" and sleeps: the Python side +// kills it there, as a closed game ends the thread that writes a dump +#include +#include +#include +#include + +#include "part_file.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +void rt_log(const char *fmt, ...) { // the platform layer's lines + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); +} + +// The text of the file at path ("" when it holds none), or NULL when there is no such file. +static char *text_of(const char *path) { + if (!plat_file_exists(path)) return NULL; + uint64_t size = 0; + char *t = (char *)plat_file_read(path, &size); + return t ? t : calloc(1, 1); +} + +static int holds(const char *path, const char *want) { + char *t = text_of(path); + int same = t && !strcmp(t, want); + free(t); + return same; +} + +int main(int argc, char **argv) { + if (argc < 2) return 2; + char path[PLAT_PATH_MAX], part[PLAT_PATH_MAX + 64]; + if (argc > 2 && !strcmp(argv[2], "hang")) { + snprintf(path, sizeof path, "%s/killed.tsv", argv[1]); + FILE *f = part_file_open(path, "wb"); + if (!f) return 3; + fputs("the first half of the new file\n", f); + fflush(f); + puts("writing"); + fflush(stdout); + for (;;) plat_sleep_ms(1000); + } + + // a new file: nothing under its name while it is written, the whole file after + snprintf(path, sizeof path, "%s/new.tsv", argv[1]); + CHECK(part_file_name(path, part, sizeof part), "the part name fits"); + FILE *f = part_file_open(path, "wb"); + CHECK(f != NULL, "the part file opens"); + if (!f) return 1; + fputs("line 1\n", f); + fflush(f); + CHECK(!plat_file_exists(path), "no file under the name while it is written"); + CHECK(holds(part, "line 1\n"), "what was written so far is in the part file"); + fputs("line 2\n", f); + CHECK(part_file_close(f, path, 1) == 1, "the complete file gets its name"); + CHECK(holds(path, "line 1\nline 2\n"), "the file holds everything written"); + CHECK(!plat_file_exists(part), "no part file stays"); + + // over an older file: the old one stays whole until the new one is complete + f = part_file_open(path, "wb"); + CHECK(f != NULL, "the part file opens beside the older file"); + if (!f) return 1; + fputs("the new text\n", f); + fflush(f); + CHECK(holds(path, "line 1\nline 2\n"), "the older file is untouched while the new one is written"); + CHECK(part_file_close(f, path, 1) == 1, "the new file replaces the older one"); + CHECK(holds(path, "the new text\n"), "the name holds the new file"); + + // given up (the writer met an error): the older file stays, the part goes + f = part_file_open(path, "wb"); + if (!f) return 1; + fputs("half of a third", f); + CHECK(part_file_close(f, path, 0) == 0, "a file given up is not put in place"); + CHECK(holds(path, "the new text\n"), "the older file stays after a write that was given up"); + CHECK(!plat_file_exists(part), "the part of a write that was given up is deleted"); + + // a folder that is not there: nothing opens, and closing nothing is no success + snprintf(path, sizeof path, "%s/no such folder/x.tsv", argv[1]); + f = part_file_open(path, "wb"); + CHECK(f == NULL, "no part file in a folder that does not exist"); + CHECK(part_file_close(f, path, 1) == 0, "closing a file that never opened reports failure"); + + // a name that does not fit the path buffers + static char longp[PLAT_PATH_MAX + 200]; + memset(longp, 'a', sizeof longp - 1); + CHECK(part_file_open(longp, "wb") == NULL, "a path too long for its part name is refused"); + + printf(fails ? "part_file_check: %d FAILED\n" : "part_file_check: all checks passed\n", fails); + return fails != 0; +} diff --git a/tests/part_file_check.py b/tests/part_file_check.py new file mode 100644 index 00000000..0275730d --- /dev/null +++ b/tests/part_file_check.py @@ -0,0 +1,76 @@ +"""Build and run tests/part_file_check.c: runtime/part_file.h, which the runtime's detached threads write their files +through (the flight recorder's dump). A file gets its name only when it is complete, so a game that closes under the +writing thread leaves the older file or none, never one cut short. + +The C side checks the helper itself. This side then kills a process in the middle of a write, as the window's close +ends the dump thread, and looks at what it left: the older file whole under its name, the half in the .part file. +No game files, no GPU. + +Usage: part_file_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'part_file' + shutil.rmtree(work / 'files', ignore_errors=True) + (work / 'files').mkdir(parents=True) + win = os.name == 'nt' + exe = work / ('part_file_check.exe' if win else 'part_file_check') + env = dict(os.environ) + if win and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + rt = ROOT / 'runtime' + plat = [rt / 'platform' / ('win32.c' if win else 'linux.c'), rt / 'platform' / 'common.c'] + libs = ['-lbcrypt', '-lws2_32', '-lpsapi', '-lwinmm', '-ldbghelp', '-lshell32', '-lole32', '-luser32', '-ladvapi32'] if win \ + else ['-lpthread'] + subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-fno-strict-aliasing', + '-march=x86-64-v3', '-I', str(rt), + str(ROOT / 'tests' / 'part_file_check.c'), *map(str, plat), '-o', str(exe), '-lm', *libs], + env=env) + files = work / 'files' + code = subprocess.run([str(exe), str(files)], env=env, timeout=120).returncode + if code: + return code + + # a writer killed in the middle: the file of the dump before it is whole, the half is named as a part + killed = files / 'killed.tsv' + killed.write_text('the whole older file\n', encoding='utf-8') + p = subprocess.Popen([str(exe), str(files), 'hang'], env=env, stdout=subprocess.PIPE, text=True) + try: + said = p.stdout.readline().strip() + finally: + p.kill() + p.wait() + p.stdout.close() + fails = [] + if said != 'writing': + fails.append(f'the writer did not start ({said!r})') + if killed.read_text(encoding='utf-8') != 'the whole older file\n': + fails.append('the older file changed although the new one was never complete') + part = killed.with_name('killed.tsv.part') + if not part.is_file() or part.read_text(encoding='utf-8') != 'the first half of the new file\n': + fails.append('the half that was written is not in killed.tsv.part') + for f in fails: + print('FAIL:', f) + print('part_file_check: the killed writer left %s' % ('a wrong file' if fails else 'the older file whole')) + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main())