diff --git a/docs/releases/v0.10.2.md b/docs/releases/v0.10.2.md index c69fc2c3..cf6a5934 100644 --- a/docs/releases/v0.10.2.md +++ b/docs/releases/v0.10.2.md @@ -79,6 +79,10 @@ cards). frame as the scene after a later one had been named, and the upscaler then took that image for one frame. At one place in Kakariko with both settings as a tester had them, 51 and 52 frames flashed in two runs of 100 seconds, and none with the fix. Both settings stay off by default. +- **One long frame about 30 seconds into every session** (55 to 82 ms, the longest frame of play in the two sessions + it was found in): the first memory report asked Windows for a summary of the game's heap, and that summary holds + every allocation of the game while it is taken. It is now taken before play only (`mem_census = 1` asks for it in + play too, for a diagnosis). - **Crashes and hangs found in a review of the whole runtime**; none was seen in our own play, and each fix has a check of its own: - a crash when the game read memory the lazy write-back was just handing to the GPU (NVIDIA and Intel cards); diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 468ef455..a11be565 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -1104,7 +1104,9 @@ # stands in for it) and where its committed memory lies every 3600 frames, and when the process's own commit (without # the video memory Windows charges to it) rose 1 GiB over the highest it was, at most once a minute (the "commit # census" lines). Each walk holds the address-space lock for 100-260 ms and delays the next frame by as much, so they -# are off unless asked for: leave it off in a run whose smoothness is judged. +# are off unless asked for: leave it off in a run whose smoothness is judged. With or without the trace, 1 also has +# every memory report ask for the C runtime heap's summary in play, which holds every allocation of the game for as +# long as it takes (50 to 70 ms with a heap of a few GB); without it the summary is taken before play only. # mem_census = 0 # 1: with trace = perf, the process's handles counted by kind (events, sections, threads, files...) at the 600-frame # report when their number grew by 16 or more since the last count (the "mem: handles" line). It reads the system's diff --git a/runtime/vk_mem_heap.h b/runtime/vk_mem_heap.h new file mode 100644 index 00000000..96ae5f9c --- /dev/null +++ b/runtime/vk_mem_heap.h @@ -0,0 +1,17 @@ +// When the memory report asks for the C runtime heap's summary (vk_memory.c system_log, plat_heap_info). Header only, +// so that tests/mem_heap_check.c checks the rule without a device. +// +// The summary holds the heap's lock while it adds the heap up, and every malloc and free of every thread waits for +// it. With a heap of a few GiB that took 54 and 70 ms in two sessions (2026-11-14, RTX 5090 PC), and the frame of the +// first report, 30 seconds into each session, took 55 to 82 ms: the longest frame of play in both. One slow summary +// switched the question off for the rest of the run, so each session paid for it exactly once. +// +// The summary is therefore not asked for in play at all: only before play (the boot and the title screen, where a +// long frame is not seen and the heap is small), and no more once one was slow there. NX2NAVI_MEM_CENSUS=1, the +// switch that already accepts a late frame for the walk of the address space, asks for it at every report. In play +// the report gives what the runtime knows it allocated instead, the same figure from one report to the next, which is +// what the growth bookkeeping compares. +#pragma once + +// census: NX2NAVI_MEM_CENSUS is on; in_play: a save's first frame was shown; slow: a summary took too long before. +static inline int mem_heap_summary_due(int census, int in_play, int slow) { return census || (!in_play && !slow); } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 20bf11f5..d04dd043 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -27,6 +27,7 @@ #include "vk_dlaa.h" #include "vk_exec.h" #include "vk_gpa.h" +#include "vk_mem_heap.h" #include "wb_lazy_choice.h" #include "wb_merge_spv.h" #include "platform/platform.h" @@ -5755,15 +5756,16 @@ static void system_log(uint64_t frame, const ReportFig *f, const VkPipeMem *pm, const double M = 1048576.0; uint64_t res, priv; plat_process_memory(&res, &priv); - // The C runtime heap's summary holds the heap's lock while it adds up: asked at the report only, and no more once - // one was slow, so that a heap grown past what the summary handles quickly does not hold every malloc up again. - static int heap_off; + // The C runtime heap's summary holds the heap's lock while it adds up, and every malloc with it: asked before play + // only (and with NX2NAVI_MEM_CENSUS at every report), and no more once one was slow (vk_mem_heap.h). + static int heap_off, heap_census = -1; + if (heap_census < 0) heap_census = cfg_bool("NX2NAVI_MEM_CENSUS"); PlatHeapInfo hi = {0}; - if (!heap_off) { + if (mem_heap_summary_due(heap_census, frame_step_world_us() >= 0, heap_off)) { uint64_t t0 = plat_now_ns(); plat_heap_info(&hi); uint64_t took = plat_now_ns() - t0; - if (took > HEAP_SLOW_NS) + if (took > HEAP_SLOW_NS && !heap_census) heap_off = 1, rt_log("mem: the C runtime heap's summary took %.1f ms; it is not asked for again in this run", took / 1e6); } diff --git a/tests/mem_heap_check.c b/tests/mem_heap_check.c new file mode 100644 index 00000000..467fd698 --- /dev/null +++ b/tests/mem_heap_check.c @@ -0,0 +1,50 @@ +// runtime/vk_mem_heap.h: when the memory report asks for the C runtime heap's summary, which holds every allocation of +// the game while it runs. Never in play, unless NX2NAVI_MEM_CENSUS asks for it. Built and run by +// tests/mem_heap_check.py; prints each failed check and exits 1 on any. +#include + +#include "vk_mem_heap.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +// A session as the report sees it: a report every 3600 presents, play from `play_at` on; the summary takes `ms` +// milliseconds each time and counts as slow past 2 ms (vk_memory.c HEAP_SLOW_NS). Returns the summaries taken in play, +// and through *before those before it. +static int session(int census, int reports, int play_at, double ms, int *before) { + int slow = 0, in_play_n = 0; + *before = 0; + for (int r = 1; r <= reports; r++) { + int in_play = r * 3600 >= play_at; + if (!mem_heap_summary_due(census, in_play, slow)) continue; + if (in_play) in_play_n++; + else ++*before; + if (ms > 2.0 && !census) slow = 1; // as system_log notes it + } + return in_play_n; +} + +int main(void) { + // ---- the rule + CHECK(mem_heap_summary_due(0, 0, 0), "before play the summary is not asked for"); + CHECK(!mem_heap_summary_due(0, 1, 0), "the summary is asked for in play"); + CHECK(!mem_heap_summary_due(0, 0, 1) && !mem_heap_summary_due(0, 1, 1), "a slow summary is asked for again"); + CHECK(mem_heap_summary_due(1, 1, 0) && mem_heap_summary_due(1, 1, 1) && mem_heap_summary_due(1, 0, 1), + "the census switch does not ask at every report"); + + int before; + // ---- the sessions of 2026-11-14: play from the first seconds on, the first report at present 3600 in play, the + // summary 54 and 70 ms. The rule before asked there once (the hitch); now it asks never. + CHECK(session(0, 6, 2400, 70.0, &before) == 0 && before == 0, "a session that plays before its first report asks"); + // ---- a long title screen: two reports before play, each fast there, then play: asked twice, both before play + CHECK(session(0, 10, 9000, 0.4, &before) == 0 && before == 2, "a title screen of two reports: %d before play", before); + // ---- a slow summary before play (a heap that is large already at the title screen): once, and no more + CHECK(session(0, 10, 20000, 30.0, &before) == 0 && before == 1, "a slow summary before play: %d taken", before); + // ---- with the census switch every report asks, in play too, slow or not + CHECK(session(1, 6, 2400, 70.0, &before) == 6 && before == 0, "the census switch: not every report in play asks"); + CHECK(session(1, 6, 9000, 70.0, &before) == 4 && before == 2, "the census switch across a title screen"); + + if (fails) printf("%d checks failed\n", fails); + else printf("mem_heap_check: ok\n"); + return fails != 0; +} diff --git a/tests/mem_heap_check.py b/tests/mem_heap_check.py new file mode 100644 index 00000000..add39a19 --- /dev/null +++ b/tests/mem_heap_check.py @@ -0,0 +1,41 @@ +"""Build and run tests/mem_heap_check.c: runtime/vk_mem_heap.h, when the memory report asks for the C runtime heap's +summary (vk_memory.c system_log): never in play, where it held every allocation of the game for 54 to 70 ms at the +first report of a session, unless mem_census asks for it. + +No GPU, no game files, any Python. + +Usage: mem_heap_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn).""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'mem_heap' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('mem_heap_check.exe' if os.name == 'nt' else 'mem_heap_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'mem_heap_check.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main())