From 7432f7375ce2c2c1255c7ad3ad4a2c66f723e290 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Sat, 14 Nov 2026 08:53:26 -0500 Subject: [PATCH] Ask for the C runtime heap's summary before play only. The memory report's system line asked for the heap's summary at every report until one was slow. The summary holds the heap's lock, and every malloc of every thread with it, while it adds the heap up: 54 and 70 ms in two sessions on an RTX 5090 PC, and the frame of the first report, 30 seconds into each session, took 55 to 82 ms, the longest frame of play in both. One slow summary switched the question off, so every session paid for it once. It is now asked for before play only (the boot and the title screen), and at every report with mem_census = 1, which already accepts a late frame for the walk of the address space. In play the line gives what the runtime knows it allocated, the same figure from one report to the next. The rule is in runtime/vk_mem_heap.h with tests/mem_heap_check, which fails against the rule before. Co-Authored-By: Claude Fable 5.2 --- docs/releases/v0.10.2.md | 4 ++++ nx2navi.ini.example | 4 +++- runtime/vk_mem_heap.h | 17 ++++++++++++++ runtime/vk_memory.c | 12 ++++++---- tests/mem_heap_check.c | 50 ++++++++++++++++++++++++++++++++++++++++ tests/mem_heap_check.py | 41 ++++++++++++++++++++++++++++++++ 6 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 runtime/vk_mem_heap.h create mode 100644 tests/mem_heap_check.c create mode 100644 tests/mem_heap_check.py diff --git a/docs/releases/v0.10.2.md b/docs/releases/v0.10.2.md index c69fc2c3..cf6a5934 100644 --- a/docs/releases/v0.10.2.md +++ b/docs/releases/v0.10.2.md @@ -79,6 +79,10 @@ cards). frame as the scene after a later one had been named, and the upscaler then took that image for one frame. At one place in Kakariko with both settings as a tester had them, 51 and 52 frames flashed in two runs of 100 seconds, and none with the fix. Both settings stay off by default. +- **One long frame about 30 seconds into every session** (55 to 82 ms, the longest frame of play in the two sessions + it was found in): the first memory report asked Windows for a summary of the game's heap, and that summary holds + every allocation of the game while it is taken. It is now taken before play only (`mem_census = 1` asks for it in + play too, for a diagnosis). - **Crashes and hangs found in a review of the whole runtime**; none was seen in our own play, and each fix has a check of its own: - a crash when the game read memory the lazy write-back was just handing to the GPU (NVIDIA and Intel cards); diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 468ef455..a11be565 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -1104,7 +1104,9 @@ # stands in for it) and where its committed memory lies every 3600 frames, and when the process's own commit (without # the video memory Windows charges to it) rose 1 GiB over the highest it was, at most once a minute (the "commit # census" lines). Each walk holds the address-space lock for 100-260 ms and delays the next frame by as much, so they -# are off unless asked for: leave it off in a run whose smoothness is judged. +# are off unless asked for: leave it off in a run whose smoothness is judged. With or without the trace, 1 also has +# every memory report ask for the C runtime heap's summary in play, which holds every allocation of the game for as +# long as it takes (50 to 70 ms with a heap of a few GB); without it the summary is taken before play only. # mem_census = 0 # 1: with trace = perf, the process's handles counted by kind (events, sections, threads, files...) at the 600-frame # report when their number grew by 16 or more since the last count (the "mem: handles" line). It reads the system's diff --git a/runtime/vk_mem_heap.h b/runtime/vk_mem_heap.h new file mode 100644 index 00000000..96ae5f9c --- /dev/null +++ b/runtime/vk_mem_heap.h @@ -0,0 +1,17 @@ +// When the memory report asks for the C runtime heap's summary (vk_memory.c system_log, plat_heap_info). Header only, +// so that tests/mem_heap_check.c checks the rule without a device. +// +// The summary holds the heap's lock while it adds the heap up, and every malloc and free of every thread waits for +// it. With a heap of a few GiB that took 54 and 70 ms in two sessions (2026-11-14, RTX 5090 PC), and the frame of the +// first report, 30 seconds into each session, took 55 to 82 ms: the longest frame of play in both. One slow summary +// switched the question off for the rest of the run, so each session paid for it exactly once. +// +// The summary is therefore not asked for in play at all: only before play (the boot and the title screen, where a +// long frame is not seen and the heap is small), and no more once one was slow there. NX2NAVI_MEM_CENSUS=1, the +// switch that already accepts a late frame for the walk of the address space, asks for it at every report. In play +// the report gives what the runtime knows it allocated instead, the same figure from one report to the next, which is +// what the growth bookkeeping compares. +#pragma once + +// census: NX2NAVI_MEM_CENSUS is on; in_play: a save's first frame was shown; slow: a summary took too long before. +static inline int mem_heap_summary_due(int census, int in_play, int slow) { return census || (!in_play && !slow); } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 20bf11f5..d04dd043 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -27,6 +27,7 @@ #include "vk_dlaa.h" #include "vk_exec.h" #include "vk_gpa.h" +#include "vk_mem_heap.h" #include "wb_lazy_choice.h" #include "wb_merge_spv.h" #include "platform/platform.h" @@ -5755,15 +5756,16 @@ static void system_log(uint64_t frame, const ReportFig *f, const VkPipeMem *pm, const double M = 1048576.0; uint64_t res, priv; plat_process_memory(&res, &priv); - // The C runtime heap's summary holds the heap's lock while it adds up: asked at the report only, and no more once - // one was slow, so that a heap grown past what the summary handles quickly does not hold every malloc up again. - static int heap_off; + // The C runtime heap's summary holds the heap's lock while it adds up, and every malloc with it: asked before play + // only (and with NX2NAVI_MEM_CENSUS at every report), and no more once one was slow (vk_mem_heap.h). + static int heap_off, heap_census = -1; + if (heap_census < 0) heap_census = cfg_bool("NX2NAVI_MEM_CENSUS"); PlatHeapInfo hi = {0}; - if (!heap_off) { + if (mem_heap_summary_due(heap_census, frame_step_world_us() >= 0, heap_off)) { uint64_t t0 = plat_now_ns(); plat_heap_info(&hi); uint64_t took = plat_now_ns() - t0; - if (took > HEAP_SLOW_NS) + if (took > HEAP_SLOW_NS && !heap_census) heap_off = 1, rt_log("mem: the C runtime heap's summary took %.1f ms; it is not asked for again in this run", took / 1e6); } diff --git a/tests/mem_heap_check.c b/tests/mem_heap_check.c new file mode 100644 index 00000000..467fd698 --- /dev/null +++ b/tests/mem_heap_check.c @@ -0,0 +1,50 @@ +// runtime/vk_mem_heap.h: when the memory report asks for the C runtime heap's summary, which holds every allocation of +// the game while it runs. Never in play, unless NX2NAVI_MEM_CENSUS asks for it. Built and run by +// tests/mem_heap_check.py; prints each failed check and exits 1 on any. +#include + +#include "vk_mem_heap.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +// A session as the report sees it: a report every 3600 presents, play from `play_at` on; the summary takes `ms` +// milliseconds each time and counts as slow past 2 ms (vk_memory.c HEAP_SLOW_NS). Returns the summaries taken in play, +// and through *before those before it. +static int session(int census, int reports, int play_at, double ms, int *before) { + int slow = 0, in_play_n = 0; + *before = 0; + for (int r = 1; r <= reports; r++) { + int in_play = r * 3600 >= play_at; + if (!mem_heap_summary_due(census, in_play, slow)) continue; + if (in_play) in_play_n++; + else ++*before; + if (ms > 2.0 && !census) slow = 1; // as system_log notes it + } + return in_play_n; +} + +int main(void) { + // ---- the rule + CHECK(mem_heap_summary_due(0, 0, 0), "before play the summary is not asked for"); + CHECK(!mem_heap_summary_due(0, 1, 0), "the summary is asked for in play"); + CHECK(!mem_heap_summary_due(0, 0, 1) && !mem_heap_summary_due(0, 1, 1), "a slow summary is asked for again"); + CHECK(mem_heap_summary_due(1, 1, 0) && mem_heap_summary_due(1, 1, 1) && mem_heap_summary_due(1, 0, 1), + "the census switch does not ask at every report"); + + int before; + // ---- the sessions of 2026-11-14: play from the first seconds on, the first report at present 3600 in play, the + // summary 54 and 70 ms. The rule before asked there once (the hitch); now it asks never. + CHECK(session(0, 6, 2400, 70.0, &before) == 0 && before == 0, "a session that plays before its first report asks"); + // ---- a long title screen: two reports before play, each fast there, then play: asked twice, both before play + CHECK(session(0, 10, 9000, 0.4, &before) == 0 && before == 2, "a title screen of two reports: %d before play", before); + // ---- a slow summary before play (a heap that is large already at the title screen): once, and no more + CHECK(session(0, 10, 20000, 30.0, &before) == 0 && before == 1, "a slow summary before play: %d taken", before); + // ---- with the census switch every report asks, in play too, slow or not + CHECK(session(1, 6, 2400, 70.0, &before) == 6 && before == 0, "the census switch: not every report in play asks"); + CHECK(session(1, 6, 9000, 70.0, &before) == 4 && before == 2, "the census switch across a title screen"); + + if (fails) printf("%d checks failed\n", fails); + else printf("mem_heap_check: ok\n"); + return fails != 0; +} diff --git a/tests/mem_heap_check.py b/tests/mem_heap_check.py new file mode 100644 index 00000000..add39a19 --- /dev/null +++ b/tests/mem_heap_check.py @@ -0,0 +1,41 @@ +"""Build and run tests/mem_heap_check.c: runtime/vk_mem_heap.h, when the memory report asks for the C runtime heap's +summary (vk_memory.c system_log): never in play, where it held every allocation of the game for 54 to 70 ms at the +first report of a session, unless mem_census asks for it. + +No GPU, no game files, any Python. + +Usage: mem_heap_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn).""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'mem_heap' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('mem_heap_check.exe' if os.name == 'nt' else 'mem_heap_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'mem_heap_check.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main())