diff --git a/docs/building.md b/docs/building.md index 551641af..861a099e 100644 --- a/docs/building.md +++ b/docs/building.md @@ -405,6 +405,11 @@ value no machine has free. What the builders do when they are refused (`warm_bui `lw_worker`) shows in a run's log: start the game with `build_free_mb` set above the machine's free commit and read the `vk_pipeline:` lines that name the commit headroom. +A change to the start's memory warning (`runtime/mem_need.h`, `runtime/memory.c` `mem_start_note`) passes +`python tests\mem_need_check.py []`, which builds the header alone and checks when the game's need counts +as short. With the executable it also runs `--probe-memneed` (it prints the machine's commit and the answer and +starts nothing) with `mem_need_mb` at 0, at 1 and at a value no machine has, and reads the warning line of the last. + A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice.h`, `runtime/vk_memory.c` `vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and its reason by platform, by GPU driver and by what the player set. No game files, no GPU. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index fc27b247..a704de2e 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -738,6 +738,11 @@ # Each walk holds the address-space lock for 100-240 ms and delays the next frame by as much, so they are off unless # asked for. # mem_census = 0 +# The memory the game is taken to need, in MB of commit (RAM plus page file not yet promised to a program). When less +# is free at the start, the log gets a warning line: short of memory the game stutters, loses textures or stops with +# an out-of-memory or lost-device error. The default is what the game took in play at 3840x2160 on an RTX 5090; the +# video card's memory counts on Windows. 0: no warning. Default: 24576. +# mem_need_mb = 24576 # The size in MiB (16 to 256) of the blocks the renderer takes video memory in. A block goes back to the driver only once # everything in it is freed, so smaller blocks hold less free space after area changes; larger ones need fewer # allocations while an area loads. Default: 64. diff --git a/runtime/cfg.c b/runtime/cfg.c index de35f51c..b5493226 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -51,7 +51,7 @@ static const char *const known[] = { "HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE", "HDR_HUE_CLIP", "HDR_LIVE", "HEAP_LAZY", "HDR_MATCH_SLOPE", "HDR_MIDGRAY", "HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_TONE_SLOPE", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", - "BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "BUILD_FREE_MB", "BUILD_RESERVE_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "OQ_BATCH", "GATE_WAIT_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST", + "BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "BUILD_FREE_MB", "BUILD_RESERVE_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "OQ_BATCH", "GATE_WAIT_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MEM_NEED_MB", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOUSE_TRACE", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", @@ -126,7 +126,7 @@ static const struct { const char *key, *section; } sections[] = { {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, - {"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, + {"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, {"MEM_NEED_MB", "diagnostics"}, {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, {"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, @@ -283,6 +283,7 @@ static const CfgMeta metas[] = { {"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, {"PIPE_BINARIES", "0", "0|1", RESTART}, {"ONESHOT_WAIT_MS", "30000", "0..60000", RESTART}, {"ONESHOT_BUDGET", "1", "0|1", RESTART}, {"PIPE_BINARIES_MAX_MB", "1024", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART}, {"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART}, {"BUILD_FREE_MB", "4096", "0..1048576", RESTART}, {"BUILD_RESERVE_MB", "64", "0..65536", RESTART}, // vk_pipeline.c gate_enter + {"MEM_NEED_MB", "24576", "0..1048576", RESTART}, // memory.c mem_start_note; the default is mem_need.h MEM_NEED_DEFAULT_MB // all|: the compute pipelines the boot keeps, those dispatched most recently (vk_pipeline.c, compute-used.bin) {"COMPUTE_KEEP", "512", NULL, RESTART}, {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again diff --git a/runtime/host.h b/runtime/host.h index 54a7f015..5c29f73d 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -191,6 +191,8 @@ typedef struct { uint64_t base, size; uint32_t state, attr, perm, ipc, device, p void mem_init(void); // reserve every region; on a collision re-run the process // (up to 20 times, NX2NAVI_REEXEC), then fatal const RtGuestLayout *mem_layout(void); +int mem_start_note(int probe); // a game's start: a warning line when the free commit is + // below the game's need (mem_need.h); 1 when it is void mem_guest_census(uint64_t frame); // NX2NAVI_TRACE=perf: committed bytes per guest region void mem_lazy_heap_report(uint64_t frame); // NX2NAVI_TRACE=perf: the lazy heap's faults since the last line uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr); // 0x01 diff --git a/runtime/main.c b/runtime/main.c index 42e21f79..5517868e 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -534,7 +534,7 @@ static int usage(const char *argv0) { "--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " "--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | --probe-coop | " - "--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-buildgate | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0); + "--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-buildgate | --probe-memneed | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0); return 2; } @@ -649,6 +649,7 @@ int main(int argc, char **argv) { if (cfg.seconds && mode) return usage(argv[0]); log_open(!mode); if (!mode) { + mem_start_note(0); // memory.c: a warning when the machine has less memory free than the game takes cfg.boot = cfg.seconds > 0; // --seconds starts the game like --boot, with its checkpoints vk_driver_cache_env(); // before the threads below and rt_boot's, any of which may load the Vulkan driver discord_init(); // the Rich Presence thread (discord_rpc.c); it reads nx2navi.ini itself and idles while off @@ -695,6 +696,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--prepare-pipelines")) return vk_pipeline_prepare_pipelines(); if (!strcmp(mode, "--probe-prewarm")) return vk_pipeline_probe_prewarm(); if (!strcmp(mode, "--probe-buildgate")) return vk_pipeline_probe_buildgate(); - if (!strcmp(mode, "--boot")) { cfg.boot = 1; rt_boot(); return 0; } + if (!strcmp(mode, "--probe-memneed")) { mem_start_note(1); return 0; } + if (!strcmp(mode, "--boot")) { mem_start_note(0); cfg.boot = 1; rt_boot(); return 0; } return usage(argv[0]); } diff --git a/runtime/mem_need.h b/runtime/mem_need.h new file mode 100644 index 00000000..6d68d62e --- /dev/null +++ b/runtime/mem_need.h @@ -0,0 +1,25 @@ +// Whether the machine has the memory the game needs, asked once at the start (memory.c mem_start_note). Own header so +// tests/mem_need_check.c checks it alone. +// +// What counts is commit: memory the system has promised to programs, out of RAM plus page file (on Linux the tightest +// of the limits platform.h plat_commit_info reads). An allocation fails when the commit in use reaches the limit, +// however much RAM is free, and on Windows the video driver charges the game's video memory to it too. Measured in +// play (RTX 5090, 3840x2160, three sessions of 19,000 to 50,000 frames on 2026-11-11): this process held 23,910 to +// 24,429 MB of commit, flat over the session, and the system stood at 65,777 to 65,914 of 73,154 MB with the other +// programs' 41,500 MB. A machine that cannot give the game that much pages, drops textures to make room (vk_texture.c +// commit_is_low) or loses the device. +#pragma once + +// NX2NAVI_MEM_NEED_MB's default: the commit the game took in the sessions above, as 24 GiB. +#define MEM_NEED_DEFAULT_MB 24576.0 + +// The commit in MB the game can still get: what is free, and what this process already holds of its need. +static inline double mem_need_room(double proc_mb, double used_mb, double limit_mb) { + return limit_mb - used_mb + (proc_mb > 0 ? proc_mb : 0); +} + +// 1 when the game's need does not fit. need_mb 0 or less: not asked. A limit below 0 is an unknown one: not short. +static inline int mem_need_short(double proc_mb, double used_mb, double limit_mb, double need_mb) { + if (need_mb <= 0 || limit_mb < 0) return 0; + return mem_need_room(proc_mb, used_mb, limit_mb) < need_mb; +} diff --git a/runtime/memory.c b/runtime/memory.c index e871c6bb..a9f4b2f8 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -14,6 +14,7 @@ #include "cfg.h" #include "host.h" #include "layout.h" +#include "mem_need.h" #include "platform/platform.h" #define PAGE 0x1000ull @@ -951,6 +952,29 @@ void mem_init(void) { const RtGuestLayout *mem_layout(void) { return &L; } +// The start of a game (main.c, before the boot): one line when the machine's free commit is below what the game takes +// in play (mem_need.h; NX2NAVI_MEM_NEED_MB, 0 no check), so a log that ends in a failed allocation, dropped textures +// or a lost device says at its top that the machine was short before the game took anything. 1 when it was. +// probe: --probe-memneed, which prints the numbers either way. +int mem_start_note(int probe) { + double proc, used, limit, need = strtod(cfg_value("NX2NAVI_MEM_NEED_MB"), NULL); + plat_commit_info(&proc, &used, &limit); + int is_short = mem_need_short(proc, used, limit, need); + if (probe) + printf("probe-memneed: system commit %.0f of %.0f MB in use, this process %.0f MB, the game needs %.0f MB " + "(NX2NAVI_MEM_NEED_MB): %s\n", used, limit, proc, need, + need <= 0 ? "not checked" : limit < 0 ? "no limit known" : is_short ? "short" : "enough"); + if (!is_short) return 0; + char note[400] = ""; + plat_commit_note(note, sizeof note); + rt_log("mem: WARNING: this machine has %.0f MB of memory left for the game and the game takes about %.0f MB in play " + "(system commit: %.0f of %.0f MB in use; the limit is RAM plus page file%s%s). Short of memory the game " + "stutters, loses textures or stops with an out-of-memory or lost-device error. Closing other programs or a " + "larger page file gives it room (mem_need_mb = 0 in nx2navi.ini turns this warning off)", + mem_need_room(proc, used, limit), need, used, limit, note[0] ? "; " : "", note); + return 1; +} + // A range inside one watched reservation (stack and heap block, or alias region) only. static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, int reset) { if (!max || !size) return 0; diff --git a/tests/mem_need_check.c b/tests/mem_need_check.c new file mode 100644 index 00000000..e5de8f82 --- /dev/null +++ b/tests/mem_need_check.c @@ -0,0 +1,42 @@ +// runtime/mem_need.h: whether the machine has the commit the game needs (runtime/memory.c mem_start_note). Built and +// run by tests/mem_need_check.py; prints each failed check and exits 1 on any. +#include + +#include "mem_need.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +int main(void) { + const double need = MEM_NEED_DEFAULT_MB; + + // The machine the need was measured on, before the game starts: 41,515 of 73,154 MB in use, the process at 369 MB. + // 32,008 MB for the game: enough. + CHECK(!mem_need_short(369, 41515, 73154, need), "short with 32 GB for the game"); + CHECK(mem_need_room(369, 41515, 73154) == 73154 - 41515 + 369, "room %.0f", mem_need_room(369, 41515, 73154)); + + // The same machine with the game in play, 65,840 of 73,154 MB in use and 24,000 MB of them the game's: what the + // game holds counts toward its need, so a check made late does not call a fitting game short. + CHECK(!mem_need_short(24000, 65840, 73154, need), "short in play although the game's own commit fits"); + + // A 32 GB machine with an 8 GB page file and 12 GB in use by other programs: 28 GB free, enough; with 20 GB in + // use, 20 GB free: short. + CHECK(!mem_need_short(300, 12000, 40960, need), "short with 29 GB free"); + CHECK(mem_need_short(300, 20000, 40960, need), "enough with 21 GB free"); + + // A 16 GB machine with a 4 GB page file never fits. + CHECK(mem_need_short(300, 6000, 20480, need), "enough on a 16 GB machine"); + + // The boundary: exactly the need is enough, one MB less is short. + CHECK(!mem_need_short(0, 1000, 1000 + need, need), "short with exactly the need free"); + CHECK(mem_need_short(0, 1001, 1000 + need, need), "enough one MB below the need"); + + // Not asked (0), and a platform without a known limit (-1): never short. A process reading of -1 (unknown) adds + // nothing to the room. + CHECK(!mem_need_short(300, 20000, 20480, 0), "short with the check off"); + CHECK(!mem_need_short(300, 20000, -1, need), "short without a known limit"); + CHECK(mem_need_room(-1, 20000, 40960) == 20960, "an unknown process commit changed the room"); + + printf("mem_need_check: %s\n", fails ? "FAILED" : "passed"); + return fails ? 1 : 0; +} diff --git a/tests/mem_need_check.py b/tests/mem_need_check.py new file mode 100644 index 00000000..e384d2cb --- /dev/null +++ b/tests/mem_need_check.py @@ -0,0 +1,67 @@ +"""Build and run tests/mem_need_check.c: runtime/mem_need.h, whether the machine has the commit the game needs. + +With the game executable's path it also runs the executable's --probe-memneed (runtime/memory.c mem_start_note: it +reads this machine's commit, prints the answer and starts nothing) three times: with mem_need_mb = 0 (not checked), +with 1 (any machine has it) and with a value no machine has, where the log's warning line must appear. + +Usage: mem_need_check.py [PATH_TO_GAME_EXE] +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests' / 'diff')) +try: + from common import MSYS_BIN, compiler # noqa: E402 +except ImportError: # a Python without the differential tests' packages: the same compiler choice without them + MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + def compiler(): + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'mem_need' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('mem_need_check.exe' if os.name == 'nt' else 'mem_need_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'mem_need_check.c'), '-o', str(exe)], env=env) + rc = subprocess.run([str(exe)], env=env).returncode + if len(sys.argv) > 1: + rc |= probe(sys.argv[1]) + return rc + + +def probe(game): + """The executable's --probe-memneed under three settings; 0 when each answers as it must.""" + fails = 0 + for need, answer, warns in (('0', 'not checked', False), ('1', 'enough', False), ('1048576', 'short', True)): + env = dict(os.environ, NX2NAVI_MEM_NEED_MB=need) + r = subprocess.run([game, '--probe-memneed'], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, errors='replace') + lines = r.stdout.splitlines() + said = [ln for ln in lines if ln.startswith('probe-memneed:')] + warned = [ln for ln in lines if 'mem: WARNING: this machine has' in ln] + print('\n'.join(said + [ln[:160] + ' ...' for ln in warned])) + # without a commit limit (a Linux machine with none) nothing is short: the forced case cannot be shown there + if said and said[0].endswith('no limit known'): + continue + if r.returncode or len(said) != 1 or not said[0].endswith(': ' + answer) or bool(warned) != warns: + fails += 1 + print('mem_need_check: FAIL: --probe-memneed with mem_need_mb = %s (exit code %d)' % (need, r.returncode)) + print('mem_need_check: the executable\'s answer %s' % ('FAILED' if fails else 'passed')) + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main())