From d48a3d59f8b6b74211b4f4261e749444e7b9e32a Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 14:57:01 -0500 Subject: [PATCH 01/15] End a helper run with its own exit code when a library faults while the process is ending. The shader preparation modes and the probes return from main, and the C runtime's exit then calls the detach of every library in the process, those other programs put there included. On a player's PC a post-processing injector's Vulkan layer read a null pointer there at the end of every --warm-shaders and --prepare-shaders, four runs over three versions: each printed "variants built into the driver cache", then a FATAL, and ended with code 71, so the launcher logged a finished preparation as one that did not finish. The crash handler now ends such a process with the code its mode returned and one "exit:" line; a fault while the mode still runs, or in the executable's own code on the way out, is reported as before. A crash inside that injector's library also gets the note that the library is not part of the game. --probe-crash-exit makes the fault from an atexit handler, and tests/exit_fault_check.py checks both cases. Co-Authored-By: Claude Fable 5.2 --- docs/building.md | 7 +++++ runtime/main.c | 51 +++++++++++++++++++++++++++---- tests/exit_fault_check.py | 63 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 tests/exit_fault_check.py diff --git a/docs/building.md b/docs/building.md index c6c0e9d8..1f138b3a 100644 --- a/docs/building.md +++ b/docs/building.md @@ -409,6 +409,13 @@ loss at the same moment end the process with exit code 72, one device report and device and opens no window; the device is not really lost, so what a driver answers after a loss is not covered. `device_lost_at = ` in `nx2navi.ini` ends a real run the same way at its n-th hand-over to the GPU queue. +A change to how a fault ends the process (`runtime/main.c` `crash_handler`, `mode_done`) passes +`python tests\exit_fault_check.py `, which runs two crash probes: a fault inside a library while a mode runs +is a crash (one FATAL line, exit code 71), and the same fault after the mode has returned, while the process is ending, +leaves the mode's own exit code and one `exit:` line. The second is what a library another program put into the +process does when it faults in its detach: an injector's Vulkan layer did so at the end of every shader preparation on +a player's PC, and the launcher took each finished preparation for a failed one. No game files, no GPU. + A change to the memory gate of the background pipeline builds (`runtime/vk_build_gate.h`, `runtime/vk_pipeline.c` `gate_enter`) passes `python tests\build_gate_check.py []`, which builds the header alone and checks what it admits and refuses and that builds under way count through their reservations. With the executable it also runs diff --git a/runtime/main.c b/runtime/main.c index a6e67e07..18901c62 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -264,20 +264,39 @@ static uint64_t lifted_function(uintptr_t rip) { static int readable(uint64_t a, uint64_t n) { return plat_mem_readable(a, n); } -// A module another program put into the game's process: the Vulkan layer or the hook of a recorder, an overlay or a -// capture tool, by the words such files carry in their names. A crash inside one is that program's: one report was an -// access violation in a recorder's layer the moment the present mode changed and the swapchain was made again. +// A module another program put into the game's process: the Vulkan layer or the hook of a recorder, an overlay, a +// capture tool or a post-processing injector, by the words such files carry in their names. A crash inside one is that +// program's: one report was an access violation in a recorder's layer the moment the present mode changed and the +// swapchain was made again, another an injector's layer reading a null pointer at the end of every shader +// preparation. static int foreign_hook(const char *module) { char low[160]; size_t n = 0; for (; module && module[n] && n < sizeof low - 1; n++) low[n] = (char)tolower((unsigned char)module[n]); low[n] = 0; - return strstr(low, "vk_layer") || strstr(low, "hook") || strstr(low, "overlay"); + return strstr(low, "vk_layer") || strstr(low, "hook") || strstr(low, "overlay") || strstr(low, "reshade"); } +// Set when main_run has returned: the mode's work is done and saved, and all that still runs is the C runtime's exit +// and, inside ExitProcess, the detach call of every library in the process, the graphics driver's and those other +// programs put there. A fault in one of them is not a failure of the run. On one PC an injector's Vulkan layer faulted +// in its detach at the end of every `--warm-shaders` and `--prepare-shaders` (four runs over three versions): each +// wrote a FATAL after "variants built into the driver cache" and ended with code 71, so the launcher took a finished +// preparation for one that failed. crash_handler ends such a process with the code its mode returned. A fault in this +// executable's own code on the way out (an atexit handler) is still reported. +static volatile int32_t mode_done; +static int mode_rc; + static void crash_handler(const PlatCrash *x) { static volatile int32_t busy; if (plat_xchg32(&busy, 1)) plat_exit(71); + if (__atomic_load_n(&mode_done, __ATOMIC_ACQUIRE) && x->module) { + rt_log("exit: %s (%#x) in %s while the process was ending, after its work was done and saved%s: the run ends " + "with its own code %d", x->what, x->code, *x->module ? x->module : "code outside every module", + foreign_hook(x->module) ? " (that library is not part of the game: another program put it there)" : "", + mode_rc); + plat_exit(mode_rc); + } rt_log_sync(); // the lines before the crash, then the report, each written as it is made char d[160]; uintptr_t rip = x->rip; @@ -536,11 +555,23 @@ static int probe_crash(int null) { return 1; } +// --probe-crash-exit: the fault of --probe-crash-call, made by an atexit handler after the mode has returned, the way +// a library another program put into the process faults in its detach. The process must end with the mode's own code +// (PROBE_EXIT_CODE) and an "exit:" line, with no FATAL (mode_done, crash_handler). tests/exit_fault_check.py runs it. +#define PROBE_EXIT_CODE 3 +static void probe_exit_fault(void) { printf("%zu\n", probe_strlen((const char *)(uintptr_t)8)); } + +static int probe_crash_exit(void) { + atexit(probe_exit_fault); + printf("probe-crash-exit: returning %d; a library then faults while the process ends\n", PROBE_EXIT_CODE); + return PROBE_EXIT_CODE; +} + static int usage(const char *argv0) { fprintf(stderr, "usage: %s [--exefs DIR] [--romfs DIR] [--profile DIR] [--audio-device VALUE] [--monitor VALUE] [--list-audio-devices | --list-monitors | --selftest | --probe-loader | " "--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " - "--probe-vkexec | --probe-meshfb | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | --probe-coop | " + "--probe-vkexec | --probe-meshfb | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-crash-exit | --probe-discord | --probe-coop | " "--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-buildgate | --probe-ffx-cache | --probe-memneed | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0); return 2; } @@ -577,7 +608,16 @@ static void portable_paths(RtConfig *c) { c->profile = portable_path(exe, "NX2NAVI_PROFILE_DIR", "profile", NULL, pp_profile, sizeof pp_profile, c->profile); } +static int main_run(int argc, char **argv); + +// A game run never comes back here: it ends in plat_exit (boot.c, gfx_present.c). Every other mode returns its code. int main(int argc, char **argv) { + mode_rc = main_run(argc, argv); + __atomic_store_n(&mode_done, 1, __ATOMIC_RELEASE); // from here on only the exit runs (mode_done, above) + return mode_rc; +} + +static int main_run(int argc, char **argv) { mem_init(); // first: claim the guest address space before anything else can land there; reads nx2navi.ini // The build that wrote the log, at its top: an attached log from another install or version shows at a glance. rt_log("version: NX2navi %s, game %d.%d.%d", rt_port_version, rt_game_version / 100, rt_game_version / 10 % 10, @@ -671,6 +711,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--probe-loader")) return probe_loader(); if (!strcmp(mode, "--probe-crash-call")) return probe_crash(0); if (!strcmp(mode, "--probe-crash-null")) return probe_crash(1); + if (!strcmp(mode, "--probe-crash-exit")) return probe_crash_exit(); if (!strcmp(mode, "--probe-cfg")) return probe_cfg(); #ifdef _WIN32 if (!strcmp(mode, "--probe-dpi")) return probe_dpi(); // the Win32 DPI awareness of nx2navi.manifest diff --git a/tests/exit_fault_check.py b/tests/exit_fault_check.py new file mode 100644 index 00000000..2d0093a0 --- /dev/null +++ b/tests/exit_fault_check.py @@ -0,0 +1,63 @@ +"""Run the game executable's crash probes and check what a fault does to the way the process ends (runtime/main.c +crash_handler, mode_done). + + - --probe-crash-call: a fault inside a library while the mode still runs is a crash: a FATAL line that names the + library, exit code 71. + - --probe-crash-exit: the same fault after the mode has returned, while the process is ending (an atexit handler here; + on a player's PC the detach of a library another program put into the process). The work is done by then, so the + process ends with the mode's own code, 3, and one "exit:" line, with no FATAL. + +No game files, no GPU, no window. + +Usage: exit_fault_check.py PATH_TO_GAME_EXE +""" +import subprocess +import sys + +CRASH_EXIT = 71 +PROBE_EXIT = 3 + + +def run(exe, mode): + p = subprocess.run([exe, mode], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors='replace', + timeout=120) + return p.returncode, p.stdout.splitlines() + + +def main(): + if len(sys.argv) != 2: + print(__doc__) + return 2 + fails = [] + + def check(ok, what): + if not ok: + fails.append(what) + + code, lines = run(sys.argv[1], '--probe-crash-call') + fatal = [ln for ln in lines if ln.startswith('FATAL: ')] + check(code == CRASH_EXIT, '--probe-crash-call: exit code %d, expected %d' % (code, CRASH_EXIT)) + check(len(fatal) == 1 and 'access violation' in fatal[0], '--probe-crash-call: %d FATAL lines, expected one: %s' + % (len(fatal), fatal)) + check(not any('exit: ' in ln and 'while the process was ending' in ln for ln in lines), + '--probe-crash-call: a fault inside the mode was taken for one at the exit') + + code, lines = run(sys.argv[1], '--probe-crash-exit') + fatal = [ln for ln in lines if ln.startswith('FATAL: ')] + said = [ln for ln in lines if 'exit: ' in ln and 'while the process was ending' in ln] + check(any(ln.startswith('probe-crash-exit: returning') for ln in lines), '--probe-crash-exit: the mode did not run') + check(code == PROBE_EXIT, '--probe-crash-exit: exit code %d, expected the mode\'s own %d' % (code, PROBE_EXIT)) + check(not fatal, '--probe-crash-exit: a FATAL line for a fault after the work was done: %s' % fatal) + check(len(said) == 1 and 'with its own code %d' % PROBE_EXIT in said[0], + '--probe-crash-exit: %d "exit:" lines, expected one that names the code: %s' % (len(said), said)) + + for f in fails: + print('exit_fault_check: FAIL:', f) + if not fails: + print('exit_fault_check: passed (a fault in the mode: FATAL and %d; a fault at the exit: the mode\'s code %d)' + % (CRASH_EXIT, PROBE_EXIT)) + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main()) From cacf458b0809fe82c4efcebab2ee31dd6cf822ba Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 14:57:21 -0500 Subject: [PATCH 02/15] Say in a lost device's report what the GPU was about to do when it stopped between two marks, and add draw marks. A pass mark names the pass or dispatch the GPU had started and the last it finished. When the two are the same mark the GPU stopped in between, and the report named nothing: in a player's log of 0.9.2 (RX 6700 XT) both were one compute dispatch, with 51 later marks made and none started. The report now adds the mark that never started and the kinds of commands the replay recorded between the two (copies, clears, the upscaler, write-backs), which the gap notes already name for the GPU pass timing and now keep for every mark. draw_crumbs = 1 (off by default, AMD buffer marks only) also writes a mark wherever the shaders or the pipeline state change inside a pass. The device losses reported on RX 6600 and RX 6800 class cards end inside one scene pass of 30 to 100 draws and the draw that hangs is not known; with the setting on, the report lists the shaders of the draws the GPU had not finished. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 5 ++ runtime/cfg.c | 5 +- runtime/vk.h | 9 +++- runtime/vk_crumbs.c | 112 +++++++++++++++++++++++++++++++++++++++-- runtime/vk_exec_draw.c | 2 + 5 files changed, 126 insertions(+), 7 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index cae47bd6..579f560b 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -845,6 +845,11 @@ # last one it finished. Unset: on AMD GPUs (VK_AMD_buffer_marker). 1: also on NVIDIA GPUs # (VK_NV_device_diagnostic_checkpoints), which NVIDIA says costs a little. 0: off. # pass_crumbs = 1 +# GPU draw marks, for a GPU crash (device lost) that comes back in the same place on an AMD GPU: inside each render pass +# a mark is written wherever the shaders change, and after the crash the log names the shaders of the draws the GPU had +# not finished, not only the pass. It costs a little frame rate, so it is off unless a crash is being tracked down. +# Needs the pass marks above (AMD GPUs). Default: 0. +# draw_crumbs = 1 # A test of how the game ends when the video card stops answering (device lost): the hand-over to the GPU queue with # this number is not made and is taken as one the driver refused with a lost device. The log then holds one report and # one FATAL line, and the game ends with exit code 72. 0: never. Default: 0. diff --git a/runtime/cfg.c b/runtime/cfg.c index ac6d7b94..ac223d57 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -44,7 +44,7 @@ static const char *const known[] = { "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CACHE_DIE", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COMPUTE_KEEP", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", - "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", + "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DRAW_CRUMBS", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "GPU_TIMELINE", "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", @@ -150,7 +150,7 @@ static const struct { const char *key, *section; } sections[] = { {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"}, - {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, + {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"DRAW_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"MESH_FALLBACK_WORKERS", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"MIRROR_CAPTURE_KB", "diagnostics"}, {"WB_SKIP_MB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"WAIT_WATCH_S", "diagnostics"}, @@ -354,6 +354,7 @@ static const CfgMeta metas[] = { {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC1", "1", "0|1", RESTART}, // vk_astc.c bc1_on {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only + {"DRAW_CRUMBS", "0", "0|1", RESTART}, // vk_crumbs.c draw marks: a mark at each change of shaders inside a pass {"STAGING_FAIL_AT", "0", "0..1000000000", RESTART}, // vk_memory.c staging_fail_now: a test of give_back {"VRAM_BALLAST_MB", "0", "0..131072", RESTART}, // vk_memory.c vk_mem_ballast: a test of a smaller card {"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end diff --git a/runtime/vk.h b/runtime/vk.h index f0b3cbbe..7e20fafc 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -148,12 +148,14 @@ typedef struct { const GfxShader *first[2], *last[2];// vertex and fragment shader of the first and last draw; a dispatch: first[0] uint32_t gp; // NX2NAVI_GPU_PASSES: the timestamp pair of this mark plus one; 0 none uint32_t layer; // NX2NAVI_GPU_PASSES: the first layer of the depth target (a shadow cascade) + uint32_t gap; // bits of VK_GAP_*: what the replay recorded between the mark before and this one } VkPassMark; void vk_gpu_passes_frame(uint64_t frame); // vk_crumbs.c, each present: NX2NAVI_GPU_PASSES, GPU time per pass and dispatch // NX2NAVI_GPU_PASSES also times what a part of a submission spends outside its passes and dispatches (vk_crumbs.c, "the // gap ledger"). vk_gpu_passes_part: vk_sync.c, at the start (end 0) and the end (end 1) of each part's command buffer. // vk_gpu_gap_note: the replay is about to record commands of this kind outside a pass or dispatch; units is their size -// (bytes for buffers, texels for images, 0 unknown). Both do nothing outside the timed frames. Replay thread only. +// (bytes for buffers, texels for images, 0 unknown). Outside the timed frames the first does nothing and the second +// only notes the kind for the next pass mark (VkPassMark.gap, a lost device's report). Replay thread only. enum { VK_GAP_COPY_B2B, VK_GAP_COPY_B2T, VK_GAP_COPY_T2B, VK_GAP_COPY_T2T, VK_GAP_CLEAR_BUF, VK_GAP_CLEAR_TEX, VK_GAP_CLEAR_TARGET, VK_GAP_UPSCALER, VK_GAP_WB_MERGE, VK_GAP_WB_COPY, VK_GAP_TWIN, VK_GAP_POOL_MARKS, @@ -184,6 +186,11 @@ const VkPassMark *vk_pass_mark_record(uint32_t id); // the ring record of mark // the extension they came from), then each with its ring record. vk_pass_marks_report logs it; the probe checks it. void vk_pass_marks_format(const char *source, uint32_t started, uint32_t finished, char *o, size_t size); void vk_pass_marks_report(void); // vk_device.c device_fault_report: the AMD words and the NVIDIA checkpoints, formatted +// Draw marks (vk_crumbs.c, NX2NAVI_DRAW_CRUMBS=1 with VK_AMD_buffer_marker; off by default): a mark where the shaders +// or the pipeline state change inside pass m, before the draw with index draw in that pass, so a lost device's report +// names the shaders of the draws the GPU had not finished. Replay thread only; nothing while vk_draw_marks_on is 0. +extern int vk_draw_marks_on; +void vk_draw_mark(VkCommandBuffer cb, const VkPassMark *m, const GfxShader *vs, const GfxShader *fs, uint32_t draw); // ---- window and present (gfx_present.c). Host threads only. // vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c index 110355aa..6cd57fbd 100644 --- a/runtime/vk_crumbs.c +++ b/runtime/vk_crumbs.c @@ -26,7 +26,7 @@ static PFN_vkCmdWriteBufferMarkerAMD write_marker; static VkBuffer buf; static VmaAllocation alloc; -static volatile uint32_t *words; // 0, 1: crumbs; 2, 3: pass marks +static volatile uint32_t *words; // 0, 1: crumbs; 2, 3: pass marks; 4, 5: draw marks // Pass marks (below) int vk_pass_marks_on; @@ -41,7 +41,7 @@ static volatile uint8_t pt_timed[VK_PASS_MARKS]; // the slot's mark writes tim static void buffer_init(VkCtx *c) { write_marker = (PFN_vkCmdWriteBufferMarkerAMD)vk_dpa(c->device, "vkCmdWriteBufferMarkerAMD"); - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 16, + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 24, .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, @@ -53,7 +53,7 @@ static void buffer_init(VkCtx *c) { return; } words = info.pMappedData; - words[0] = words[1] = words[2] = words[3] = 0; + words[0] = words[1] = words[2] = words[3] = words[4] = words[5] = 0; rt_log("vk: GPU breadcrumbs on (VK_AMD_buffer_marker)"); } @@ -69,7 +69,7 @@ void vk_crumbs_release(void) { if (pt_pool && c) vkDestroyQueryPool(c->device, pt_pool, NULL); pt_pool = VK_NULL_HANDLE; buf = VK_NULL_HANDLE, alloc = NULL, words = NULL, write_marker = NULL; - vk_pass_marks_on = pm_amd = 0, set_checkpoint = NULL; + vk_pass_marks_on = pm_amd = vk_draw_marks_on = 0, set_checkpoint = NULL; } void vk_crumb(VkCommandBuffer cb, int kind, uint64_t n) { @@ -132,10 +132,44 @@ static void pass_marks_init(VkCtx *c) { else rt_log("vk: pass marks off (%s)", off ? "NX2NAVI_PASS_CRUMBS=0" : "no VK_AMD_buffer_marker; NX2NAVI_PASS_CRUMBS=1 asks for NVIDIA checkpoints"); + vk_draw_marks_on = pm_amd && cfg_bool("NX2NAVI_DRAW_CRUMBS"); + if (vk_draw_marks_on) + rt_log("vk: draw marks on (NX2NAVI_DRAW_CRUMBS): a mark at every change of shaders inside a pass, so a lost " + "device names the shaders of the draws the GPU had not finished"); + else if (cfg_bool("NX2NAVI_DRAW_CRUMBS")) + rt_log("vk: draw marks off: NX2NAVI_DRAW_CRUMBS needs the pass marks of VK_AMD_buffer_marker"); } +// ---- draw marks (NX2NAVI_DRAW_CRUMBS=1, off by default; with VK_AMD_buffer_marker only) +// A pass mark names the pass the GPU was in and the shaders of its first and last draw. The losses reported on RDNA 2 +// cards end inside one scene pass of 30 to 100 draws, and which draw the GPU never finished is not known. With this +// on, the replay writes a mark each time the shaders or the pipeline state change inside a pass (vk_exec_draw.c +// prepare_draw_ex), into words 4 and 5 of the breadcrumb buffer as a pass mark goes into words 2 and 3, and a ring +// keeps the pass, the draw's index in it and the two shaders of each. After a loss word 5 names the last mark before +// which every draw had finished: the draw that hung was recorded after it, with its shaders or those of one of the few +// marks up to the one in word 4. It costs two marker writes per change, a few thousand a frame, so it is for the +// player who can reproduce a loss, not for everyone. +#define VK_DRAW_MARKS 8192 // power of two +typedef struct { uint32_t id, pass, draw; const GfxShader *vs, *fs; } DrawMark; +static DrawMark dring[VK_DRAW_MARKS]; +static uint32_t dseq; // draw marks made; only the replay thread writes it +int vk_draw_marks_on; + +void vk_draw_mark(VkCommandBuffer cb, const VkPassMark *m, const GfxShader *vs, const GfxShader *fs, uint32_t draw) { + if (!vk_draw_marks_on || !cb) return; + uint32_t id = ++dseq; + if (!id) id = ++dseq; // 0 means no mark + dring[id & (VK_DRAW_MARKS - 1)] = (DrawMark){id, m ? m->id : 0, draw, vs, fs}; + rec_pass_mark(cb, id, 2); +} void vk_pass_mark_cmd(VkCommandBuffer cb, uint32_t id, int end) { + if (end == 2) { // a draw mark (vk_draw_mark): the same two points, into its own words + if (!pm_amd) return; + write_marker(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, buf, 16, id); + write_marker(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, buf, 20, id); + return; + } if (pt_pool && pt_timed[id & (VK_PASS_MARKS - 1)] == 1) vkCmdWriteTimestamp(cb, end ? VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, pt_pool, (id & (VK_PASS_MARKS - 1)) * 2 + (end != 0)); @@ -238,7 +272,12 @@ static void gp_gap_close(uint32_t to) { gp_gap_from = UINT32_MAX, gp_gap_kinds = 0; } +// The kinds of commands noted since the last mark ended (bits of VK_GAP_*), for a lost device's report: the next mark +// takes them as the commands recorded before it (VkPassMark.gap). Replay thread only, as the notes are. +static uint32_t gap_now; + void vk_gpu_gap_note(int kind, uint64_t units) { + if (kind >= 0 && kind < VK_GAP_N) gap_now |= 1u << kind; if (!gp_capture || gp_part == UINT32_MAX || kind < 0 || kind >= VK_GAP_N) return; gp_gap_kinds |= 1u << kind; gp_kind_n[kind]++, gp_kind_units[kind] += units; @@ -571,6 +610,7 @@ VkPassMark *vk_pass_mark_begin(VkCommandBuffer cb, int kind, uint64_t fence) { if (!id) id = ++seq; // 0 means no mark VkPassMark *m = &ring[id & (VK_PASS_MARKS - 1)]; m->id = id, m->kind = (uint8_t)kind, m->fence = fence, m->draws = UINT32_MAX; + m->gap = gap_now, gap_now = 0; // what was recorded between the mark before and this one // timed unless the slot's previous mark is still waiting to be read (its queries are not reset yet) if (pt_pool) pt_timed[id & (VK_PASS_MARKS - 1)] = !pt_timed[id & (VK_PASS_MARKS - 1)] ? 1 : 2; rec_pass_mark(cb, id, 0); @@ -584,6 +624,7 @@ void vk_pass_mark_end(VkCommandBuffer cb, const VkPassMark *m) { gp_in_mark = 0; gp_end(cb, m); rec_pass_mark(cb, m->id, 1); + gap_now = 0; // the gap after this mark starts empty } int vk_pass_marks_read(uint32_t *started, uint32_t *finished) { @@ -703,6 +744,67 @@ static void nv_report(void) { log_lines(t); } +// The kinds of bits (VK_GAP_*) as text; "barriers and state only" when none was noted. +static void gap_text(uint32_t kinds, char *o, size_t size) { + size_t n = 0; + o[0] = 0; + for (int b = 0; b < VK_GAP_N && n < size; b++) + if (kinds >> b & 1) n += (size_t)snprintf(o + n, size - n, "%s%s", n ? ", " : "", gp_kind_names[b]); + if (!n) snprintf(o, size, "barriers and state only"); +} + +// A GPU that finished the last mark it started stopped between two marks, where the two lines of +// vk_pass_marks_format name nothing: in a player's log both were one compute dispatch, with 51 later marks made and +// none of them started. This adds the mark that never started and what the replay recorded between the two (the gap +// notes: copies, clears, the upscaler, write-backs), or what it recorded after the last mark of all. +static void report_between(uint32_t started, uint32_t finished) { + if (!finished || started != finished) return; + char what[512], next[640]; + const VkPassMark *m = vk_pass_mark_record(finished + 1); + if (m) { + describe_mark(finished + 1, next, sizeof next); + gap_text(m->gap, what, sizeof what); + rt_log("vk: the GPU stopped after mark %u and before the next one, which it never started: %s", (unsigned)finished, + next); + rt_log("vk: recorded between the two: %s", what); + } else if (finished == seq) { + gap_text(gap_now, what, sizeof what); + rt_log("vk: the GPU stopped after mark %u, the last one made; recorded since: %s", (unsigned)finished, what); + } else { + rt_log("vk: the GPU stopped after mark %u; the record of the mark after it is gone (a later mark took its slot)", + (unsigned)finished); + } +} + +static void describe_draw_mark(uint32_t id, char *o, size_t size) { + const DrawMark *d = &dring[id & (VK_DRAW_MARKS - 1)]; + if (!id) snprintf(o, size, "nothing yet"); + else if (d->id != id) snprintf(o, size, "no record (a later mark took its slot)"); + else + snprintf(o, size, "before draw %u of pass mark %u: vs %016llx fs %016llx", d->draw, d->pass, shader_id(d->vs), + shader_id(d->fs)); +} + +// Words 4 and 5 (vk_draw_mark): the draws the GPU had not finished were recorded after the mark in word 5, so its +// shaders and those of the marks up to the one in word 4 are the candidates; up to DRAW_MARK_LINES of them are listed. +#define DRAW_MARK_LINES 12 +static void draw_marks_report(void) { + if (!vk_draw_marks_on || !words) return; + uint32_t reached = words[4], finished = words[5]; + char t[256]; + rt_log("vk: draw marks (NX2NAVI_DRAW_CRUMBS, %u made): the GPU last reached draw mark %u, and every draw before draw " + "mark %u had finished; the draws it had not finished follow these marks:", (unsigned)dseq, (unsigned)reached, + (unsigned)finished); + uint32_t n = reached >= finished ? reached - finished + 1 : 1; + if (n > DRAW_MARK_LINES) n = DRAW_MARK_LINES; + for (uint32_t i = 0; i < n; i++) { + describe_draw_mark(finished + i, t, sizeof t); + rt_log("vk: draw mark %u: %s", (unsigned)(finished + i), t); + } + if (reached >= finished && reached - finished + 1 > n) + rt_log("vk: and %u more up to draw mark %u", (unsigned)(reached - finished + 1 - n), (unsigned)reached); +} + void vk_pass_marks_report(void) { if (!vk_pass_marks_on) return; uint32_t started, finished; @@ -710,6 +812,8 @@ void vk_pass_marks_report(void) { if (vk_pass_marks_read(&started, &finished)) { vk_pass_marks_format("VK_AMD_buffer_marker", started, finished, t, sizeof t); log_lines(t); + report_between(started, finished); + draw_marks_report(); } nv_report(); } diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 1c743a08..3caadc02 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -4588,6 +4588,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int ex->recorded++; cx->B.st = s, cx->B.st_valid = 1; if (ex->mark && vk_exec_frame.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; + if (vk_draw_marks_on) // NX2NAVI_DRAW_CRUMBS: a lost device then names these shaders (vk_crumbs.c vk_draw_mark) + vk_draw_mark(ex->cb, ex->mark, vs, fs, (uint32_t)(vk_exec_frame.draws - ex->mark_draws0)); } vk_pipeline_line_width(ex->cb, &s, ex->line_width); // a draw of lines: the game's width; else nothing if (!mesh) { From 677eb5c2429c11b8ea7fe96d469bc8a7423336e3 Mon Sep 17 00:00:00 2001 From: Vasilis Manetas Date: Thu, 12 Nov 2026 19:50:35 +0100 Subject: [PATCH 03/15] Keep an area load's texture staging inside the staging pool's blocks made at boot, instead of making about 1.1 GiB of new host memory in the load frame. The staging pool keeps 4 blocks of 64 MiB from its first use on a discrete GPU (2 on an integrated one, as before). A staging buffer or arena chunk that does not fit them takes the room of the kept staging buffers and idle arena chunks, then of the oldest upload batches in flight, which it waits for; an arena chunk a waited-for batch gives back is taken instead of a new one. Only when that makes no room (the open batch's own staging, staging buffers their callers hold, the blocks' fragmentation) does the pool grow by a block, and the log says so. The outermost vk_upload_end submits the open batch once it holds 64 MiB of staging, besides the NX2NAVI_UPLOAD_BATCH_MB limit; nothing is submitted or waited for once the device is lost. Entering Hyrule Field made 221 staging buffers (1,131 MiB) in one frame; on Linux each new 64 MiB block is a kernel allocation that tries 2-4 MiB pages first, and with the free 2 MiB blocks used up the kernel compacted memory inside those allocations: 30-360 ms each, and the load frame took up to 4.7 s instead of about 0.3 s (RX 6800 XT, amdgpu, 32 GB). --- runtime/vk_memory.c | 119 ++++++++++++++++++++++++++++++++++++------- runtime/vk_texture.c | 5 +- 2 files changed, 103 insertions(+), 21 deletions(-) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 2d2f6c48..ac28225b 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -4507,6 +4507,17 @@ static void free_staging(VkCtx *c, Batch *b) { b->bytes = 0; } +// An upload batch holding this much staging (its buffers and arena chunks) is submitted at the outermost vk_upload_end, +// whatever NX2NAVI_UPLOAD_BATCH_MB says (the staging pool's comment, below). +#define STG_FLUSH (64ull << 20) + +// cmd_lock held: the staging bytes of batch b (its buffers and arena chunks). +static uint64_t batch_staging(const Batch *b) { + uint64_t n = (uint64_t)b->nchunk * ARENA_CHUNK; + for (int i = 0; i < b->nstg; i++) n += b->stg[i].size; + return n; +} + // The cached staging buffer that fits n with the least waste: at most four times the request, so a small upload does not // take a block-sized buffer another upload wanted. static int staging_cache_take(VkCtx *c, VkDeviceSize n, Staging *out, void **map) { @@ -4676,9 +4687,11 @@ VkCommandBuffer vk_upload_begin(void) { return open_b.cb; } -// NX2NAVI_UPLOAD_BATCH_MB (256 with low_end_gpu = 1; 0, the default, = no limit): once the open batch's staging buffers hold that much, the -// outermost vk_upload_end submits it. A save load made thousands of textures between two replay submissions, and every -// one of them went into one batch: about 1.5 GB of raw ASTC blocks plus the decoder's work in a single submission, +// NX2NAVI_UPLOAD_BATCH_MB (256 with low_end_gpu = 1; 0, the default, = no limit of its own): once the open batch's +// staging buffers hold that much, the outermost vk_upload_end submits it. It does so too once the batch's staging +// reaches STG_FLUSH, so that an area load's staging stays inside the staging pool's blocks (the staging pool's +// comment). A save load made thousands of textures between two replay submissions, and every one of them went into +// one batch: about 1.5 GB of raw ASTC blocks plus the decoder's work in a single submission, // which AMD's 23.10 driver on an RX 580 refused with VK_ERROR_OUT_OF_DEVICE_MEMORY. Submitted early, a batch still // precedes the replay submission that uses its textures (the queue keeps order), and its staging is freed as it retires. static uint64_t upload_cap = UINT64_MAX - 1; // UINT64_MAX - 1: not read yet @@ -4690,9 +4703,10 @@ void vk_upload_end(void) { long mb = e && *e ? atol(e) : 0; upload_cap = mb > 0 ? (uint64_t)mb << 20 : UINT64_MAX; if (mb > 0) rt_log("vk: upload batches submitted early above %ld MiB of staging (NX2NAVI_UPLOAD_BATCH_MB)", mb); - else rt_log("vk: upload batches not limited (NX2NAVI_UPLOAD_BATCH_MB=0)"); + else rt_log("vk: upload batches submitted early above %llu MiB of staging only, the staging pool's limit " + "(NX2NAVI_UPLOAD_BATCH_MB=0)", (unsigned long long)(STG_FLUSH >> 20)); } - if (open_active && open_b.bytes >= upload_cap && cmd_lock.depth == 1) { + if (open_active && cmd_lock.depth == 1 && (open_b.bytes >= upload_cap || batch_staging(&open_b) >= STG_FLUSH)) { n_early++; if (n_early <= 4 || !(n_early & (n_early - 1))) rt_log("vk: upload batch submitted early with %.0f MiB of staging in %d buffers and %d arena chunks (%llu so far)", @@ -4736,10 +4750,72 @@ void vk_mem_oom_report(const char *what, VkResult r) { // the headroom thread (vk_headroom.c) or a burst of uploads held every staging allocation of the render thread: one // took 47-50 ms, for a few bytes, in the first frame after a save load. Larger buffers than a block go to the default // pool. +// The pool's blocks are also all the staging an area load gets (stg_create). Entering Hyrule Field uploads about 3,100 +// textures (1.2 GiB) in one frame; each upload kept its staging until its batch completed, so that frame made 221 new +// staging buffers, 1.1 GiB of new host memory. On Linux every new block is a fresh kernel allocation that asks for +// 2-4 MiB pages first, and with the free 2 MiB blocks used up the kernel compacted memory for each: on an RX 6800 XT +// (amdgpu, 32 GB) the load frame then took 2.4 to 4.7 s instead of 0.4, the texture thread in the kernel all along, +// the compaction scanning millions of pages per half second. Now a load stays inside the blocks made at boot: a +// batch whose staging reaches STG_FLUSH is submitted at the end of that upload (vk_upload_end), and a staging buffer that +// does not fit the blocks takes the room of the kept buffers and idle arena chunks, then of the batches in flight, +// which it waits for in order (a wait covers all the queue's work submitted ahead of that batch, tens of ms on a heavy +// frame, with cmd_lock held). Only when none of that makes room (the open batch's own staging, staging buffers their +// callers hold, the blocks' fragmentation) does the pool grow by a block, and the log says so. +// A discrete GPU keeps STG_MIN_BLOCKS blocks; an integrated one, whose host memory is the machine's RAM and where low +// memory ended the game before (vk_mem_give_back), keeps STG_MIN_BLOCKS_UMA, and its loads wait for batches sooner. #define STG_BLOCK (64ull << 20) -#define STG_MIN_BLOCKS 2 +#define STG_MIN_BLOCKS 4 +#define STG_MIN_BLOCKS_UMA 2 static VmaPool stg_pool; static int stg_pool_tried; +static uint64_t stg_room_cache, stg_room_wait, stg_room_new; // cmd_lock: how stg_create found room (rt_log below) + +// vmaCreateBuffer of a staging buffer (*ai names stg_pool) inside the pool's blocks: when they are full, the kept +// staging buffers and the arena's idle chunks go, then the oldest batch in flight is waited for and retired, until the +// buffer fits. chunk: the buffer is an arena chunk, and VK_INCOMPLETE says that a retired batch gave chunks back to +// arena_free, to take one of those instead. Without a batch to wait for (or once the device is lost) the pool grows. +static VkResult stg_create(VkCtx *c, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *b, + VmaAllocation *a, VmaAllocationInfo *info, int chunk) { + VmaAllocationCreateInfo in = *ai; + in.flags |= VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT; + plat_rmutex_lock(&cmd_lock); + VkResult r; + for (;;) { + if ((r = vmaCreateBuffer(c->vma, bi, &in, b, a, info)) == VK_SUCCESS) break; + if (stg_ncache || narena_free) { + for (int i = 0; i < stg_ncache; i++) vmaDestroyBuffer(c->vma, stg_cache[i].buf, stg_cache[i].alloc); + for (int i = 0; i < narena_free; i++) vmaDestroyBuffer(c->vma, arena_free[i].buf, arena_free[i].alloc); + stg_ncache = 0, stg_cache_bytes = 0, narena_free = 0; + stg_room_cache++; + continue; + } + if (ninflight && !vk_device_is_lost()) { + int before = ninflight; + VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX); + if (w != VK_SUCCESS) { + vk_device_lost_check(w, "the wait for an upload batch"); + rt_fatal("vk: the wait for an upload failed (%d)", (int)w); + } + retire(c, 0); + stg_room_wait++; + if (chunk && narena_free) { + r = VK_INCOMPLETE; + break; + } + if (ninflight < before) continue; + } + r = vmaCreateBuffer(c->vma, bi, ai, b, a, info); + if (r == VK_SUCCESS && stg_room_new++ < 16) + rt_log("vk: the staging pool grew by a block for a buffer of %.1f MiB (the open upload batch holds %.0f MiB, " + "%d batches in flight; room found so far by dropping kept buffers %llu times, by waiting for a batch " + "%llu times; buffers callers hold and the blocks' fragmentation also take room)", + bi->size / 1048576.0, open_active ? batch_staging(&open_b) / 1048576.0 : 0.0, ninflight, + (unsigned long long)stg_room_cache, (unsigned long long)stg_room_wait); + break; + } + plat_rmutex_unlock(&cmd_lock); + return r; +} // A staging buffer the driver refused (vk_staging): the buffers kept for reuse go back to the driver before one more // try, with the headroom's spare blocks (vk_headroom_release): the staging cache, the arena's free chunks and the @@ -4787,13 +4863,14 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { plat_rmutex_lock(&cmd_lock); if (!stg_pool_tried) { uint32_t type; - VmaPoolCreateInfo pi = {.blockSize = STG_BLOCK, .minBlockCount = STG_MIN_BLOCKS}; + int keep = c->props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? STG_MIN_BLOCKS : STG_MIN_BLOCKS_UMA; + VmaPoolCreateInfo pi = {.blockSize = STG_BLOCK, .minBlockCount = (size_t)keep}; if (vmaFindMemoryTypeIndexForBufferInfo(c->vma, &bi, &aci, &type) == VK_SUCCESS) { pi.memoryTypeIndex = type; if (vmaCreatePool(c->vma, &pi, &stg_pool) != VK_SUCCESS) stg_pool = NULL; } rt_log("vk: staging pool %s (%d blocks of %llu MiB kept)", stg_pool ? "created" : "unavailable: default pool", - STG_MIN_BLOCKS, (unsigned long long)(STG_BLOCK >> 20)); + keep, (unsigned long long)(STG_BLOCK >> 20)); __atomic_store_n(&stg_pool_tried, 1, __ATOMIC_RELEASE); } plat_rmutex_unlock(&cmd_lock); @@ -4812,7 +4889,7 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { VmaAllocationInfo info; VkBuffer b; VmaAllocation a; - VkResult r = vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info); + VkResult r = aci.pool ? stg_create(c, &bi, &aci, &b, &a, &info, 0) : vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, &info); if (r == VK_ERROR_OUT_OF_DEVICE_MEMORY && aci.pool) { // The staging pool can grow only in 64 MiB blocks of its one memory type. A smaller allocation may // still fit in a default block or another host-visible type. Keep the same mapping requirements. @@ -4861,23 +4938,27 @@ void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) { VkDeviceSize at = (open_b.chunk_used + ARENA_ALIGN - 1) & ~(ARENA_ALIGN - 1); if (!open_b.nchunk || at + size > ARENA_CHUNK) { StgChunk k; - if (narena_free) { - k = arena_free[--narena_free]; - } else { + VkResult r = VK_INCOMPLETE; + if (!narena_free) { VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = ARENA_CHUNK, .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; - if (stg_pool) aci.pool = stg_pool; // half of a staging block (STG_BLOCK) + if (stg_pool) aci.pool = stg_pool; // an eighth of a staging block (STG_BLOCK) VmaAllocationInfo info; - if (vmaCreateBuffer(c->vma, &bi, &aci, &k.buf, &k.alloc, &info) != VK_SUCCESS) - return vk_staging(size, buf, NULL); - k.map = info.pMappedData; - vk_texstat_add(VK_TS_STG_NEW, 1, ARENA_CHUNK); - if (arena_made++ < 16) rt_log("vk: staging arena chunk %llu made (%llu MiB)", (unsigned long long)arena_made, - (unsigned long long)(ARENA_CHUNK >> 20)); + r = stg_pool ? stg_create(c, &bi, &aci, &k.buf, &k.alloc, &info, 1) + : vmaCreateBuffer(c->vma, &bi, &aci, &k.buf, &k.alloc, &info); + if (r != VK_SUCCESS && r != VK_INCOMPLETE) return vk_staging(size, buf, NULL); + if (r == VK_SUCCESS) { + k.map = info.pMappedData; + vk_texstat_add(VK_TS_STG_NEW, 1, ARENA_CHUNK); + if (arena_made++ < 16) + rt_log("vk: staging arena chunk %llu made (%llu MiB)", (unsigned long long)arena_made, + (unsigned long long)(ARENA_CHUNK >> 20)); + } } + if (r == VK_INCOMPLETE) k = arena_free[--narena_free]; // one already free, or one a waited-for batch gave back if (open_b.nchunk == open_b.cap_chunk) grow((void **)&open_b.chunk, &open_b.cap_chunk, sizeof *open_b.chunk); open_b.chunk[open_b.nchunk++] = k; open_b.bytes += ARENA_CHUNK; // the staging the batch holds (upload_cap) diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 00052131..2c1302ef 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -3533,8 +3533,9 @@ static void upload(Tex *e) { } int check = bc1 || try1; // A CPU decode or copy stages at most UPLOAD_PART bytes per staging buffer (a region larger than that alone) and - // records one copy per buffer; every part stays in the same upload batch, submitted as before. The GPU decoder - // takes the raw blocks of every region from one buffer. + // records one copy per buffer into the open upload batch (vk_upload_end may submit a batch that holds much staging + // between two parts; queue order and the barriers keep them in order). The GPU decoder takes the raw blocks of + // every region from one buffer. for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) { VkDeviceSize base = rg[k0].bufferOffset, size; for (k1 = k0 + 1; k1 < n && (agpu || (k1 + 1 < n ? rg[k1 + 1].bufferOffset : total) - base <= UPLOAD_PART);) k1++; From eaf10677fa1c1bbd4a98bb115f178d03c944d497 Mon Sep 17 00:00:00 2001 From: Vasilis Manetas Date: Thu, 12 Nov 2026 18:12:08 +0100 Subject: [PATCH 04/15] Import guest memory as dma-bufs through udmabuf on Linux instead of keeping shadow copies of it. On Linux every CPU-visible pool had a host-visible shadow copy, because amdgpu treats a host-pointer import as a userptr object that it looks up again at every submission and invalidates whenever the write watch or a freeze re-protects its pages. Now the watched reservations are shared mappings of one memfd, /dev/udmabuf makes dma-bufs of page ranges of it, and the device imports them (VK_KHR_external_memory_fd, VK_EXT_external_memory_dma_buf). A dma-buf pins its pages once and is not tied to the process's page tables, so protection changes leave the GPU's mapping alone, and nothing is copied. RADV puts every imported memory object into every submission, so pools share their imports: guest addresses are cut into 64 MiB spans (NX2NAVI_DMABUF_MB), and each pool is a buffer bound into its span's memory at its offset. A new span is cut back at the spans already there, so spans overlap only where a pool itself lies across an older span's edge, since amdgpu counts every span's pages in its GTT use. A decommit punches the pages out of the memfd, or zeroes them in place where a dma-buf pins them, and the pages a freed span pinned that nothing committed go back to the system. It is on by default where /dev/udmabuf opens, udmabuf's size_limit_mb is 1024 or more, the device has both extensions and NVIDIA's kernel module is not loaded. A start-up probe imports one page as the pools will be; any refusal keeps the shadow copies, and the log says why. NX2NAVI_DMABUF=0 keeps the copies. tests/platform_dmabuf_test.c checks the shared mapping without a GPU, and docs/building.md ("Memory") describes the imports and what they need. RX 6800 XT (RADV, Mesa 26.2.6), one build, interleaved pairs with NX2NAVI_DMABUF=1 and 0: Kokiri Forest 60.7 against 40.3 fps (3 pairs, +51%, standard error 0.4), 1% low 50.9 against 34.0, frames over 25 ms 5 against 768; Hyrule Field 55.7 against 42.4 fps (2 pairs, +31%, standard error 0.4). No pool was left without a span, no span was refused, and --probe-vkmem passes with and without the imports. --- docs/README-BuildKit.md | 7 + docs/building.md | 36 +++- runtime/cfg.c | 2 +- runtime/platform/linux.c | 218 +++++++++++++++++++++- runtime/platform/platform.h | 11 ++ runtime/platform/win32.c | 5 + runtime/vk.h | 8 + runtime/vk_device.c | 16 ++ runtime/vk_memory.c | 351 ++++++++++++++++++++++++++++++++++- tests/platform_dmabuf_test.c | 111 +++++++++++ 10 files changed, 750 insertions(+), 15 deletions(-) create mode 100644 tests/platform_dmabuf_test.c diff --git a/docs/README-BuildKit.md b/docs/README-BuildKit.md index 6df775cc..734f6c2c 100644 --- a/docs/README-BuildKit.md +++ b/docs/README-BuildKit.md @@ -121,6 +121,13 @@ memory runs short, it ends the process, so while less than the larger of 4 GB an drops stale textures and evicts idle ones. The log names the limits at the first check (`platform: memory limits`). That cannot give back what the game needs after its start: about 10 GB in Kokiri Forest at a 2160p game resolution. +Without NVIDIA's driver, the game hands its memory to the GPU through `/dev/udmabuf` instead of copying it there for +every submission, which is faster on AMD cards (61 frames a second instead of 40 in Kokiri Forest on an RX 6800 XT). +That needs the device readable and writable by you (recent systemd gives it to the logged-in user) and udmabuf's +`size_limit_mb` (`/sys/module/udmabuf/parameters/size_limit_mb`) at 1024 or more (`udmabuf.size_limit_mb=1024` on the +kernel command line raises it). Otherwise the game copies as before, and the log's `mem:` lines say why +(`docs/building.md`, "Memory"). + AMD FSR 3.1 loads AMD's `amd_fidelityfx_vk.dll`, which only the Windows build can use: on Linux the menu offers neither FSR 3.1 upscaling nor FSR 3.1 frame generation, and `taa = fsr3` or `framegen = fsr3` in `nx2navi.ini` leave the game's own scaling and swapchain as they are (the log says so). AMD FSR 4 is in the Windows build alone for the same diff --git a/docs/building.md b/docs/building.md index 1f138b3a..c9985f4c 100644 --- a/docs/building.md +++ b/docs/building.md @@ -279,7 +279,8 @@ folder can be deleted; `ocarina --warm-shaders` fills it again. Memory: on Windows the GPU imports guest memory and `GetWriteWatch` tracks the CPU's writes to it. On Linux, re-protecting imported memory invalidates the driver's userptr buffer on every submit (about 1.3 ms per 16 MiB on -amdgpu), and importing pins the pages, which drops their write protection. Each CPU-visible pool therefore gets a +amdgpu), and importing pins the pages, which drops their write protection. Where guest memory cannot be imported as +dma-bufs (below), each CPU-visible pool therefore gets a host-visible copy (shadow storage, `runtime/vk_memory.c`): the pages the CPU wrote go in before each submit, and the GPU's writes come back at its fence. Pages both sides write keep a base copy, so each direction moves only its own bytes; a VRAM mirror's write-back merge marks the pages it changed, and only those are compared at the fence; a lookup that @@ -295,6 +296,30 @@ copies are only the slower path. In Kokiri Forest on 0.7.1 (RTX 4090, driver 615 51.9 frames a second and the copies at 27.4; on that version the copies also flickered between two looks almost every frame, which the wait for the write-backs under shadow storage (`vk_sync_exec_wait`) has since addressed. +On Linux the pools are imports after all where the system allows dma-bufs of guest memory (`runtime/platform/linux.c`, +"Guest memory as a memfd's"; `runtime/vk_memory.c`, "dma-buf imports"). The watched reservations are then shared +mappings of one memfd, `/dev/udmabuf` makes dma-bufs of page ranges of it, and the device imports those +(`VK_KHR_external_memory_fd`, `VK_EXT_external_memory_dma_buf`). A dma-buf pins its pages once and is not tied to the +process's page tables, so the write watch and the freezes leave the GPU's mapping alone, and nothing is copied. RADV +puts every imported memory object into every submission (about 0.6 us each), and the game keeps over 4,000 pools, so +pools share their imports: guest addresses are cut into spans of 64 MiB (`NX2NAVI_DMABUF_MB`), and a pool is a buffer +bound into its span's memory at its offset. A new span is cut back at the spans already there, so spans overlap only +where a pool itself lies across an older span's edge: amdgpu counts every span's pages in its GTT use, an overlap twice. +A span pins all of its pages, those the game never touched included, until some frames after its last pool is gone. It +takes `/dev/udmabuf` readable and writable by the player (recent systemd gives it to the logged-in user with its +`uaccess` tag; elsewhere it is often `root` and group `kvm` only), udmabuf's `size_limit_mb` +(`/sys/module/udmabuf/parameters/size_limit_mb`) at 1024 or more, since the largest pools need about 300 MiB in one +dma-buf (where it is lower, `udmabuf.size_limit_mb=1024` on the kernel command line raises it), a device with both +extensions, and NVIDIA's kernel module not loaded: NVIDIA's own driver imports host pointers without amdgpu's cost, as +above. A start-up probe imports one page of guest memory as the pools will be; where anything is missing or refused, the +pools keep their shadow copies and the log says why (`mem: dma-buf imports off`, `mem: guest memory is not a memfd's`). +`NX2NAVI_DMABUF=0` keeps the copies, and `NX2NAVI_DMABUF=1` makes guest memory a memfd's even where NVIDIA's module is +loaded; both only from the environment, since the reservations are made before `nx2navi.ini` is read. `NX2NAVI_SHADOW=1` +still forces the copies. In Kokiri Forest (1.1.1, RX 6800 XT, Mesa 26.2) the imports ran at 60.7 frames a second and the +copies at 40.3, in Hyrule Field at 55.7 and 42.4; the spans then held 8.0 to 9.6 GiB, and amdgpu counted 8.6 to 10.1 GiB +of its 15.8 GiB host heap in use, against 5.2 to 5.6 GiB with the copies, which are memory of their own beside guest +memory. + The device address table (`runtime/vk_gpa.c`) names a pool's shadow copy as it names an import, so shaders follow guest pointers into either. A shader that reads through the table resolves nothing, so the pool it reaches is not queued for the copy-in: a guest submission whose record binds the table (`vk_exec_draw.c` `ssbo_info`) brings the shadow of @@ -315,12 +340,13 @@ write-backs landed on pages the CPU had written since their copy-in, and what th plain CPU write and a guest submission that does not bind the table, and hold the written `0x2468ACE0` after one that does; and with `NX2NAVI_SHADOW_TABLE_STORES=1` the store: a GPU write into a shadow page that nothing notes must survive the CPU's writes of that page and reach guest memory. On -Linux, run it for each backing: +Linux, run it for each backing (with `NX2NAVI_DMABUF=1` and `NX2NAVI_SHADOW=0` the pools are dma-buf imports, with +`NX2NAVI_DMABUF=0` and `NX2NAVI_SHADOW=0` host-pointer imports): ```sh -for shadow in 0 1; do for memory in import mirror; do - NX2NAVI_GPA=1 NX2NAVI_SHADOW=$shadow NX2NAVI_MEMORY=$memory ./ocarina --probe-vkmem -done; done +for dmabuf in 0 1; do for shadow in 0 1; do for memory in import mirror; do + NX2NAVI_DMABUF=$dmabuf NX2NAVI_GPA=1 NX2NAVI_SHADOW=$shadow NX2NAVI_MEMORY=$memory ./ocarina --probe-vkmem +done; done; done ``` The Deck's GPU (Van Gogh) has no native ASTC, so its textures go through the compute decoder (`runtime/vk_astc.c`, diff --git a/runtime/cfg.c b/runtime/cfg.c index ac223d57..8075fbf4 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -44,7 +44,7 @@ static const char *const known[] = { "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CACHE_DIE", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COMPUTE_KEEP", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", - "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DRAW_CRUMBS", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", + "DESC_TEMPLATE", "DEVICE_PROFILE", "DMABUF", "DMABUF_MB", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DRAW_CRUMBS", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "GPU_TIMELINE", "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index 8c531f28..c5fa433f 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -1113,6 +1113,189 @@ static void ps_set(uint64_t addr, uint64_t n, uint8_t v) { if (addr < PS_LIMIT) memset(pstate + addr / PG, v, (size_t)((n + PG - 1) / PG)); } +// Guest memory as a memfd's (NX2NAVI_DMABUF, read from the environment: the reservations come before nx2navi.ini; on +// by default where /dev/udmabuf opens, udmabuf's size limit is 1 GiB or more and NVIDIA's kernel module is not loaded; +// 0 off, 1 on wherever it can be). +// The watched reservations are MAP_SHARED mappings of one memfd, each at its own offset, so that /dev/udmabuf can make +// a dma-buf of the game's own pages and the GPU driver import it (vk_memory.c "dma-buf imports"). A host-pointer +// import is a userptr object to amdgpu, which the kernel looks up page by page again at every submission and +// invalidates whenever the write watch or a freeze changes the pages' protection; a udmabuf pins the memfd's pages +// once and never looks at the process's page tables, so protection changes leave the GPU's mapping alone. Everything +// else stays as it was: the write watch, the freezes and the access faults work the same on a shared mapping. +// Two things differ from private memory, and both are handled here under vm_lock: +// - Decommitted pages must read zero when committed again. MADV_DONTNEED leaves a shared mapping's pages in the +// memfd, so a decommit punches them out of it (fallocate) instead. A page a dma-buf pins cannot go that way: the +// dma-buf would keep the old page while the CPU got a new one, and the GPU and the CPU would no longer see the same +// memory. Those pages are zeroed in place (gm_decommit). +// - A dma-buf pins every page of its range, allocating the ones nothing has touched. When it is gone +// (plat_mem_dmabuf_done) the pages of its range that are not committed and that no other dma-buf pins are punched +// out, so they cost no memory. +#if __has_include() +#include +#else // the kernel's ABI (include/uapi/linux/udmabuf.h), for headers older than it +struct udmabuf_create { __u32 memfd; __u32 flags; __u64 offset; __u64 size; }; +#define UDMABUF_FLAGS_CLOEXEC 0x01 +#define UDMABUF_CREATE _IOW('u', 0x42, struct udmabuf_create) +#endif +#define GM_RES_MAX 4 +typedef struct { uint64_t addr, n, off; } GmRes; +typedef struct { uint64_t lo, hi; } GmPin; +static int gm_state; // vm_lock: 0 not decided, 1 the watched reservations are the memfd's, -1 not +static int gm_fd = -1, gm_udev = -1; // the memfd and /dev/udmabuf +static uint64_t gm_size, gm_limit; // bytes of the memfd handed out; udmabuf's size limit +static GmRes gm_res[GM_RES_MAX]; // the shared reservations and their offsets in the memfd +static int gm_nres; +static GmPin *gm_pins; // vm_lock: the ranges live dma-bufs pin (one entry per dma-buf) +static int gm_npins, gm_cpins; +static char gm_note[200]; // why the reservations are not a memfd's ("" when NX2NAVI_DMABUF=0 or they are) + +// The largest dma-buf the udmabuf ioctl accepts, in bytes: the module's size_limit_mb parameter, or the module's +// default of 64 MiB where the parameter cannot be read. +static uint64_t udmabuf_limit(void) { + uint64_t mib; + if (!read_u64_file("/sys/module/udmabuf/parameters/size_limit_mb", &mib)) mib = 64; + return (mib < (1ull << 40) ? mib : 1ull << 40) << 20; +} + +static void gm_open(void) { // vm_lock held: the first watched reservation decides + const char *e = getenv("NX2NAVI_DMABUF"); + int asked = e && !strcmp(e, "1"); + gm_state = -1; + if (e && !strcmp(e, "0")) return; + // Unset: on where it can work and pays. NVIDIA's driver imports host pointers without amdgpu's costs (vk_memory.c + // mirror_mode), and its import of these dma-bufs has not been tried, so where its kernel module is loaded the memory + // stays private. + if (!asked && !access("/sys/module/nvidia", F_OK)) { + snprintf(gm_note, sizeof gm_note, "NVIDIA's kernel module is loaded (NX2NAVI_DMABUF=1 makes it a memfd's anyway)"); + return; + } + gm_udev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); + if (gm_udev < 0) { // many systems give the device to root and group kvm only + snprintf(gm_note, sizeof gm_note, "/dev/udmabuf cannot be opened (%s)", strerror(errno)); + fprintf(stderr, "platform: guest memory as dma-bufs (NX2NAVI_DMABUF): %s: guest memory stays private\n", gm_note); + return; + } + gm_limit = udmabuf_limit(); + // The game's largest pools need about 300 MiB in one dma-buf (vk_memory.c mirror_mode): below 1 GiB the back end + // would keep the shadow copies anyway, so unasked the memory stays private. + if (!asked && gm_limit < (1ull << 30)) { + snprintf(gm_note, sizeof gm_note, "udmabuf's size_limit_mb is %llu, below the 1024 a pool of the game may need " + "(/sys/module/udmabuf/parameters/size_limit_mb)", (unsigned long long)(gm_limit >> 20)); + close(gm_udev), gm_udev = -1; + return; + } + gm_fd = (int)syscall(SYS_memfd_create, "nx2navi-guest", MFD_CLOEXEC | MFD_ALLOW_SEALING); + if (gm_fd < 0) { + snprintf(gm_note, sizeof gm_note, "no memfd (%s)", strerror(errno)); + fprintf(stderr, "platform: guest memory as dma-bufs (NX2NAVI_DMABUF): %s: guest memory stays private\n", gm_note); + close(gm_udev), gm_udev = -1; + return; + } + gm_state = 1; +} + +static int gm_find(uint64_t addr, uint64_t n) { // the shared reservation holding [addr, addr + n), -1 none + for (int i = 0; i < gm_nres; i++) + if (addr >= gm_res[i].addr && n <= gm_res[i].n && addr - gm_res[i].addr <= gm_res[i].n - n) return i; + return -1; +} + +// vm_lock held. Whether x is pinned by a live dma-buf; *end gets the end of the run of [x, limit) that is the same. +static int gm_pinned_run(uint64_t x, uint64_t limit, uint64_t *end) { + uint64_t cur = x; + for (int grew = 1; grew && cur < limit;) { + grew = 0; + for (int i = 0; i < gm_npins; i++) + if (gm_pins[i].lo <= cur && cur < gm_pins[i].hi) cur = gm_pins[i].hi, grew = 1; + } + if (cur > x) { + *end = cur < limit ? cur : limit; + return 1; + } + uint64_t next = limit; + for (int i = 0; i < gm_npins; i++) + if (gm_pins[i].lo > x && gm_pins[i].lo < next) next = gm_pins[i].lo; + *end = next; + return 0; +} + +static int gm_punch(int gi, uint64_t addr, uint64_t n) { + return !fallocate(gm_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, (off_t)(gm_res[gi].off + addr - gm_res[gi].addr), + (off_t)n); +} + +// vm_lock held: a decommit of [addr, addr + n) in shared reservation gi. +static int gm_decommit(int gi, uint64_t addr, uint64_t n) { + uint64_t end = addr + n; + for (uint64_t x = addr, y; x < end; x = y) { + if (!gm_pinned_run(x, end, &y)) { + if (!gm_punch(gi, x, y - x)) return 0; + continue; + } + for (uint64_t a = x, b; a < y; a = b) { // the committed runs; a reserved page is zero already + uint8_t v = pstate[a / PG]; + for (b = a + PG; b < y && pstate[b / PG] == v; b += PG) {} + if (v < PS_NONE) continue; + if (v != PS_RW && mprotect((void *)(uintptr_t)a, b - a, PROT_READ | PROT_WRITE)) return 0; + memset((void *)(uintptr_t)a, 0, b - a); + } + } + if (mprotect((void *)(uintptr_t)addr, n, PROT_NONE)) return 0; + ps_set(addr, n, PS_RESERVED); + return 1; +} + +int plat_mem_dmabuf_ok(uint64_t *limit, uint64_t *any, const char **why) { + plat_mutex_lock(&vm_lock); + int ok = gm_state > 0 && gm_nres > 0; + *limit = gm_limit; + *any = gm_nres ? gm_res[0].addr : 0; + *why = ok ? "" : gm_state > 0 ? "the memfd could not back a reservation" : gm_note; + plat_mutex_unlock(&vm_lock); + return ok; +} + +int plat_mem_dmabuf(uint64_t addr, uint64_t n) { + if (!n || ((addr | n) & (PG - 1))) return -1; + plat_mutex_lock(&vm_lock); + int gi = gm_state > 0 && n <= gm_limit ? gm_find(addr, n) : -1, fd = -1, err = EINVAL; + if (gi >= 0 && gm_npins == gm_cpins) { + int c = gm_cpins ? gm_cpins * 2 : 64; + GmPin *p = realloc(gm_pins, (size_t)c * sizeof *p); + if (p) gm_pins = p, gm_cpins = c; + } + if (gi >= 0 && gm_npins < gm_cpins) { + struct udmabuf_create c = {.memfd = (__u32)gm_fd, .flags = UDMABUF_FLAGS_CLOEXEC, + .offset = gm_res[gi].off + addr - gm_res[gi].addr, .size = n}; + fd = ioctl(gm_udev, UDMABUF_CREATE, &c); + err = errno; + if (fd >= 0) gm_pins[gm_npins++] = (GmPin){addr, addr + n}; + } + plat_mutex_unlock(&vm_lock); + if (fd < 0) errno = err; + return fd; +} + +void plat_mem_dmabuf_done(uint64_t addr, uint64_t n) { + plat_mutex_lock(&vm_lock); + for (int i = 0; i < gm_npins; i++) + if (gm_pins[i].lo == addr && gm_pins[i].hi == addr + n) { + gm_pins[i] = gm_pins[--gm_npins]; + break; + } + int gi = gm_find(addr, n); + uint64_t end = addr + n; + for (uint64_t x = addr, y; gi >= 0 && x < end; x = y) { + if (gm_pinned_run(x, end, &y)) continue; + for (uint64_t a = x, b; a < y; a = b) { + uint8_t v = pstate[a / PG]; + for (b = a + PG; b < y && pstate[b / PG] == v; b += PG) {} + if (v < PS_NONE) gm_punch(gi, a, b - a); + } + } + plat_mutex_unlock(&vm_lock); +} + static void ww_open(void) { int fd = (int)syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); struct uffdio_api api = {.api = UFFD_API, @@ -1138,12 +1321,29 @@ int plat_mem_reserve(uint64_t addr, uint64_t n, int watch) { pstate = p == MAP_FAILED ? NULL : p; } if (watch && ww_fd < 0 && pm_fd < 0) ww_open(); + if (watch && !gm_state) gm_open(); + int shared = watch && gm_state > 0 && gm_nres < GM_RES_MAX; void *p = pstate && addr + n <= PS_LIMIT ? mmap((void *)(uintptr_t)addr, n, PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED_NOREPLACE, -1, 0) + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS) | MAP_NORESERVE | MAP_FIXED_NOREPLACE, + shared ? gm_fd : -1, shared ? (off_t)gm_size : 0) : MAP_FAILED; int ok = p == (void *)(uintptr_t)addr; if (p != MAP_FAILED && !ok) munmap(p, n); // a kernel without MAP_FIXED_NOREPLACE put it elsewhere + if (ok && shared) { // the memfd grows by the reservation; shrinking is sealed off, as udmabuf requires + if (ftruncate(gm_fd, (off_t)(gm_size + n)) || (!gm_nres && fcntl(gm_fd, F_ADD_SEALS, F_SEAL_SHRINK))) { + fprintf(stderr, "platform: the memfd for %#llx+%#llx failed (%s): private memory there\n", + (unsigned long long)addr, (unsigned long long)n, strerror(errno)); + munmap(p, n); + p = mmap((void *)(uintptr_t)addr, n, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED_NOREPLACE, -1, 0); + ok = p == (void *)(uintptr_t)addr; + if (p != MAP_FAILED && !ok) munmap(p, n); + } else { + gm_res[gm_nres++] = (GmRes){addr, n, gm_size}; + gm_size += n; + } + } if (ok) { ps_set(addr, n, PS_RESERVED); // Transparent huge pages would populate (and count as written) 2 MiB at the first write of one page; writes are @@ -1164,12 +1364,22 @@ void *plat_mem_alloc(size_t n) { } int plat_mem_commit(uint64_t addr, size_t n) { - if (mprotect((void *)(uintptr_t)addr, n, PROT_READ | PROT_WRITE)) return 0; - ps_set(addr, n, PS_RW); - return 1; + // In a shared reservation under vm_lock: plat_mem_dmabuf_done punches out the pages it finds not committed. + int locked = gm_nres > 0 && gm_find(addr, n) >= 0; + if (locked) plat_mutex_lock(&vm_lock); + int ok = !mprotect((void *)(uintptr_t)addr, n, PROT_READ | PROT_WRITE); + if (ok) ps_set(addr, n, PS_RW); + if (locked) plat_mutex_unlock(&vm_lock); + return ok; } int plat_mem_decommit(uint64_t addr, uint64_t n) { + if (gm_nres > 0) { + plat_mutex_lock(&vm_lock); + int gi = gm_find(addr, n), ok = gi >= 0 && gm_decommit(gi, addr, n); + plat_mutex_unlock(&vm_lock); + if (gi >= 0) return ok; + } if (madvise((void *)(uintptr_t)addr, n, MADV_DONTNEED) || mprotect((void *)(uintptr_t)addr, n, PROT_NONE)) return 0; ps_set(addr, n, PS_RESERVED); return 1; diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 64c0b056..6ea98cea 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -258,6 +258,17 @@ uint64_t plat_mem_written(uint64_t addr, uint64_t n, uint64_t *pages, uint64_t m // pages in one call, where the watch works by protecting pages (Linux), so the stores do not fault page by page. The // pages count as written. Returns 0 when there is nothing to clear or it cannot be done; the write is valid either way. int plat_mem_prepare_write(uint64_t addr, uint64_t n); +// Guest memory as dma-bufs (Linux, NX2NAVI_DMABUF, on by default; vk_memory.c "dma-buf imports"): the watched +// reservations are shared mappings of one memfd, and /dev/udmabuf makes a dma-buf of a page range of it, which the GPU +// driver imports like its own memory. plat_mem_dmabuf_ok: 1 when that is so, with udmabuf's size limit in bytes in *limit and the +// first page of a shared reservation in *any; 0 with the reason in *why ("" when NX2NAVI_DMABUF=0 turned it off). +// plat_mem_dmabuf: a new dma-buf file descriptor over [addr, addr + n) (whole pages inside one watched reservation), +// -1 with errno on failure. Its pages stay pinned (a decommit zeroes them in place) until plat_mem_dmabuf_done is +// told, after the last user of the dma-buf (the imported memory) is gone; the pages of its range that nothing has +// committed then go back to the system. Elsewhere these report 0 and -1. +int plat_mem_dmabuf_ok(uint64_t *limit, uint64_t *any, const char **why); +int plat_mem_dmabuf(uint64_t addr, uint64_t n); +void plat_mem_dmabuf_done(uint64_t addr, uint64_t n); // A write that faulted on a read-only page at addr goes to fn first; fn returns 1 when it made the page writable and // the write should run again, 0 to let the fault go on (to the crash handler). One hook per process. void plat_mem_fault_hook(int (*fn)(uint64_t addr)); diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index cfaa5c9e..96c58e77 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -866,6 +866,11 @@ int plat_mem_prepare_write(uint64_t addr, uint64_t n) { return 0; } +// Guest memory as dma-bufs is Linux's (platform.h). +int plat_mem_dmabuf_ok(uint64_t *limit, uint64_t *any, const char **why) { *limit = *any = 0, *why = ""; return 0; } +int plat_mem_dmabuf(uint64_t addr, uint64_t n) { (void)addr; (void)n; return -1; } +void plat_mem_dmabuf_done(uint64_t addr, uint64_t n) { (void)addr; (void)n; } + // Contains code Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md). // GetWriteWatch with the reset flag. uint64_t plat_mem_written(uint64_t addr, uint64_t n, uint64_t *pages, uint64_t max, int reset) { diff --git a/runtime/vk.h b/runtime/vk.h index 7e20fafc..871233e8 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -414,6 +414,14 @@ int vk_mem_flush_add_probe(void); // --probe-vkexec: the flush range list uint64_t vk_mem_mirror_uploaded(void); // --probe-vkexec: bytes copied into mirrors since the last summary line int vk_mem_mirror_probe(uint64_t addr); // --probe-vkexec: the pool at addr gets its mirror as on a first buffer use; // 1 when it has one +// Guest memory as dma-bufs (vk_memory.c "dma-buf imports"): pick_device tells whether the device has +// VK_KHR_external_memory_fd and VK_EXT_external_memory_dma_buf (before vk_mem_class_log); create_device enables them +// when the pools are imported that way. +void vk_mem_dmabuf_caps(int exts); +int vk_mem_dmabuf_on(void); +// After vkCreateDevice, before vk_mem_lazy_decide: a page of guest memory imported as the pools will be; any refusal +// turns the dma-buf imports off for the run (shadow storage then). bda: buffers get device addresses. +void vk_mem_dmabuf_probe(VkDevice dev, const VkPhysicalDeviceMemoryProperties *mp, int bda); void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDeviceMemoryProperties *m); // once int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked) void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and, diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 6ad94c89..2212fe15 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -14,6 +14,7 @@ #include "vk.h" #include "platform/platform.h" #include "vk_dlaa.h" +#include "vk_gpa.h" #include "vk_fg.h" #include "vk_dlssg.h" @@ -311,6 +312,15 @@ retry: free(qp); rt_log("vk: using gpu %d: %s", best, g_vk.props.deviceName); vk_caps_driver(g_vk.phys); // before vk_mem_class_log, which chooses the pools' storage by driver + { // and by whether the device imports dma-bufs (vk_memory.c "dma-buf imports") + uint32_t ne = 0; + vkEnumerateDeviceExtensionProperties(g_vk.phys, NULL, &ne, NULL); + VkExtensionProperties *ep = calloc(ne ? ne : 1, sizeof *ep); + vkEnumerateDeviceExtensionProperties(g_vk.phys, NULL, &ne, ep); + vk_mem_dmabuf_caps(has_ext(ep, ne, VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME) && + has_ext(ep, ne, VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME)); + free(ep); + } vk_mem_class_log(&g_vk.props, &g_vk.mem_props); return 1; } @@ -492,6 +502,11 @@ static int create_device(void) { g_vk.has_nv_checkpoints = 1; ext[n_ext++] = VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME; } + // Guest memory imported as dma-bufs (vk_memory.c "dma-buf imports"; pick_device checked the extensions) + if (vk_mem_dmabuf_on() && n_ext + 4 <= sizeof ext / sizeof *ext) { + ext[n_ext++] = VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME; + ext[n_ext++] = VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME; + } // Graphics pipeline library: chained last so it can be dropped again when vk_caps_gpl declines it. void **gpl_link = NULL; if (has_ext(ep, ne, VK_KHR_PIPELINE_LIBRARY_EXTENSION_NAME) && has_ext(ep, ne, VK_EXT_GRAPHICS_PIPELINE_LIBRARY_EXTENSION_NAME)) { @@ -644,6 +659,7 @@ static int create_device(void) { vkGetPhysicalDeviceProperties2(g_vk.phys, &p2); g_vk.min_imported_host_alignment = (uint32_t)emh.minImportedHostPointerAlignment; vk_caps_init(&g_vk); + vk_mem_dmabuf_probe(g_vk.device, &g_vk.mem_props, vk_gpa_wanted_ctx(&g_vk)); // may turn shadow storage on vk_mem_lazy_decide(); // the driver is known: the lazy write-back's default depends on it rt_fatal_note_add(fatal_note); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ac28225b..33308bc8 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -8,6 +8,7 @@ // users. Virtual pools are a GPU-side page table // over device-local memory and dirty pages come from GetWriteWatch (mem_written_pages), so neither host VM aliasing // nor page protection (a fault handler) is needed. +#include #include #include #include @@ -47,6 +48,7 @@ typedef struct PoolRec { uint64_t import_lo, import_hi; // POOL_IMPORT: the imported host range (aligned outwards) VkBuffer buf; VkDeviceMemory mem; // POOL_IMPORT + struct Span *span; // POOL_IMPORT as a dma-buf: the span whose memory buf is bound to ("dma-buf imports") VmaAllocation alloc; // POOL_PHYS VPage *pt; // POOL_VIRT: size / 64 KiB entries uint64_t npages; @@ -158,9 +160,310 @@ static int memory_type(const VkCtx *c, uint32_t bits, VkMemoryPropertyFlags want return -1; } -// CPU-visible storage: the guest pages themselves become the buffer's memory (VK_EXT_external_memory_host). +// ---------------------------------------------------------------- dma-buf imports +// Where guest memory is a memfd's (Linux, NX2NAVI_DMABUF, on by default: platform/linux.c "Guest memory as a memfd's"), a +// CPU-visible pool is imported as a dma-buf of its own pages, not as a host pointer and not as a shadow copy. It keeps +// what an import is to the rest of this file (the GPU's writes are in guest memory once the GPU has made them, the +// device address table's twin is guest memory, mirrors sync from it and write back into it), without amdgpu's costs +// for host pointers: those are userptr objects, which the kernel looks up page by page at every submission and +// invalidates whenever the write watch or a freeze re-protects their pages (about 0.2 frames a second in Kokiri +// Forest), which is why Linux had shadow storage. A dma-buf's pages are pinned once and the process's page tables +// never matter to it again: the write watch, the freezes and the lazy write-back's armed pages leave the GPU alone. +// RADV puts every imported memory object in every submission, at about 0.6 us each (RX 6800 XT, Linux 7.3, Mesa 26.2: +// 2.6 ms a vkQueueSubmit with 4096 of them, 0.04 ms with 64), and the game keeps over 4,000 pools, so a pool does not +// get a dma-buf of its own: guest addresses are cut into aligned spans of NX2NAVI_DMABUF_MB (64 by default), the first +// pool made in a span imports the span's dma-buf, and every pool is a buffer bound into the memory of a span that holds +// it, at its offset. A pool that no span holds gets a new one over the aligned range around it, cut back at the spans +// already there (span_get): spans overlap only where such a pool itself lies in an older span, because every span's +// pages count in amdgpu's GTT use (an overlap twice) and stay pinned. A span whose +// last pool is gone stays SPAN_KEEP frames for the pools the game makes again at the same addresses, then is freed +// (span_gc); its dma-buf goes with the memory. +// A new span pins its pages: about 13 us per MiB the game had touched and 260 us per MiB it had not, which are +// allocated then (the same RX 6800 XT machine), so a 64 MiB span costs 1-17 ms once. A span over udmabuf's size limit +// (/sys/module/udmabuf/parameters/size_limit_mb) is not made; such a pool, or one outside the memfd, falls back to a +// host-pointer import, which a shared mapping's pages refuse on RADV: it gets no GPU buffer (logged). +typedef struct Span { uint64_t lo, hi; VkDeviceMemory mem; int32_t refs; uint64_t idle_frame; } Span; +#define SPAN_KEEP 600 +static int dmabuf_on; // mirror_mode +static int dmabuf_exts; // vk_mem_dmabuf_caps: the device has VK_KHR_external_memory_fd and _dma_buf +static uint64_t dmabuf_limit; // udmabuf's size limit, bytes +static uint64_t span_size = 64ull << 20; +static PlatMutex span_lock = PLAT_MUTEX_INIT; // after pool_lock +static Span **spans; // span_lock +static int nspans, cspans; +static uint32_t dmabuf_type = UINT32_MAX; // span_lock: the memory type of the imports, asked at the first one +static PFN_vkGetMemoryFdPropertiesKHR p_fd_props; +static int64_t sp_live_bytes; // span_lock +static uint64_t sp_made, sp_made_bytes, sp_ns, sp_max_ns, sp_freed, sp_freed_bytes, sp_fail, sp_fallback; // span_lock, since the log +static uint64_t sp_fail_all; // span_lock +static volatile uint64_t span_frame; // the frame vk_mem_census last saw +#ifdef _WIN32 +static void dmabuf_close(int fd) { (void)fd; } +#else +#include +static void dmabuf_close(int fd) { close(fd); } +#endif + +void vk_mem_dmabuf_caps(int exts) { dmabuf_exts = exts; } +int vk_mem_dmabuf_on(void) { return dmabuf_on; } + +// The memory type of the imports, from the types the driver offers for the dma-buf fd: cached where offered, as guest +// memory is to the CPU (RADV: the GPU snoops it); not one of AMD's device-coherent or device-uncached types, which the +// third pass takes only when nothing else is offered. Sets dmabuf_type (UINT32_MAX: none fits); returns the offer. +static uint32_t dmabuf_pick_type(VkDevice dev, const VkPhysicalDeviceMemoryProperties *mp, int fd) { + if (!p_fd_props) p_fd_props = (PFN_vkGetMemoryFdPropertiesKHR)vk_dpa(dev, "vkGetMemoryFdPropertiesKHR"); + VkMemoryFdPropertiesKHR fp = {.sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR}; + uint32_t bits = p_fd_props && p_fd_props(dev, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, fd, &fp) == VK_SUCCESS + ? fp.memoryTypeBits : 0; + const VkMemoryPropertyFlags hv = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + dmabuf_type = UINT32_MAX; + for (int w = 0; w < 3 && dmabuf_type == UINT32_MAX; w++) + for (uint32_t i = 0; i < mp->memoryTypeCount && dmabuf_type == UINT32_MAX; i++) { + VkMemoryPropertyFlags f = mp->memoryTypes[i].propertyFlags; + if ((bits >> i & 1) && (w == 0 ? f == (hv | VK_MEMORY_PROPERTY_HOST_CACHED_BIT) : w == 1 ? f == hv : (f & hv) == hv)) + dmabuf_type = i; + } + rt_log("mem: dma-bufs of guest memory may have memory types %#x; type %d taken", bits, + dmabuf_type == UINT32_MAX ? -1 : (int)dmabuf_type); + return bits; +} + +static uint64_t dmabuf_any; // a page of guest memory a dma-buf can be made of (vk_mem_dmabuf_probe) + +// After device creation, before the first pool and before anything asks for the storage mode (vk_mem_lazy_decide): one +// page of guest memory is made a dma-buf, imported and bound to a buffer, as every pool will be. Any step refused turns +// the dma-buf imports off for the run: the pools are shadow copies, as with NX2NAVI_DMABUF=0. A refusal later, for +// one span, leaves its pools without a GPU buffer (span_log counts them), so the probe asks once for what all need. +static int s_learn, s_check; +void vk_mem_dmabuf_probe(VkDevice dev, const VkPhysicalDeviceMemoryProperties *mp, int bda) { + if (!dmabuf_on) return; + const char *no = NULL; + long err = 0; + uint32_t bits = 0; + VkDeviceMemory mem = VK_NULL_HANDLE; + VkBuffer buf = VK_NULL_HANDLE; + int fd = plat_mem_dmabuf(dmabuf_any, 4096); + if (fd < 0) { + no = "udmabuf refused a dma-buf of guest memory", err = errno; + } else if (bits = dmabuf_pick_type(dev, mp, fd), dmabuf_type == UINT32_MAX) { + no = "the device offers no host-visible coherent memory type for it"; + dmabuf_close(fd); + } else { + VkImportMemoryFdInfoKHR im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, .fd = fd}; + VkMemoryAllocateFlagsInfo fi = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, .pNext = &im, + .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT}; + VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = bda ? (void *)&fi : (void *)&im, + .allocationSize = 4096, .memoryTypeIndex = dmabuf_type}; + VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT}; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = 4096, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + (bda ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0), + .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + VkMemoryRequirements req; + VkResult r = vkAllocateMemory(dev, &ai, NULL, &mem); + if (r != VK_SUCCESS) { + mem = VK_NULL_HANDLE; + dmabuf_close(fd); // a failed import leaves the descriptor with us + no = "the device refused to import it", err = r; + } else if ((r = vkCreateBuffer(dev, &bi, NULL, &buf)) != VK_SUCCESS) { + buf = VK_NULL_HANDLE; + no = "no buffer for it", err = r; + } else if (vkGetBufferMemoryRequirements(dev, buf, &req), !(req.memoryTypeBits >> dmabuf_type & 1)) { + no = "a buffer cannot be bound to its memory type", err = (long)req.memoryTypeBits; + } else if ((r = vkBindBufferMemory(dev, buf, mem, 0)) != VK_SUCCESS) { + no = "binding a buffer to it failed", err = r; + } + } + if (buf) vkDestroyBuffer(dev, buf, NULL); + if (mem) vkFreeMemory(dev, mem, NULL); + if (fd >= 0) plat_mem_dmabuf_done(dmabuf_any, 4096); + if (!no) return; + dmabuf_on = 0, shadow_on = 1; + s_learn = rt_knob("SHADOW_TABLE_STORES", "0") == 1; + s_check = cfg_bool("NX2NAVI_SHADOW_CHECK"); + rt_log("mem: dma-buf imports off: %s (%ld; memory types %#x): CPU-visible pools are host-visible copies of guest " + "memory (shadow storage), as with NX2NAVI_DMABUF=0", no, err, bits); +} + +static void span_fail(const char *what, uint64_t lo, uint64_t hi, long err) { // span_lock held + sp_fail++; + if (++sp_fail_all <= 8) + rt_log("mem: dma-buf of %#llx+%#llx: %s (%ld)%s", (unsigned long long)lo, (unsigned long long)(hi - lo), what, err, + sp_fail_all == 8 ? "; no more of these lines" : ""); +} + +// span_lock held: a new span over [lo, hi), NULL when udmabuf or the import refuses it. +static Span *span_create(VkCtx *c, uint64_t lo, uint64_t hi) { + uint64_t t0 = qpc(); + int fd = plat_mem_dmabuf(lo, hi - lo); + if (fd < 0) { + span_fail("udmabuf refused it", lo, hi, errno); + return NULL; + } + if (dmabuf_type == UINT32_MAX) { // every dma-buf of guest memory has the same types: asked once + dmabuf_pick_type(c->device, &c->mem_props, fd); + if (dmabuf_type == UINT32_MAX) { + dmabuf_close(fd); + plat_mem_dmabuf_done(lo, hi - lo); + span_fail("no host-visible coherent memory type", lo, hi, 0); + dmabuf_type = UINT32_MAX - 1; // not asked again: every span fails + return NULL; + } + } + if (dmabuf_type >= c->mem_props.memoryTypeCount) { + dmabuf_close(fd); + plat_mem_dmabuf_done(lo, hi - lo); + return NULL; + } + VkImportMemoryFdInfoKHR im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, .fd = fd}; + // A buffer with device address usage needs memory allocated for device addresses (vk_gpa.c). + VkMemoryAllocateFlagsInfo fi = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, .pNext = &im, + .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT}; + VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = vk_gpa_wanted() ? (void *)&fi : (void *)&im, .allocationSize = hi - lo, + .memoryTypeIndex = dmabuf_type}; + VkDeviceMemory mem = VK_NULL_HANDLE; + VkResult res = vkAllocateMemory(c->device, &ai, NULL, &mem); + Span *s = res == VK_SUCCESS ? calloc(1, sizeof *s) : NULL; + if (!s) { + if (res == VK_SUCCESS) vkFreeMemory(c->device, mem, NULL); + else dmabuf_close(fd); // a failed import leaves the descriptor with us; a successful one took it + plat_mem_dmabuf_done(lo, hi - lo); + span_fail("the import failed", lo, hi, res); + return NULL; + } + s->lo = lo, s->hi = hi, s->mem = mem; + if (nspans == cspans) grow((void **)&spans, &cspans, sizeof *spans); + spans[nspans++] = s; + uint64_t ns = qpc() - t0; + sp_made++, sp_made_bytes += hi - lo, sp_ns += ns, sp_live_bytes += (int64_t)(hi - lo); + if (ns > sp_max_ns) sp_max_ns = ns; + return s; +} + +// span_lock held: the span a pool's [lo, hi) is bound into, with a reference for it; NULL when none can be made. al is +// the alignment the pool's buffer needs inside the span's memory. +// A new span covers the span_size-aligned range around the pool, less what other spans hold below and above the pool: +// a pool across an edge of an older span then pins only its own pages twice, not the older span's whole range. With +// the whole aligned range, Hyrule Field's area loads left 11.5-11.7 GiB of spans counted in amdgpu's GTT use; cut back, +// 9.5-9.6 GiB, over the same 9.3 GiB of distinct pages, at the same frame rate (RX 6800 XT, 2 pairs). +static Span *span_get(VkCtx *c, uint64_t lo, uint64_t hi, uint64_t al) { + Span *best = NULL; + for (int i = 0; i < nspans; i++) + if (spans[i]->lo <= lo && hi <= spans[i]->hi && (!best || spans[i]->hi - spans[i]->lo > best->hi - best->lo)) + best = spans[i]; + if (!best) { + uint64_t a = lo / span_size * span_size, b = (hi + span_size - 1) / span_size * span_size; + for (int i = 0; i < nspans; i++) { // no span holds [lo, hi): none reaches below lo and above hi at once + const Span *s = spans[i]; + if (s->lo < lo && s->hi > a) a = s->hi < lo ? s->hi : lo; + if (s->hi > hi && s->lo < b) b = s->lo > hi ? s->lo : hi; + } + if (al > 1) a = lo - (lo - a) / al * al; // the pool's offset in the span a multiple of al + if (b - a <= dmabuf_limit) best = span_create(c, a, b); + // the aligned range may leave the memfd's reservation: the pool's own pages then + if (!best && (a != lo || b != hi) && hi - lo <= dmabuf_limit) best = span_create(c, lo, hi); + } + if (best) best->refs++; + return best; +} + +static void span_put(Span *s) { + plat_mutex_lock(&span_lock); + if (!--s->refs) s->idle_frame = __atomic_load_n(&span_frame, __ATOMIC_RELAXED); + plat_mutex_unlock(&span_lock); +} + +// Frees the spans no pool has used for SPAN_KEEP frames (all: every span without a pool). A span without a reference +// is out of the GPU's work: a pool's reference goes when its record is destroyed, after the GPU is done with it. +static void span_gc(uint64_t frame, int all) { + VkCtx *c = vk_ctx(); + if (!c || !dmabuf_on) return; + Span *gone[64]; + int ng; + do { + ng = 0; + plat_mutex_lock(&span_lock); + for (int i = 0; i < nspans && ng < 64;) + if (!spans[i]->refs && (all || frame - spans[i]->idle_frame >= SPAN_KEEP)) { + gone[ng++] = spans[i]; + sp_freed++, sp_freed_bytes += spans[i]->hi - spans[i]->lo; + sp_live_bytes -= (int64_t)(spans[i]->hi - spans[i]->lo); + spans[i] = spans[--nspans]; + } else { + i++; + } + plat_mutex_unlock(&span_lock); + for (int i = 0; i < ng; i++) { // the dma-buf goes with the memory; then its pages are let go + vkFreeMemory(c->device, gone[i]->mem, NULL); + plat_mem_dmabuf_done(gone[i]->lo, gone[i]->hi - gone[i]->lo); + free(gone[i]); + } + } while (ng == 64); +} + +static void span_log(uint64_t frame) { + if (!dmabuf_on) return; + plat_mutex_lock(&span_lock); + int64_t refs = 0; + for (int i = 0; i < nspans; i++) refs += spans[i]->refs; + rt_log("mem: dma-buf spans at frame %llu: %d live, %.0f MiB, %lld pools in them; since the last line %llu made " + "(%.0f MiB, %.1f ms, the longest %.1f ms), %llu freed (%.0f MiB), %llu refused, %llu pools left out", + (unsigned long long)frame, nspans, sp_live_bytes / 1048576.0, (long long)refs, (unsigned long long)sp_made, + sp_made_bytes / 1048576.0, qpc_ms(sp_ns), qpc_ms(sp_max_ns), (unsigned long long)sp_freed, + sp_freed_bytes / 1048576.0, (unsigned long long)sp_fail, (unsigned long long)sp_fallback); + sp_made = sp_made_bytes = sp_ns = sp_max_ns = sp_freed = sp_freed_bytes = sp_fail = sp_fallback = 0; + plat_mutex_unlock(&span_lock); +} + +// A pool's buffer bound into a span's memory; 0 when no span can be made for it. +static int import_dmabuf(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) { + r->import_lo = addr & ~0xfffull; + r->import_hi = (addr + size + 0xfff) & ~0xfffull; + uint64_t n = r->import_hi - r->import_lo; + VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT}; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = n, + .usage = pool_usage(c), .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + VkResult res = vkCreateBuffer(c->device, &bi, NULL, &r->buf); + if (res != VK_SUCCESS) { + r->buf = VK_NULL_HANDLE; + return 0; + } + VkMemoryRequirements req; + vkGetBufferMemoryRequirements(c->device, r->buf, &req); + uint64_t need = ((req.size > n ? req.size : n) + 0xfff) & ~0xfffull; + plat_mutex_lock(&span_lock); + Span *s = span_get(c, r->import_lo, r->import_lo + need, req.alignment); + int ok = s && (r->import_lo - s->lo) % (req.alignment ? req.alignment : 1) == 0 && (req.memoryTypeBits >> dmabuf_type & 1); + if (!s) sp_fallback++; + plat_mutex_unlock(&span_lock); + if (ok && (res = vkBindBufferMemory(c->device, r->buf, s->mem, r->import_lo - s->lo)) != VK_SUCCESS) ok = 0; + if (!ok) { + if (s) { + rt_log("vk: pool %#llx+%#llx: binding into its dma-buf span failed (alignment %llu, types %#x, %d)", + (unsigned long long)addr, (unsigned long long)size, (unsigned long long)req.alignment, + req.memoryTypeBits, (int)res); + span_put(s); + } + vkDestroyBuffer(c->device, r->buf, NULL); + r->buf = VK_NULL_HANDLE; + return 0; + } + r->span = s; + plat_inc32(&n_imports); + return 1; +} + +// CPU-visible storage: the guest pages themselves become the buffer's memory (VK_EXT_external_memory_host, or a +// dma-buf of them: "dma-buf imports" above). static int import_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) { if (shadow_on) return shadow_pool(c, r, addr, size); + if (dmabuf_on && import_dmabuf(c, r, addr, size)) return 1; if (!p_host_props) p_host_props = (PFN_vkGetMemoryHostPointerPropertiesEXT)vk_dpa(c->device, "vkGetMemoryHostPointerPropertiesEXT"); uint64_t al = c->min_imported_host_alignment ? c->min_imported_host_alignment : 0x1000; @@ -295,6 +598,7 @@ static void destroy_rec(VkCtx *c, PoolRec *r) { if (r->salloc) vmaDestroyBuffer(c->vma, r->buf, r->salloc), r->buf = VK_NULL_HANDLE; if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL); if (r->mem) vkFreeMemory(c->device, r->mem, NULL); + if (r->span) span_put(r->span); if (r->alloc) vmaFreeMemory(c->vma, r->alloc); for (uint32_t i = 0; i < r->csb; i++) free(r->sb[i].base); // shadow base copies shadow_forget left (shutdown) free(r->sb); @@ -357,7 +661,7 @@ static int pool_init(pool_t *p) { // The imported pages, or the shadow that stands for them, go into the device address table: shaders that read a // pointer from memory reach them there. A shadow in the table is kept up to date for the table's readers whether // or not the replay resolves it (vk_mem_shadow_sync). - if (r->kind == POOL_IMPORT && (r->mem || r->smap) && r->buf && r->import_hi > r->import_lo) { + if (r->kind == POOL_IMPORT && (r->mem || r->span || r->smap) && r->buf && r->import_hi > r->import_lo) { r->gpa_rec = vk_gpa_map(r->import_lo, r->import_hi, r->buf, r->import_lo); if (r->smap && r->gpa_rec) shadow_table_add(r); } @@ -670,12 +974,14 @@ void vk_mem_snapshot(uint64_t frame) { void vk_mem_census(uint64_t frame) { mirror_frame = frame; + __atomic_store_n(&span_frame, frame, __ATOMIC_RELAXED); + if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0); static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) - mirror_log(frame), shadow_log(frame); // mirror_log also logs the write-watch line + mirror_log(frame), shadow_log(frame), span_log(frame); // mirror_log also logs the write-watch line else if (shadow_check_on() && frame % 60 == 0) shadow_log(frame); // NX2NAVI_SHADOW_CHECK: the shadow lines at that rate without the perf trace if (frame == 1200) { // loading is over: the first window starts here @@ -1305,8 +1611,8 @@ static uint64_t s_wb_seen, s_wb_ticks; // shadow_lock: bytes the fenc static uint64_t s_q_ticks; // shadow_lock: the copy-ins' write-watch queries static int wb_no_reup(void); // Pages a shader stores into through the device address table. The two switches are read in mirror_mode. -static int s_learn; // NX2NAVI_SHADOW_TABLE_STORES: such pages get a base copy and a write-back -static int s_check; // NX2NAVI_SHADOW_CHECK: the counts below marked "check" +// s_learn (NX2NAVI_SHADOW_TABLE_STORES: such pages get a base copy and a write-back) and s_check (NX2NAVI_SHADOW_CHECK: +// the counts below marked "check") are defined with vk_mem_dmabuf_probe, which may turn shadow storage on late. static PoolRec **tpools; // shadow_lock: the pools that have such pages static int ntpools, ctpools; static uint64_t s_tab_pages; // shadow_lock: such pages, all pools @@ -2200,6 +2506,40 @@ static void mirror_mode(int discrete, int nvidia_own) { // memory (vk_gpa.c). The pools stay imports unless NX2NAVI_SHADOW asks for the copies. The open driver for the same // cards keeps the copies: it is not measured. The AppImage's play.sh has set the same for this driver since 0.6.0. shadow_on = sh && *sh ? strcmp(sh, "0") != 0 : !nvidia_own; + // Guest memory a memfd's (NX2NAVI_DMABUF, platform/linux.c): the pools are dma-buf imports ("dma-buf imports"), + // whatever the driver, unless NX2NAVI_SHADOW=1 asks for the copies. A device that cannot import dma-bufs keeps the + // copies: a shared mapping's pages are no host-pointer import on RADV. + uint64_t lim = 0; + const char *dmwhy = ""; + static char small[200]; + if (plat_mem_dmabuf_ok(&lim, &dmabuf_any, &dmwhy)) { + int forced = sh && *sh && strcmp(sh, "0"); + // The game's largest pools take about 300 MiB, each in one dma-buf: below 1 GiB (the module's default is 64 MB) + // some would get none, so the copies stay. + if (lim < (1ull << 30)) + snprintf(small, sizeof small, "udmabuf's size_limit_mb is %llu, below the 1024 a pool of the game may need " + "(/sys/module/udmabuf/parameters/size_limit_mb)", (unsigned long long)(lim >> 20)); + dmabuf_on = dmabuf_exts && !forced && lim >= (1ull << 30); + shadow_on = !dmabuf_on; + dmabuf_limit = lim; + const char *mbs = cfg_get("NX2NAVI_DMABUF_MB"); + int mb = mbs && *mbs ? atoi(mbs) : 64; + span_size = (uint64_t)(mb > 0 ? mb : 64) << 20; + if (span_size > dmabuf_limit) span_size = dmabuf_limit & ~0xfffull; + if (span_size < (2ull << 20)) span_size = 2ull << 20; + if (dmabuf_on) + rt_log("mem: guest memory is a memfd's: CPU-visible pools are imported as dma-bufs of it, in spans of %llu MiB " + "(NX2NAVI_DMABUF_MB; udmabuf's limit %llu MiB)", (unsigned long long)(span_size >> 20), + (unsigned long long)(dmabuf_limit >> 20)); + else + rt_log("mem: guest memory is a memfd's, but its pools get shadow copies: %s", + forced ? "NX2NAVI_SHADOW asks for them" + : !dmabuf_exts ? "the device cannot import dma-bufs (VK_KHR_external_memory_fd, VK_EXT_external_memory_dma_buf)" + : small); + } else if (*dmwhy) { + rt_log("mem: guest memory is not a memfd's (NX2NAVI_DMABUF): %s: CPU-visible pools are %s, as with NX2NAVI_DMABUF=0", dmwhy, + shadow_on ? "host-visible copies of guest memory (shadow storage)" : "host-pointer imports"); + } #endif if (shadow_on) rt_log("mem: shadow storage: CPU-visible pools are host-visible copies of guest memory, not imports%s", sh && *sh ? " (NX2NAVI_SHADOW)" : ""); @@ -5151,6 +5491,7 @@ void vk_mem_shutdown(void) { __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); nphys = 0; plat_mutex_unlock(&pool_lock); + span_gc(0, 1); // every pool is gone: so are the spans' references plat_mutex_lock(&gen_lock); for (uint64_t i = 0; i < GEN_CHUNKS; i++) free(gen_chunk[i]), gen_chunk[i] = NULL, free(cgen_chunk[i]), cgen_chunk[i] = NULL; plat_mutex_unlock(&gen_lock); diff --git a/tests/platform_dmabuf_test.c b/tests/platform_dmabuf_test.c new file mode 100644 index 00000000..8dd3358f --- /dev/null +++ b/tests/platform_dmabuf_test.c @@ -0,0 +1,111 @@ +// Guest memory as a memfd's (runtime/platform/linux.c, NX2NAVI_DMABUF): a dma-buf of a watched range maps the very +// pages the CPU writes; a decommit inside it zeroes them in place (the dma-buf still sees the CPU's pages), one outside +// it reads zero when committed again; the write watch keeps working. No GPU, no game files. Build and run from the +// repository root: +// clang -O2 -ffunction-sections -fdata-sections -Iruntime -c runtime/platform/linux.c -o /tmp/plat.o +// clang -O2 -Wall -Iruntime tests/platform_dmabuf_test.c /tmp/plat.o -Wl,--gc-sections -lpthread -ldl -o /tmp/dmabuf +// /tmp/dmabuf (exit 77: no /dev/udmabuf or no write watch here) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "platform/platform.h" + +#define MiB (1ull << 20) + +static uint64_t memfd_bytes(void) { // the guest memfd's allocated bytes + DIR *dir = opendir("/proc/self/fd"); + struct dirent *e; + uint64_t got = 0; + while (dir && (e = readdir(dir))) { + char path[64], link[128]; + snprintf(path, sizeof path, "/proc/self/fd/%s", e->d_name); + ssize_t k = readlink(path, link, sizeof link - 1); + struct stat st; + if (k > 0 && (link[k] = 0, strstr(link, "memfd:nx2navi-guest")) && !stat(path, &st)) got = (uint64_t)st.st_blocks * 512; + } + if (dir) closedir(dir); + return got; +} + +int main(void) { + const uint64_t a = 0x6100000000ull, n = 64 * MiB; + uint64_t pages[16], limit = 0, any = 0; + const char *why = ""; + setenv("NX2NAVI_DMABUF", "1", 0); + assert(plat_mem_reserve(a, n, 1)); + if (!plat_mem_dmabuf_ok(&limit, &any, &why)) { printf("SKIP: guest memory is not a memfd's here: %s\n", why); return 77; } + assert(any == a); + assert(plat_mem_commit(a, 16 * MiB)); + memset((void *)(uintptr_t)a, 0x11, 16 * MiB); + for (uint64_t k = 16; k == 16;) // read and reset until a call has fewer than it could take + if ((k = plat_mem_written(a, n, pages, 16, 1)) == UINT64_MAX) { puts("SKIP: asynchronous write watch unavailable"); return 77; } + assert(plat_mem_written(a, n, pages, 16, 0) == 0); + + // a dma-buf of the first 8 MiB maps the guest's own pages + int fd = plat_mem_dmabuf(a, 8 * MiB); + assert(fd >= 0); + volatile uint8_t *d = mmap(NULL, 8 * MiB, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + assert(d != MAP_FAILED); + assert(d[0] == 0x11 && d[8 * MiB - 1] == 0x11); + ((volatile uint8_t *)(uintptr_t)a)[4096 + 5] = 0x22; + assert(d[4096 + 5] == 0x22); + d[3 * 4096] = 0x33; // as a GPU write: the guest sees it, the write watch does not + assert(((volatile uint8_t *)(uintptr_t)a)[3 * 4096] == 0x33); + assert(plat_mem_written(a, n, pages, 16, 1) == 1 && pages[0] == a + 4096); + + // protection changes (a freeze, an armed lazy range) leave the dma-buf alone + assert(plat_mem_protect(a, 2 * MiB, PLAT_PROT_READ)); + assert(d[4096 + 5] == 0x22); + assert(plat_mem_protect(a, 2 * MiB, PLAT_PROT_RW)); + + // a decommit inside the dma-buf: zero in place, the same pages for both + assert(plat_mem_decommit(a + 4096, 4096)); + assert(plat_mem_commit(a + 4096, 4096)); + assert(((volatile uint8_t *)(uintptr_t)a)[4096 + 5] == 0 && d[4096 + 5] == 0); + ((volatile uint8_t *)(uintptr_t)a)[4096 + 6] = 0x44; + assert(d[4096 + 6] == 0x44); + + // a decommit that straddles the dma-buf's end: zero both sides when committed again + assert(plat_mem_decommit(a + 8 * MiB - 4096, 2 * 4096)); + assert(plat_mem_commit(a + 8 * MiB - 4096, 2 * 4096)); + assert(((volatile uint8_t *)(uintptr_t)(a + 8 * MiB - 1))[0] == 0 && d[8 * MiB - 1] == 0); + assert(((volatile uint8_t *)(uintptr_t)(a + 8 * MiB))[0] == 0); + + // a decommit outside any dma-buf: the pages leave the memfd and read zero + assert(plat_mem_decommit(a + 12 * MiB, MiB)); + assert(plat_mem_commit(a + 12 * MiB, MiB)); + assert(((volatile uint8_t *)(uintptr_t)(a + 12 * MiB))[77] == 0); + + // a dma-buf over pages nothing committed yet allocates them; when it is gone they go back, the committed ones stay + uint64_t before = memfd_bytes(); + int fd2 = plat_mem_dmabuf(a + 32 * MiB, 8 * MiB); + assert(fd2 >= 0); + uint64_t pinned = memfd_bytes(); + close(fd2); + plat_mem_dmabuf_done(a + 32 * MiB, 8 * MiB); + uint64_t after = memfd_bytes(); + printf("memfd: %.1f MiB, %.1f MiB with an 8 MiB dma-buf over uncommitted pages, %.1f MiB after it\n", before / 1048576.0, + pinned / 1048576.0, after / 1048576.0); + assert(pinned >= before + 8 * MiB && after == before); + assert(plat_mem_commit(a + 32 * MiB, 4096)); + assert(((volatile uint8_t *)(uintptr_t)(a + 32 * MiB))[9] == 0); + + munmap((void *)d, 8 * MiB); + close(fd); + plat_mem_dmabuf_done(a, 8 * MiB); + assert(((volatile uint8_t *)(uintptr_t)a)[4096 + 6] == 0x44 && ((volatile uint8_t *)(uintptr_t)a)[2 * MiB] == 0x11); + + // outside the reservation, or not whole pages: no dma-buf + assert(plat_mem_dmabuf(a + n - 4096, 2 * 4096) < 0); + assert(plat_mem_dmabuf(a + 1, 4096) < 0); + *(volatile uint8_t *)(uintptr_t)(a + 5 * MiB) = 1; + assert(plat_mem_written(a, n, pages, 16, 1) >= 1); + puts("dma-buf of guest memory: same pages both ways, decommits zero in place inside and punch outside, write watch kept"); + return 0; +} From afe44e0e5488e27d17743ff7db785656d0a1d096 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:09:42 -0500 Subject: [PATCH 05/15] Let the device address table read a new VRAM mirror only once the mirror's first copy has run. A mirror is made while a submission is recorded, and its first copy of the pool is in that submission's prologue. The table's pages moved to the mirror at once, and the table is memory the shaders read as they run: every part already handed to the queue read the mirror through it before the copy had run, memory nothing had written yet and, on a card over its budget, a block the driver had not made resident for those parts. A GTX 1650 4 GB lost its device that way at area loads in every session of 0.7.1 (#507): six reports of a GPU read fault at byte 0 of a mirror made in the faulting submission, each the first byte of a new 2 MiB-aligned block, with mirrors being made over the budget, and none with the table kept empty. The mirror code is the same in 0.9.2. Now the pages keep reading the imported buffer and take the mirror as their twin, so what shaders write through the table goes to both and what they read is guest memory, as before the mirror existed. Once the GPU has finished the submission that holds the copy, the pages move to the mirror with the imported buffer as the twin. gpa_late = 0 moves them at once, as before. tests/gpa_check.py covers the new table step; the mirror side was compiled and run through --probe-vkmem, not played. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 5 +++ runtime/cfg.c | 2 +- runtime/vk_gpa.c | 48 +++++++++++++++++++++------- runtime/vk_gpa.h | 4 +++ runtime/vk_memory.c | 77 +++++++++++++++++++++++++++++++++++++++++---- tests/gpa_check.c | 23 ++++++++++++++ 6 files changed, 140 insertions(+), 19 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 579f560b..076febae 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -976,3 +976,8 @@ # those second writes cost about 3 ms of GPU time a frame. 0 (default): every mirror sends them. A pool that needs them # and is left without shows as foliage stretched into columns or flickering. # gpa_twin_max_kb = 0 +# 1 (default): shaders that follow the game's pointers read a memory pool's new copy in video memory only once that +# copy has been filled; until then they read the game's own memory. 0: they read the new copy at once, as up to 0.9.2, +# where a card over its video memory budget could lose the device at an area load (a GPU read fault at the first byte +# of the new copy). +# gpa_late = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index 8075fbf4..0d843815 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -47,7 +47,7 @@ static const char *const known[] = { "DESC_TEMPLATE", "DEVICE_PROFILE", "DMABUF", "DMABUF_MB", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DRAW_CRUMBS", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "GPU_TIMELINE", "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", - "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", + "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_LATE", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_LIVE", "HEAP_LAZY", "HDR_PEAK_NITS", "HDR_UI_NITS", // retired: the HDR look's settings of earlier versions, now fixed (vk_hdr.c, shaders/hdr_tonemap.frag); read by // nothing, listed so that a file which still sets them logs no unknown key diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index 32cec945..9daf0a14 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -319,20 +319,20 @@ void vk_gpa_clear(uint64_t lo, uint64_t hi) { blocks_release(p0, p1); } -static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { - if (!vk_gpa_on() || !buf) return old_rec; - uint64_t dev = vk_gpa_device_address(buf); - uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; - plat_mutex_lock(&gpa_lock); - // the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too. - // NX2NAVI_GPA_TWIN=0 (diagnosis): no twin, the writes go to the mirror alone, as before patch 0034. +// NX2NAVI_GPA_TWIN=0 (diagnosis): no twin, the writes go to the mirror alone, as before patch 0034. +static int twins_on(void) { static int twins = -1; if (twins < 0) { const char *e = cfg_get("NX2NAVI_GPA_TWIN"); twins = !(e && !strcmp(e, "0")); } - uint64_t twin = old_rec && twins && keep_twin ? (uint64_t)G.cpu[REC_WORD + old_rec * REC_WORDS] | - (uint64_t)G.cpu[REC_WORD + old_rec * REC_WORDS + 1] << 32 - : dev - buf_lo; - uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0; - if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } + return twins; +} + +// gpa_lock held: the first address of record rec (the buffer its pages read). +static uint64_t rec_first(uint32_t rec) { + return (uint64_t)G.cpu[REC_WORD + rec * REC_WORDS] | (uint64_t)G.cpu[REC_WORD + rec * REC_WORDS + 1] << 32; +} + +// gpa_lock held: the pages of [p0, p1) that name old_rec (or none) name r from here on. +static void pages_move(uint64_t p0, uint64_t p1, uint32_t old_rec, uint32_t r) { for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 1); if (!w) { @@ -342,11 +342,35 @@ static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, } if (!G.cpu[w] || G.cpu[w] == old_rec) page_put(w, r); } +} + +static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { + if (!vk_gpa_on() || !buf) return old_rec; + uint64_t dev = vk_gpa_device_address(buf); + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + // the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too. + uint64_t twin = old_rec && twins_on() && keep_twin ? rec_first(old_rec) : dev - buf_lo; + uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0; + if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } + pages_move(p0, p1, old_rec, r); plat_mutex_unlock(&gpa_lock); if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); return r; } +uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { + if (!old_rec || !buf || !vk_gpa_on() || !twins_on()) return 0; + uint64_t dev = vk_gpa_device_address(buf); + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + uint32_t r = dev ? rec_alloc(rec_first(old_rec), dev - buf_lo) : 0; + if (r) pages_move(p0, p1, old_rec, r); + plat_mutex_unlock(&gpa_lock); + if (r) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); + return r; +} + uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { return remap(lo, hi, old_rec, buf, buf_lo, 1); } diff --git a/runtime/vk_gpa.h b/runtime/vk_gpa.h index f5e1d4e1..29df805e 100644 --- a/runtime/vk_gpa.h +++ b/runtime/vk_gpa.h @@ -24,5 +24,9 @@ uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, // The same without the twin: the pages name buf alone (a pool whose VRAM mirror is evicted goes back to its imported // buffer). uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); +// The pages of [lo, hi) that name old_rec keep reading old_rec's buffer and get buf as their twin: shaders read the +// pool where they did and write buf too (a VRAM mirror until its first copy has run, vk_memory.c mirror_create). +// Returns the new record, or 0 when there is none to give or twins are off (old_rec then stays as it is). +uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); void vk_gpa_report(void); void vk_gpa_shutdown(void); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 33308bc8..c1f1e447 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -61,6 +61,8 @@ typedef struct PoolRec { // VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only. uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED uint32_t gpa_rec; // POOL_IMPORT: its record in the device address table (vk_gpa.c), 0 none + uint64_t gpa_at; // pool_lock: the submission after which the table reads the mirror (gpa_switch); + // 0 when it does already, or there is no mirror struct GpaMap { uint64_t lo, hi; uint32_t rec; } *vmaps; // POOL_VIRT: its mapped ranges in the table int nvmaps, cvmaps; VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf @@ -2756,6 +2758,52 @@ static int twin_wanted(const PoolRec *r) { return !twin_max || (int64_t)(r->import_hi - r->import_lo) <= twin_max; } +// ---- when the device address table starts reading a new mirror (NX2NAVI_GPA_LATE, default 1; 0 = at once, as before) +// A mirror is made while the replay records a submission, and its first copy of the pool is in that submission's +// prologue. The table is memory the shaders read as they run, so with its pages moved to the mirror at once, every +// part already handed to the queue (the submissions before, and the earlier parts of this one) read the mirror through +// the table before that copy had run: memory nothing had written yet, and on a card over its budget a block the +// driver had not made resident for those parts. A GTX 1650 4 GB lost its device that way at area loads in every +// session: six reports of a GPU read fault at byte 0 of a mirror made in the faulting submission, each the first byte +// of a new 2 MiB-aligned block, with mirrors being made over the budget; with the table kept empty the same routes ran +// without a loss. +// Now the pages keep reading the imported buffer and name the mirror as their twin (vk_gpa_remap_writes): what shaders +// write through the table goes to both, so the mirror misses nothing, and what they read is guest memory, as before +// the mirror existed. Once the GPU has finished the submission whose prologue holds the copy, the pages move to the +// mirror with the imported buffer as the twin (gpa_switch, at the start of a later submission). In between, a shader +// that reads through the table what another wrote through a buffer binding sees guest memory, which gets such a write +// at the write-back; data written through the table itself, as the game's foliage, instance and light records are, +// is read where it was written. +static int n_gpa_wait; // pool_lock: mirrors whose pages still read the imported buffer (PoolRec.gpa_at) +static uint64_t n_gpa_late; // replay: pages moved to their mirror after its first copy, so far + +static int gpa_late(void) { + static int on = -1; + if (on < 0) { + on = rt_knob("GPA_LATE", "1") == 1; + rt_log("mem: the device address table reads a new VRAM mirror %s (NX2NAVI_GPA_LATE)", + on ? "once the mirror's first copy has run" : "at once, as before"); + } + return on; +} + +// Replay thread, at the start of a submission: the mirrors whose first copy the GPU has finished take the table's reads. +static void gpa_switch(void) { + if (!__atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) return; + uint64_t done = vk_sync_fence_done(); + plat_mutex_lock(&pool_lock); + for (int i = 0; i < nm && n_gpa_wait; i++) { + PoolRec *r = mlist[i]; + if (!r->gpa_at || done < r->gpa_at) continue; + r->gpa_at = 0, n_gpa_wait--; + if (!r->gpa_rec || !r->mbuf) continue; + r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) + : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + n_gpa_late++; + } + plat_mutex_unlock(&pool_lock); +} + // Replay thread: the pool's mirror on its first buffer use. static void mirror_create(PoolRec *r) { VkCtx *c = vk_ctx(); @@ -2848,16 +2896,28 @@ static void mirror_create(PoolRec *r) { r->acc_sub = msub; r->mstate = MS_ON; r->asked_room = 0; - // The device address table names the mirror from now on: the GPU's writes go there, and the sync copies the CPU's. - // With a twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. + // The device address table names the mirror: the GPU's writes go there, and the sync copies the CPU's. With a + // twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. Reads + // move to the mirror once its first copy has run (gpa_late, below). r->twin = (uint8_t)twin_wanted(r); if (!r->twin) { plat_inc64(&m_untwinned_n); plat_add64(&m_untwinned_bytes, (int64_t)n); } - if (r->gpa_rec) - r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) - : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + if (r->gpa_rec) { + plat_mutex_lock(&pool_lock); + uint32_t late = gpa_late() ? vk_gpa_remap_writes(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) : 0; + if (late) { + uint64_t f = vk_exec_recording_fence(); + r->gpa_rec = late; + r->gpa_at = f ? f : vk_sync_issued() + 1; // the submission being recorded: its prologue holds the copy + n_gpa_wait++; + } else { + r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) + : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + } + plat_mutex_unlock(&pool_lock); + } range_add(&pend, &npend, &cpend, r, 0, n); mrep.first_n++, mrep.first_bytes += n; plat_inc64(&m_live_n); @@ -2866,6 +2926,7 @@ static void mirror_create(PoolRec *r) { static void mirror_forget(PoolRec *r) { __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); + if (r->gpa_at) r->gpa_at = 0, n_gpa_wait--; // the caller takes the table's pages off the mirror for (int i = 0; i < nm; i++) if (mlist[i] == r) { memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist); @@ -3932,7 +3993,7 @@ static void hot_check(void) { } void vk_mem_mirror_begin(void) { - if (vk_mirror_on) hot_check(); + if (vk_mirror_on) gpa_switch(), hot_check(); if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions if (!freeze_idle) return; plat_mutex_lock(&gen_lock); @@ -4387,6 +4448,10 @@ static void mirror_log(uint64_t frame) { "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; flushed and not copied (write-watched) %.3f MiB%s", (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M, a.flush_skipped / f / M, capl); + if (n_gpa_late || __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) + rt_log("mem: mirror frame %llu: the device address table took up %llu mirrors after their first copy so far, %d " + "wait for theirs (NX2NAVI_GPA_LATE)", (unsigned long long)frame, (unsigned long long)n_gpa_late, + __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)); } // After vram_log: every heap's usage as the driver reports it (VK_EXT_memory_budget) beside VMA's share of it, VMA by diff --git a/tests/gpa_check.c b/tests/gpa_check.c index 2d194db0..7bda3d39 100644 --- a/tests/gpa_check.c +++ b/tests/gpa_check.c @@ -125,6 +125,29 @@ int main(void) { CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "a plain page is its own twin"); gpu_done(); + // a new mirror takes the writes first and the reads once its first copy has run (vk_memory.c mirror_create, + // gpa_switch): until then the pages read the imported buffer and the mirror is their twin + VkBuffer m1 = new_buf(); + uint32_t w0 = vk_gpa_remap_writes(a0, a1, r0, m1, a0); + CHECK(w0 && w0 != r0, "a new mirror gets a record for the writes"); + expect(a0, a1, b0, a0, "a new mirror before its first copy: reads stay on the imported buffer"); + CHECK(read_through(a0 + 4096, 1) == dev_of(m1) + 4096 && read_through(a1 - 4096, 1) == dev_of(m1) + (a1 - 4096 - a0), + "a new mirror before its first copy is the twin, which takes the writes"); + gpu_done(); + r0 = vk_gpa_remap(a0, a1, w0, m1, a0); + expect(a0, a1, m1, a0, "a new mirror after its first copy: reads move to it"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "and the imported buffer is the twin again"); + // evicted while it still waits for its first copy: the pages go back to the imported buffer alone + r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0); + gpu_done(); + VkBuffer m2 = new_buf(); + w0 = vk_gpa_remap_writes(a0, a1, r0, m2, a0); + r0 = vk_gpa_remap_plain(a0, a1, w0, b0, a0); + expect(a0, a1, b0, a0, "a mirror gone before its first copy"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "leaves the imported buffer as its own twin"); + CHECK(!vk_gpa_remap_writes(a0, a1, 0, m2, a0), "a pool without a record gets none for a mirror's writes"); + gpu_done(); + // an unmapped range reads the zero buffer at once, and its blocks serve no other range while the GPU may still read // the old L1 entries uint32_t old_blk[3] = {block_of(a0), block_of(a0 + 16 * MiB), block_of(a0 + 32 * MiB)}; From 823e0cdcd423749e426169b83a0ef001aa9e03f8 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:13:40 -0500 Subject: [PATCH 06/15] Raise the pipeline cache's size limit to 1024 MB, and leave never-drawn list pipelines to their first draw on a machine short of memory. The limit was 512 MB, and a file over it starts the run empty until a play file replaces it. With the pipeline list of 0.9.1 one run builds more than that: on an RTX 2080 Ti (#722) pipeline-cache.bin held 551 MB and the play file the run then wrote 541 MB, so both were dropped at every start, 7,247 of 8,531 pipelines were built again each time (17 s of the boot) and 541 MB were written in every session. Seven of the players' logs read here have a file over 512 MB, five of them between 517 and 691 MB. At 1024 MB those are read, the play file is folded in once and the loop ends; a start that still finds both files over the limit now says so. pipe_seed_boot (auto, 1, 0) decides whether the boot builds the pipelines of the shared list that the install has never drawn. Each is a pipeline the driver holds for the whole run: the boot's pipeline stage went from about 1.6 GB of commit on 0.8.0 to 3.9-4.4 GB on NVIDIA cards with the list (1.5-1.8 GB on AMD's driver), and 0.3-0.6 GiB of video memory. auto, the default, builds them as before except on a machine that starts with less commit free than the game takes in play, the start's own memory warning: there they are made at their first draw from the pipeline cache and recorded as the install's own. Mesh pipelines are built at boot either way. Compiled; not run in the game. Co-Authored-By: Claude Fable 5.2 --- AUDIT.md | 2 +- nx2navi.ini.example | 13 +++++++++-- runtime/cfg.c | 8 +++---- runtime/vk_pipeline.c | 53 ++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index c474a278..bfa900a8 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -1150,7 +1150,7 @@ back. | 115 | Pipeline layouts, set layouts, update templates | `P.lock` | None found | Shutdown | | 116 | Stage libraries `libs[1024]` chained; failed builds cached as null | `P.lock` | None found | Shutdown (`libs_destroy`) | | 117 | Library and pipeline job queues | `P.lock` | None found | When built; dropped at shutdown | -| 118 | `VkPipelineCache` in memory and `pipeline-cache.bin` | `cache_rw`, `save_lock` | File > `NX2NAVI_PIPELINE_CACHE_MAX_MB` (512) starts empty; in memory: the driver's, none found | Saved by "pipe autosave" at most every 20 s and at exit (waits ≤ 5 s) | +| 118 | `VkPipelineCache` in memory and `pipeline-cache.bin` | `cache_rw`, `save_lock` | File > `NX2NAVI_PIPELINE_CACHE_MAX_MB` (1024) starts empty; in memory: the driver's, none found | Saved by "pipe autosave" at most every 20 s and at exit (waits ≤ 5 s) | | 119 | Pipeline binaries `pipeline-binaries.bin` | `PB.lock` | `NX2NAVI_PIPE_BINARIES_MAX_MB` 1024; `NX2NAVI_PIPE_BINARIES` default 0 | — | | 120 | `pipeline-keys.bin` and its `seen` set | `K.io` | None found (append-only) | Never | | 121 | `compute-used.bin` | `cu_io` | `NX2NAVI_COMPUTE_KEEP` 512 | — | diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 076febae..2f107f78 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -188,8 +188,10 @@ # build_reserve_mb = 64 # 1: the log says how many pipelines came from pipeline-cache.bin; 2 also logs every compute pipeline built. 0: off. # pipe_feedback = 1 -# pipeline-cache.bin larger than this (MB) starts empty and is rewritten (0: no limit). -# pipeline_cache_max_mb = 512 +# pipeline-cache.bin larger than this (MB) starts empty and is rewritten (0: no limit). With the pipeline list of +# 0.9.1 and later a full cache is about 550 MB on an NVIDIA card, so the limit has to stay well above that: under +# it the cache is thrown away at every start and the pipelines are built again each time. Default: 1024. +# pipeline_cache_max_mb = 1024 # 1: the pipelines a run compiles go into pipeline-cache-play.bin, a small file of its own, so pipeline-cache.bin # (hundreds of MB) is not written again while you play; the small file is folded into the large one at a start once # it has grown. 0: one file, written whole whenever a run made a new pipeline. @@ -211,6 +213,13 @@ # Count the shared pipeline list that comes with the release (seed-pipeline-keys.bin) as known pipelines, so a new # install builds them at the start instead of at their first draw. 0: only the pipelines this install drew. # pipe_seed = 1 +# Whether the start builds the pipelines of that list which this install has never drawn. They take 2 to 4 GB of +# memory (commit) on an NVIDIA card and about 1.5 GB on an AMD one for the whole run, drawn or not. +# auto (default): built at the start, except on a PC that starts with less memory free than the game takes in play +# (the start's "this machine has ... MB of memory left for the game" warning): there each is made at its first +# draw, from the pipeline cache, and built at the start from then on. +# 1: always built at the start. 0: never; a first visit to an area can then hold a frame for a moment. +# pipe_seed_boot = auto # 1: the start waits until every known pipeline is built. Default 0: with the graphics pipeline library the start # builds them for at most 20 seconds and the rest in the background while you play. # prewarm_full = 0 diff --git a/runtime/cfg.c b/runtime/cfg.c index 0d843815..c50354f3 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -59,7 +59,7 @@ static const char *const known[] = { "MOTION_GUARD_DEGREES", "MOTION_POLICY_TRACE", "SELECTIVE_MOTION", "SELECTIVE_MOTION_RAW", // motion_owner.c, vk_fg_pose.inc "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", - "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_SAME_BIG", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC", + "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SEED_BOOT", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_SAME_BIG", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC", "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "POOL_PRESNAP_RUNS", "SUBMIT_FINISH_GPU", "SVC_WAITS", "SNAP_WIDEN", "SUBMIT_FINISH_RESULTS", "SUBMIT_FINISH_KINDS", "FOLIAGE", "MIRROR_STAGE", "FLUSH_SKIP_CLEAN", "MIRROR_HOT", "MESH_EMULATION", "MESH_EMU_MB", "MESH_EMU_TASKS", "MESH_EMU_BATCH", "GUEST_OPT", "LOG_TEX_NEW", "DUMP_KEY", "VEC_INDEX_FIX", "PASS_TIMES", @@ -141,7 +141,7 @@ static const struct { const char *key, *section; } sections[] = { {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"PIPE_FRAME_WAIT_MS", "performance"}, {"PIPE_SKIP_STEADY", "performance"}, {"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"}, - {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"LINE_WIDTH", "performance"}, {"POOL_SHOWN_FIRST", "performance"}, {"TEX_REUSE_NAMED", "performance"}, {"TEX_OVERWRITTEN_USED", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PIPE_SEED", "performance"}, + {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"LINE_WIDTH", "performance"}, {"POOL_SHOWN_FIRST", "performance"}, {"TEX_REUSE_NAMED", "performance"}, {"TEX_OVERWRITTEN_USED", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PIPE_SEED", "performance"}, {"PIPE_SEED_BOOT", "performance"}, {"COMPUTE_KEEP", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_SPLIT_FILL", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"WATER_PROJECTIVE", "performance"}, @@ -298,10 +298,10 @@ static const CfgMeta metas[] = { {"GATE_WAIT_MS", "10", "0..100", RESTART}, // vk_exec.c vk_exec_gate_wait {"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART}, {"LIB_WARM", "1", "0|1|force", RESTART}, // vk_pipeline.c lw_start: the driver's cache of stage libraries, warmed when cold - {"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PIPE_SEED", "1", "0|1", RESTART}, // vk_pipeline.c keys_open_seed + {"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PIPE_SEED", "1", "0|1", RESTART}, {"PIPE_SEED_BOOT", "auto", "0|1|auto", RESTART}, // vk_pipeline.c keys_open_seed {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "", "0|1", RESTART}, // vk_caps.c vk_caps_gpl: by the driver, off with AMD's for RDNA 2 and older {"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART}, - {"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, + {"PIPELINE_CACHE_MAX_MB", "1024", "0..65536", RESTART}, {"FFX_PIPE_CACHE", "1", "0|1", RESTART}, // vk_ffx.cpp vk_ffx_proc_addr: FSR 3.1's pipelines through the pipeline caches {"PIPELINE_CACHE_PLAY", "1", "0|1", RESTART}, {"PIPELINE_CACHE_PLAY_MB", "64", "1..65536", RESTART}, // vk_pipeline.c "the play cache": what a run compiles goes into a file of its own {"PIPE_BINARIES", "0", "0|1", RESTART}, {"ONESHOT_WAIT_MS", "30000", "0..60000", RESTART}, {"ONESHOT_BUDGET", "1", "0|1", RESTART}, {"PIPE_BINARIES_MAX_MB", "1024", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART}, diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 37f59814..36040e6e 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -46,6 +46,7 @@ #include "mesh_emu_spirv.h" #include "gfx.h" #include "host.h" +#include "mem_need.h" #include "nvn_hmap.h" #include "platform/platform.h" #include "prp_file.h" @@ -746,8 +747,12 @@ static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) { // The data of the pipeline cache file at path for vkCreatePipelineCache (malloc'd, its size in *size), NULL when the // run starts without it, which the log says with the file's size. Cache data whose header names another device or // driver is dropped before Vulkan sees it. -// NX2NAVI_PIPELINE_CACHE_MAX_MB (default 512; 0 no limit): a larger file starts the run empty, and the run's -// saves replace it. The file only grows: entries of earlier translators and builds stay, and on NVIDIA +// NX2NAVI_PIPELINE_CACHE_MAX_MB (default 1024; 0 no limit): a larger file starts the run empty, and the run's +// saves replace it. The limit has to stay above what one run builds, or nothing ever replaces the file: at 512, with +// the pipeline list of 0.9.1 (8,849 records), an RTX 2080 Ti's file held 551 MB and the play file the run then wrote +// 541 MB, so both were dropped at every start, 7,247 of 8,531 pipelines were built again in each (17 s of the +// boot) and 541 MB were written again in each session (cache_load says so when both files are over). The file only +// grows: entries of earlier translators and builds stay, and on NVIDIA // (621.01) an entry another executable wrote is neither used nor written again, so a renamed or updated exe // compiles while the file keeps its size (2026-11-09: 650 MB, 15-18 % of the pipelines read from it). The // whole file sits in process memory twice at the boot (this read and the driver's copy) and is copied out @@ -830,6 +835,10 @@ static void cache_load(VkCtx *c) { int want = cfg_bool("NX2NAVI_PIPELINE_CACHE_PLAY") && !prep_mode, on = want && P.cache_control; size_t psize = 0, pkept = 0; void *pdata = cache_data_read(c, P.play_path, &psize); + if (base_over && cache_over_cap) // what one run builds is over the limit: nothing will ever replace either file + rt_log("vk_pipeline: both pipeline cache files are over NX2NAVI_PIPELINE_CACHE_MAX_MB, so what one run builds does " + "not fit it: this start builds every pipeline again, and so will each one after it until the limit is " + "raised above the play file's size (pipeline_cache_max_mb in nx2navi.ini; 0 = no limit)"); VkPipelineCache play = psize || on ? cache_make(c, pdata, psize, &pkept) : VK_NULL_HANDLE; free(pdata); uint64_t fold_at = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_PLAY_MB"), NULL, 10) << 20; @@ -4173,6 +4182,7 @@ static struct { uint32_t pend_n, pend_cap; uint8_t *file; // the file as read at boot, until the prewarm is done with it size_t file_n; // records in it + size_t own_n; // of them, the first ones: this install's own (keys_open); the shared list's follow PlatMutex io; // one writer // prebuild progress (under P.lock) int loading; // the prewarm still queues records @@ -4276,6 +4286,7 @@ static void keys_open_seed(void); static void keys_open_own(void); static void keys_open(void) { keys_open_own(); + K.own_n = K.file_n; if (K.path[0]) keys_open_seed(); } @@ -5287,6 +5298,30 @@ void vk_pipeline_prewarm(void) { __atomic_store_n(&pw_phase, 3, __ATOMIC_RELEASE); } +// NX2NAVI_PIPE_SEED_BOOT (auto, the default; 1; 0): whether the boot builds the pipelines of the shared list that this +// install has never drawn (keys_open_seed), 1, or leaves them to their first draw, 0. Built at boot they are there +// when a draw first needs them, which is what the list is for, and every one is a pipeline the driver holds for the +// whole run: with the list of 0.9.1 the boot's pipeline stage went from about 1.6 GB of commit (0.8.0, 3,256 +// pipelines on an RTX 5080) to 3.9-4.4 GB (8,531 on an RTX 2080 Ti, 6,977 on an RTX 5070 Ti) and 0.3-0.6 GiB of +// video memory, 1.5-1.8 GB of commit on AMD's driver, of which the list's share is half to nearly all. On a machine +// that has that to spare it buys the first visit of every area without a wait. On one that starts short of the +// commit the game takes in play (mem_need.h, the start's own warning) it is memory taken from the game for +// pipelines that may never be drawn, and running out of commit ends the game (a failed allocation or a lost device). +// auto leaves them out there and only there. A pipeline left out is made at its first draw as any new one is, from +// pipeline-cache.bin when the preparation built it (0.5-0.8 ms on a worker), and is the install's own from then on: +// the next start builds it at boot. Mesh pipelines are built at boot either way: they have no fast link, and a draw +// waits for the whole build. The list's pairs still count in the vertex output unions, so the modules stay the ones +// the preparation built. +static int seed_boot(void) { + static int on = -1; + if (on >= 0) return on; + const char *e = cfg_get("NX2NAVI_PIPE_SEED_BOOT"); + if (e && (!strcmp(e, "0") || !strcmp(e, "1"))) return on = e[0] == '1'; + double proc, used, limit, need = strtod(cfg_value("NX2NAVI_MEM_NEED_MB"), NULL); + plat_commit_info(&proc, &used, &limit); + return on = !mem_need_short(proc, used, limit, need); +} + static void prewarm_run(void) { const char *gpu = rt_config()->gpu; uint32_t pr_hist[16] = {0}, fs_hist[4] = {0}; // library state of the recorded pipelines (lb_start) @@ -5354,7 +5389,8 @@ static void prewarm_run(void) { return; } pm_set(); - uint32_t queued = 0, dropped = 0, known = 0; + uint32_t queued = 0, dropped = 0, known = 0, listed = 0; + int at_boot = seed_boot(); plat_mutex_lock(&P.lock); K.t0 = t1; K.loading = 1; @@ -5385,6 +5421,10 @@ static void prewarm_run(void) { pr_hist[(k.clamp & 1) | (k.poly_mode & 3) << 1 | (k.minus_w & 1) << 3]++; fs_hist[k.samples >= 8 ? 3 : k.samples >= 4 ? 2 : k.samples >= 2 ? 1 : 0]++; } + if (!at_boot && i >= K.own_n && !is_mesh(&s)) { // of the shared list alone: made at its first draw (seed_boot) + listed++; + continue; + } uint64_t h = key_hash(&k, sizeof k); int q = 0; if (pipe_find(&k, sizeof k, h)) known++; @@ -5392,6 +5432,13 @@ static void prewarm_run(void) { } K.loading = 0; plat_mutex_unlock(&P.lock); + if (listed) { + const char *e = cfg_get("NX2NAVI_PIPE_SEED_BOOT"); + rt_log("vk_pipeline: prewarm: %u records of the shared pipeline list that this install has not drawn are left to " + "their first draw, when pipeline-cache.bin gives them (NX2NAVI_PIPE_SEED_BOOT=%s%s)", listed, + e && !strcmp(e, "0") ? "0" : "auto", + e && !strcmp(e, "0") ? "" : ": this machine started with less commit free than the game takes in play"); + } free(ids), free(sh); __atomic_store_n(&pw_pipe_done, 0, __ATOMIC_RELAXED); From 222a025c521cba72a13516a1a41ea8ba41dc4f44 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:18:25 -0500 Subject: [PATCH 07/15] Give back what a host thread keeps for itself when it ends. Every guest thread is a host thread, and the game makes and ends them all through play. A thread that slept through plat_wait_ns kept a waitable timer, one that read or wrote a file an event for each, and one that decoded zstd through the host a decoder context of about 100 KiB; nothing closed or freed them when the thread ended, and a detached thread's own record stayed allocated. They are now given back when the thread's function returns: the handles in win32.c thread_main, the decoder context through plat_thread_end_hook, and the record by whichever of the thread and its detach comes last. tests/thread_end_check.py runs 416 such threads and compares the process's handle count before and after. tests/sync_wake_check.c gets the two stand-ins kernel.c has needed since the service wait log, so that check builds and passes again. The Linux file only gains the hook call and was not compiled here. Co-Authored-By: Claude Fable 5.2 --- docs/building.md | 5 ++ runtime/ov_zstd.c | 16 ++++++- runtime/platform/common.c | 17 +++++++ runtime/platform/linux.c | 1 + runtime/platform/platform.h | 5 ++ runtime/platform/win32.c | 35 +++++++++++--- tests/sync_wake_check.c | 3 +- tests/thread_end_check.c | 91 +++++++++++++++++++++++++++++++++++++ tests/thread_end_check.py | 48 +++++++++++++++++++ 9 files changed, 212 insertions(+), 9 deletions(-) create mode 100644 tests/thread_end_check.c create mode 100644 tests/thread_end_check.py diff --git a/docs/building.md b/docs/building.md index c9985f4c..825112c1 100644 --- a/docs/building.md +++ b/docs/building.md @@ -395,6 +395,11 @@ waits on its event and leaves the others asleep, a wait on several handles repor the wait on it, timed waits keep their time, a cancel ends any wait, and 4,000 signals to eight looping waiters are each seen. It runs a second time with `NX2NAVI_SYNC_WAKE_ALL=1`. No game files. +A change to what a host thread gives back when it ends (`runtime/platform/win32.c` `thread_main`, +`plat_thread_end_hook`) passes `python tests\thread_end_check.py`, which builds the platform layer alone: 416 threads +sleep, read a file and end, half joined and half detached; the end hook runs once on each, and on Windows the process +holds as many handles afterwards as before (each thread's sleep timer and read event are closed). No game files. + A change to the log queue (`runtime/logq.c`) passes `python tests\logq_check.py`, which builds it with the platform layer and a destination of its own: 160,000 lines from eight threads arrive whole and in each thread's order, a line logged while the queue's lock is taken waits and does not pass the lines queued before it, a destination that takes diff --git a/runtime/ov_zstd.c b/runtime/ov_zstd.c index db64c8fa..8911ec67 100644 --- a/runtime/ov_zstd.c +++ b/runtime/ov_zstd.c @@ -113,10 +113,19 @@ static ZSTD_DDict *dict_for(uint64_t guest, uint64_t content, uint64_t size) { return dd; } +// A thread's one-shot decoder context (host_decode), freed when the thread ends (plat_thread_end_hook): the game +// makes and ends threads all through play, and each that had decoded left its context behind, about 100 KiB. +static __thread ZSTD_DCtx *dctx; +static volatile int32_t dctx_hooked; + +static void dctx_thread_end(void) { + if (dctx) ZSTD_freeDCtx(dctx); + dctx = NULL; +} + // Decodes on this thread's host context. Returns libzstd's result; *ok 0 when the inputs cannot be taken. static size_t host_decode(uint64_t dst, uint64_t cap, uint64_t src, uint64_t size, uint64_t dict, uint64_t dict_size, uint64_t ddict, int *ok) { - static __thread ZSTD_DCtx *dctx; *ok = 0; if ((cap && !dst) || !src || src + size > RT_ADDRESS_LIMIT) return 0; // dst: decode() checks a guest one ZSTD_DDict *dd = NULL; @@ -139,7 +148,10 @@ static size_t host_decode(uint64_t dst, uint64_t cap, uint64_t src, uint64_t siz } else if (dict && dict_size) { if (!(dd = dict_for(0, dict, dict_size))) return 0; } - if (!dctx && !(dctx = ZSTD_createDCtx())) return 0; + if (!dctx) { + if (!plat_xchg32(&dctx_hooked, 1)) plat_thread_end_hook(dctx_thread_end); + if (!(dctx = ZSTD_createDCtx())) return 0; + } *ok = 1; return dd ? ZSTD_decompress_usingDDict(dctx, gp(dst), (size_t)cap, gp(src), (size_t)size, dd) : ZSTD_decompressDCtx(dctx, gp(dst), (size_t)cap, gp(src), (size_t)size); diff --git a/runtime/platform/common.c b/runtime/platform/common.c index 856200a0..d2bda9b6 100644 --- a/runtime/platform/common.c +++ b/runtime/platform/common.c @@ -159,3 +159,20 @@ int plat_addr_str(const PlatAddr *a, char *out, size_t cap) { memcpy(out, s, (size_t)n + 1); return n; } + +// ---- what a thread keeps for itself (platform.h plat_thread_end_hook) +static void (*end_hooks[PLAT_THREAD_END_HOOKS])(void); +static volatile int32_t n_end_hooks; + +void plat_thread_end_hook(void (*fn)(void)) { + int32_t i = __atomic_fetch_add(&n_end_hooks, 1, __ATOMIC_ACQ_REL); + if (i < PLAT_THREAD_END_HOOKS) __atomic_store_n(&end_hooks[i], fn, __ATOMIC_RELEASE); +} + +void plat_thread_end_run(void) { + int32_t n = __atomic_load_n(&n_end_hooks, __ATOMIC_ACQUIRE); + for (int32_t i = 0; i < n && i < PLAT_THREAD_END_HOOKS; i++) { + void (*fn)(void) = __atomic_load_n(&end_hooks[i], __ATOMIC_ACQUIRE); + if (fn) fn(); + } +} diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index c5fa433f..a47fbf16 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -114,6 +114,7 @@ static void *thread_main(void *p) { __atomic_store_n(&t->tid, (int32_t)syscall(SYS_gettid), __ATOMIC_RELEASE); if (t->name[0]) pthread_setname_np(pthread_self(), t->name); t->fn(t->arg); + plat_thread_end_run(); // what the other files keep per thread (platform.h plat_thread_end_hook) return NULL; } diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 6ea98cea..57b91b2b 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -28,6 +28,11 @@ void plat_thread_join(PlatThread *t); // waits for fn to return, then frees // Waits at most ns for fn to return: 1 when it did (t is freed), 0 when it still runs (t stays valid). int plat_thread_join_ns(PlatThread *t, uint64_t ns); void plat_thread_detach(PlatThread *t); // frees t; the thread runs on +// fn runs on every thread these calls make, when its function has returned and before the thread ends: for what a +// thread keeps for itself alone (a decoder context). At most PLAT_THREAD_END_HOOKS, each added once. +#define PLAT_THREAD_END_HOOKS 4 +void plat_thread_end_hook(void (*fn)(void)); +void plat_thread_end_run(void); // the platform files' thread start functions call it // Restricts t (NULL: the calling thread) to the logical processors in mask (bit n = processor n of the first group). // Returns the previous mask, 0 on failure. uint64_t plat_thread_affinity(PlatThread *t, uint64_t mask); diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index 96c58e77..92643200 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -31,11 +31,23 @@ struct PlatThread { HANDLE h; PlatThreadFn fn; void *arg; + volatile int32_t state; // 0 running and owned by its maker, 1 detached, 2 its function has returned }; +// What a thread made here keeps for itself is given back when its function returns: the handles below (the sleep +// timer of plat_wait_ns, the events of plat_file_pread and plat_file_pwrite), what the other files keep per thread +// (plat_thread_end_hook) and, for a detached thread, its PlatThread. The game makes and ends threads all through +// play, each a host thread: nothing closed them, so every ended thread that had slept, read or written left up +// to three kernel handles behind for the rest of the run. A guest thread ends by returning here too (kernel.c: +// ExitThread jumps back to the thread's start). +static void thread_handles_close(void); + static DWORD WINAPI thread_main(LPVOID p) { PlatThread *t = p; t->fn(t->arg); + plat_thread_end_run(); + thread_handles_close(); + if (plat_xchg32(&t->state, 2) == 1) free(t); // detached while it ran: nobody else frees it return 0; } @@ -73,10 +85,12 @@ int plat_thread_join_ns(PlatThread *t, uint64_t ns) { return 1; } -// The thread keeps using t->fn and t->arg, so the struct stays allocated (a few bytes per detached thread). +// The thread still reads t->fn and t->arg when it starts, so the struct goes when the thread's function has +// returned (thread_main), or here when it already has. void plat_thread_detach(PlatThread *t) { CloseHandle(t->h); t->h = NULL; + if (plat_xchg32(&t->state, 1) == 2) free(t); } int plat_cpu_cores(uint64_t *all, uint64_t *first) { @@ -474,13 +488,22 @@ uint64_t plat_now_ns(void) { return plat_scaled(qpc_ns, (uint64_t)t.QuadPart); } -// A high-resolution waitable timer per sleeping thread (Windows 10 1803+; one handle per thread, kept until exit) +// A high-resolution waitable timer per sleeping thread (Windows 10 1803+; one handle per thread, closed when the +// thread's function returns, thread_handles_close) // wakes within about 0.1-0.5 ms; the last PLAT_SPIN_NS are spent yielding so the deadline is met closely. // Without that timer kind, Sleep(1) granularity applies. #define PLAT_SPIN_NS 300000u static __thread HANDLE sleep_timer; static __thread int sleep_timer_tried; +static __thread HANDLE pread_ev, pwrite_ev; // plat_file_pread, plat_file_pwrite: the event of their overlapped call + +static void thread_handles_close(void) { + if (sleep_timer) CloseHandle(sleep_timer); + if (pread_ev) CloseHandle(pread_ev); + if (pwrite_ev) CloseHandle(pwrite_ev); + sleep_timer = pread_ev = pwrite_ev = NULL, sleep_timer_tried = 0; +} void plat_wait_ns(uint64_t ns) { if (!sleep_timer_tried) { @@ -1302,8 +1325,8 @@ void plat_file_close(PlatFile *f) { } int64_t plat_file_pread(PlatFile *f, void *buf, uint32_t n, uint64_t off) { - static __thread HANDLE ev; - if (!ev && !(ev = CreateEventW(NULL, TRUE, FALSE, NULL))) return -1; + HANDLE ev = pread_ev; + if (!ev && !(ev = pread_ev = CreateEventW(NULL, TRUE, FALSE, NULL))) return -1; OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32), .hEvent = ev}; DWORD got = 0; if (ReadFile((HANDLE)f, buf, n, &got, &ov)) return got; @@ -1561,8 +1584,8 @@ PlatFile *plat_fs_open(const char *path, int write, int *err) { PlatFile *plat_fs_create(const char *path, int *err) { return fs_open(path, GENERIC_WRITE, 0, CREATE_NEW, err); } int64_t plat_file_pwrite(PlatFile *f, const void *buf, uint32_t n, uint64_t off) { - static __thread HANDLE ev; - if (!ev && !(ev = CreateEventW(NULL, TRUE, FALSE, NULL))) return -1; + HANDLE ev = pwrite_ev; + if (!ev && !(ev = pwrite_ev = CreateEventW(NULL, TRUE, FALSE, NULL))) return -1; OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32), .hEvent = ev}; DWORD put = 0; if (WriteFile((HANDLE)f, buf, n, &put, &ov)) return put; diff --git a/tests/sync_wake_check.c b/tests/sync_wake_check.c index 6c8b9030..cbf3f03d 100644 --- a/tests/sync_wake_check.c +++ b/tests/sync_wake_check.c @@ -12,7 +12,8 @@ static int fails, wake_all; // ---- the runtime kernel.c reaches unsigned rt_trace; uint64_t rt_guest_start_ns; -volatile uint32_t prof_replay_tid; +volatile uint32_t prof_replay_tid, prof_main_tid; +int mem_readable(uint64_t a, uint64_t n) { (void)a, (void)n; return 0; } // NX2NAVI_SVC_WAITS walks guest frames: off here const char *cfg_get(const char *name) { return !strcmp(name, "NX2NAVI_SYNC_WAKE_ALL") && wake_all ? "1" : NULL; } const char *cfg_value(const char *name) { (void)name; return "0"; } // no asynchronous preemption, no slow-service log int cfg_bool(const char *name) { (void)name; return 0; } diff --git a/tests/thread_end_check.c b/tests/thread_end_check.c new file mode 100644 index 00000000..555e1a3b --- /dev/null +++ b/tests/thread_end_check.c @@ -0,0 +1,91 @@ +// runtime/platform: what a thread keeps for itself is given back when its function returns (win32.c thread_main, +// platform.h plat_thread_end_hook). Threads that sleep through plat_wait_ns and read a file are made and ended by the +// hundred, joined and detached; the end hook must run once on each, and on Windows the process must not hold more +// handles afterwards than a few threads' worth. Built and run by tests/thread_end_check.py. +#include +#include +#include + +#include "platform/platform.h" + +#ifdef _WIN32 +#include +static long handles(void) { + DWORD n = 0; + return GetProcessHandleCount(GetCurrentProcess(), &n) ? (long)n : -1; +} +#else +static long handles(void) { return 0; } // no handle per thread there; the hook is what is checked +#endif + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +void rt_log(const char *fmt, ...) { // the platform layer's lines + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); +} + +static const char *path; +static volatile int32_t ended, hook_runs, hook_other; +static __thread int worked; + +static void hook(void) { + if (worked) plat_inc32(&hook_runs); + else plat_inc32(&hook_other); + worked = 0; +} + +static void worker(void *arg) { + (void)arg; + worked = 1; + plat_wait_ns(1200000); // over a millisecond: the thread's own timer + PlatFile *f = plat_file_open(path); + char b[16]; + if (!f || plat_file_pread(f, b, sizeof b, 0) != (int64_t)sizeof b) plat_inc32(&hook_other); // counted as a failure + if (f) plat_file_close(f); + plat_inc32(&ended); +} + +static void wait_ended(int32_t want) { + for (int i = 0; i < 20000 && __atomic_load_n(&ended, __ATOMIC_ACQUIRE) < want; i++) plat_sleep_ms(1); + plat_sleep_ms(100); // the last ones leave their start function and the system lets go of them +} + +int main(int argc, char **argv) { + if (argc != 2) { + printf("usage: thread_end_check FILE_OF_16_BYTES_OR_MORE\n"); + return 2; + } + path = argv[1]; + plat_thread_end_hook(hook); + enum { WARM = 16, ROUNDS = 25, PER = 16 }; + for (int i = 0; i < WARM; i++) plat_thread_spawn(worker, NULL, "warm", 0); + wait_ended(WARM); + long h0 = handles(); + int32_t made = WARM; + for (int r = 0; r < ROUNDS; r++) { + PlatThread *t[PER / 2]; + for (int i = 0; i < PER / 2; i++) { + t[i] = plat_thread_new(worker, NULL, "joined"); + made += t[i] != NULL; + made += plat_thread_spawn(worker, NULL, "detached", 0); + } + for (int i = 0; i < PER / 2; i++) + if (t[i]) plat_thread_join(t[i]); + wait_ended(made); + } + long h1 = handles(); + CHECK(made == WARM + ROUNDS * PER, "%d threads made of %d", (int)made, WARM + ROUNDS * PER); + CHECK(__atomic_load_n(&ended, __ATOMIC_ACQUIRE) == made, "%d of %d threads ended", (int)ended, (int)made); + CHECK(hook_runs == made, "the end hook ran on %d of %d threads", (int)hook_runs, (int)made); + CHECK(!hook_other, "%d threads could not read the file, or ran the hook without having worked", (int)hook_other); + // each thread made a timer and a read event: left open, 400 threads would hold 800 more handles + CHECK(h1 - h0 < 64, "%ld handles more after %d ended threads (%ld before, %ld after)", h1 - h0, ROUNDS * PER, h0, h1); + printf("thread_end_check: %s (%d threads, the hook ran on %d, handles %ld before and %ld after)\n", + fails ? "FAILED" : "ok", (int)made, (int)hook_runs, h0, h1); + return fails ? 1 : 0; +} diff --git a/tests/thread_end_check.py b/tests/thread_end_check.py new file mode 100644 index 00000000..1eaeea72 --- /dev/null +++ b/tests/thread_end_check.py @@ -0,0 +1,48 @@ +"""Build and run tests/thread_end_check.c: what a thread keeps for itself is given back when it ends +(runtime/platform/win32.c thread_main, platform.h plat_thread_end_hook). 416 threads sleep, read a file and end, half +joined and half detached; the end hook must run once on each, and on Windows the process's handle count must be what +it was. No game files, no GPU. + +Usage: thread_end_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'thread_end' + work.mkdir(parents=True, exist_ok=True) + win = os.name == 'nt' + exe = work / ('thread_end_check.exe' if win else 'thread_end_check') + env = dict(os.environ) + if win and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + rt = ROOT / 'runtime' + plat = [rt / 'platform' / ('win32.c' if win else 'linux.c'), rt / 'platform' / 'common.c'] + libs = ['-lbcrypt', '-lws2_32', '-lpsapi', '-lwinmm', '-ldbghelp', '-lshell32', '-lole32', '-luser32', '-ladvapi32'] if win \ + else ['-lpthread'] + subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-fno-strict-aliasing', + '-march=x86-64-v3', '-I', str(rt), + str(ROOT / 'tests' / 'thread_end_check.c'), *map(str, plat), '-o', str(exe), '-lm', *libs], + env=env) + data = work / 'read.bin' + data.write_bytes(bytes(range(64))) + return subprocess.run([str(exe), str(data)], env=env, timeout=300).returncode + + +if __name__ == '__main__': + sys.exit(main()) From 18aa16f2df5691ddd7efbaa31170cc432ee520be Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:19:59 -0500 Subject: [PATCH 08/15] Skip the menu's draw for a frame when its vertex or index buffer cannot be allocated, instead of copying through a null pointer. The menu's Vulkan back end went on after a failed buffer allocation: it bound and mapped memory it did not have and copied the frame's vertices to the address the failed mapping left, null. Two reports of 0.8.0 end that way right after the settings menu opens with video memory used up, one on an integrated AMD GPU and one on an RTX 2050: two "got VkResult -2" lines, then an access violation in the C runtime's copy, writing address 0x20, not on a guest thread. The allocation now leaves no buffer and size 0 when it fails, the draw returns before anything is recorded, and the next frame asks for the buffer again. The change is marked in the vendored file and named in THIRD_PARTY_NOTICES.md. Compiled; the failing allocation was not reproduced here. Co-Authored-By: Claude Fable 5.2 --- THIRD_PARTY_NOTICES.md | 2 +- .../imgui/backends/imgui_impl_vulkan.cpp | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index dc929eac..e2a9f316 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -29,7 +29,7 @@ Code copied or adapted from another project is listed here with its source, the | SDL 3.4.18, dynamic link, `SDL3.dll` copied next to `nx2navi.exe`. Developer builds use the MSYS2 build (`mingw-w64-ucrt-x86_64-sdl3`, which imports `libiconv-2.dll`, LGPL-2.1, unmodified, copied beside it); the installer release ships the upstream release `SDL3-3.4.18-win32-x64.zip`, which imports no libiconv | https://libsdl.org, Copyright (c) 1997-2026 Sam Lantinga | zlib | `runtime/input_win.c` (gamepad input, rumble, gyro and accelerometer) | | libopus (MSYS2 `mingw-w64-ucrt-x86_64-opus`), dynamic link, `libopus-0.dll` copied next to `nx2navi.exe` | https://opus-codec.org, Copyright (c) 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon | BSD-3-Clause | `runtime/ov_media.c` (nn::codec hardware Opus decoder) | | Monocypher 4.0.2 (`monocypher.c`, `monocypher.h`, unmodified; licence text in `runtime/third_party/monocypher/LICENCE.md`) | https://monocypher.org/download/monocypher-4.0.2.tar.gz (SHA-512 `bf275d4c53ff94af6cdc723a4e002e9f080f4d1436c86c76bb37870b34807f1d7b32331d8ff8a1aeb369e946f3769021e03e63efac25b82efc5abf54dc084714`), Copyright (c) 2017-2023 Loup Vaillant, Michael Savage, Fabio Scotoni | BSD-2-Clause (dual-licensed with CC0-1.0; the BSD licence is the one used here) | `runtime/third_party/monocypher/` (compiled into `nx2navi.exe` for co-op: key derivation with BLAKE2b and the room's messages sealed with XChaCha20-Poly1305, `runtime/coop_proto.c`) | -| Dear ImGui 1.91.9 (core and Vulkan back end, unmodified), as vendored by LAS Recomp | https://github.com/ocornut/imgui, via https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f` (`third_party/imgui/`), Copyright (c) 2014-2025 Omar Cornut | MIT | `runtime/third_party/imgui/` (compiled into `nx2navi.exe` for the in-game menu, `runtime/ui/`) | +| Dear ImGui 1.91.9 (core unmodified; the Vulkan back end with one change, marked NX2NAVI: a vertex or index buffer that cannot be allocated or mapped skips the frame's draw), as vendored by LAS Recomp | https://github.com/ocornut/imgui, via https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f` (`third_party/imgui/`), Copyright (c) 2014-2025 Omar Cornut | MIT | `runtime/third_party/imgui/` (compiled into `nx2navi.exe` for the in-game menu, `runtime/ui/`) | | SavageWindRecomp settings menu: two-pane option rows (row look, left / right stepping, hold speed-up, description pane), the menu style and tab bar | https://github.com/Revivedx/SavageWindRecomp @ `575ad88e02f9a89a9a885d90039d48af33c982f4` (`runtime/ui/option_rows.mm`, `runtime/ui/menu.mm`), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/ui/option_rows.cpp`, `runtime/ui/option_rows.h`, `runtime/ui/menu.cpp` | | SavageWindRecomp performance overlay: the frame ring and its one-second summary, the NVML calls, the sampling loop, the full panel's layout | https://github.com/Revivedx/SavageWindRecomp @ `575ad88e02f9a89a9a885d90039d48af33c982f4` (`runtime/perfmon.c`, `runtime/perfmon.h`, `perf_overlay_draw` in `runtime/ui/menu.mm`), Copyright (c) 2026 Revivedx and the Savage Wind Recomp contributors, Copyright (c) 2026 Peiikai | MIT | `runtime/perfmon.c`, `runtime/perfmon.h`, `runtime/ui/menu.cpp` | | Roboto Medium font (Dear ImGui's `misc/fonts/Roboto-Medium.ttf`, unmodified) | https://fonts.google.com/specimen/Roboto, via https://github.com/peiikai/LASRecomp @ `da35b83a9611d046a9a681aae668d7085954d37f` (`third_party/imgui/misc/fonts/`), Copyright 2011 Google Inc. | Apache-2.0 | `runtime/ui/fonts/Roboto-Medium.ttf` (licence text in `runtime/ui/fonts/LICENSE-Roboto.txt`), embedded by `runtime/ui/menu.cpp` | diff --git a/runtime/third_party/imgui/backends/imgui_impl_vulkan.cpp b/runtime/third_party/imgui/backends/imgui_impl_vulkan.cpp index e67b74c0..d2cacb7e 100644 --- a/runtime/third_party/imgui/backends/imgui_impl_vulkan.cpp +++ b/runtime/third_party/imgui/backends/imgui_impl_vulkan.cpp @@ -427,6 +427,13 @@ static void CreateOrResizeBuffer(VkBuffer& buffer, VkDeviceMemory& buffer_memory buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; err = vkCreateBuffer(v->Device, &buffer_info, v->Allocator, &buffer); check_vk_result(err); + if (err != VK_SUCCESS) // NX2NAVI: as for the memory below + { + buffer = VK_NULL_HANDLE; + buffer_memory = VK_NULL_HANDLE; + buffer_size = 0; + return; + } VkMemoryRequirements req; vkGetBufferMemoryRequirements(v->Device, buffer, &req); @@ -435,8 +442,20 @@ static void CreateOrResizeBuffer(VkBuffer& buffer, VkDeviceMemory& buffer_memory alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; alloc_info.allocationSize = req.size; alloc_info.memoryTypeIndex = ImGui_ImplVulkan_MemoryType(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, req.memoryTypeBits); + buffer_memory = VK_NULL_HANDLE; // NX2NAVI: the handle freed above is not kept when the allocation fails err = vkAllocateMemory(v->Device, &alloc_info, v->Allocator, &buffer_memory); check_vk_result(err); + // NX2NAVI: out of memory. No buffer, no memory and size 0, so the caller draws nothing this frame and the next + // frame asks again (ImGui_ImplVulkan_RenderDrawData). Unchecked, the mapping below it failed too and the copy of + // the vertices went through a null pointer. + if (err != VK_SUCCESS) + { + vkDestroyBuffer(v->Device, buffer, v->Allocator); + buffer = VK_NULL_HANDLE; + buffer_memory = VK_NULL_HANDLE; + buffer_size = 0; + return; + } err = vkBindBufferMemory(v->Device, buffer, buffer_memory, 0); check_vk_result(err); @@ -523,14 +542,25 @@ void ImGui_ImplVulkan_RenderDrawData(ImDrawData* draw_data, VkCommandBuffer comm CreateOrResizeBuffer(rb->VertexBuffer, rb->VertexBufferMemory, rb->VertexBufferSize, vertex_size, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); if (rb->IndexBuffer == VK_NULL_HANDLE || rb->IndexBufferSize < index_size) CreateOrResizeBuffer(rb->IndexBuffer, rb->IndexBufferMemory, rb->IndexBufferSize, index_size, VK_BUFFER_USAGE_INDEX_BUFFER_BIT); + // NX2NAVI: a buffer that could not be made (CreateOrResizeBuffer): nothing is drawn this frame + if (rb->VertexBufferMemory == VK_NULL_HANDLE || rb->IndexBufferMemory == VK_NULL_HANDLE) + return; // Upload vertex/index data into a single contiguous GPU buffer ImDrawVert* vtx_dst = nullptr; ImDrawIdx* idx_dst = nullptr; VkResult err = vkMapMemory(v->Device, rb->VertexBufferMemory, 0, vertex_size, 0, (void**)&vtx_dst); check_vk_result(err); + // NX2NAVI: a mapping that failed leaves nothing to copy into + if (err != VK_SUCCESS || !vtx_dst) + return; err = vkMapMemory(v->Device, rb->IndexBufferMemory, 0, index_size, 0, (void**)&idx_dst); check_vk_result(err); + if (err != VK_SUCCESS || !idx_dst) + { + vkUnmapMemory(v->Device, rb->VertexBufferMemory); + return; + } for (int n = 0; n < draw_data->CmdListsCount; n++) { const ImDrawList* draw_list = draw_data->CmdLists[n]; From e1b1691e2801b10b35e9fc8f4a1b1b7631b3a7e8 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:21:06 -0500 Subject: [PATCH 09/15] Make a refused upload submission again once the batches in flight have finished and the kept buffers are given back. vkQueueSubmit of an upload batch that the driver refused for memory ended the game at once ("upload submit failed (-2)": five logs of three reports, all on integrated GPUs, whose device heap is the machine's memory and fills before any commit check sees it). A refused submit leaves the command buffer and its fence as they were, so the batch is now submitted a second time after the batches in flight are waited for, the objects that waited for them are destroyed and the staging cache, the arena's free chunks, the snapshot pool and the spare blocks go back to the driver, as a refused staging buffer already does. A second refusal ends the run as before. Compiled; the refusal was not reproduced here. Co-Authored-By: Claude Fable 5.2 --- runtime/vk_memory.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index c1f1e447..998ef77c 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -5396,6 +5396,28 @@ uint64_t vk_upload_flush(int wait) { open_b.sub_ns = plat_now_ns(); VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); + if (r == VK_ERROR_OUT_OF_DEVICE_MEMORY || r == VK_ERROR_OUT_OF_HOST_MEMORY) { + // The driver had no memory to take the batch, which ended the game here (five logs of three reports, all on + // integrated GPUs, one with 9 GB of commit to spare: the device's own heap was full). A refused + // submit leaves the command buffer and the fence as they were, so it can be made again: first the batches + // in flight are waited for, which gives their staging back and lets the objects that waited for them be + // destroyed, then the kept buffers and spare blocks go back to the driver (vk_mem_give_back), as for a + // refused staging buffer. Only a second refusal ends the run. + VkResult first = r; + int flying = ninflight; + retire(c, 1); + collect_locked(c); + uint64_t freed = vk_mem_give_back(); + vk_queue_lock(); + open_b.sub_ns = plat_now_ns(); + r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); + vk_queue_unlock(); + static volatile int32_t told; + if (plat_inc32(&told) <= 16) + rt_log("vk: upload submit refused (%d): waited for %d batches in flight, %.1f MiB of kept buffers and " + "spare blocks given back, %s", (int)first, flying, freed / 1048576.0, + r == VK_SUCCESS ? "taken at the second try" : "refused again"); + } if (r != VK_SUCCESS) { vk_mem_oom_report("upload submit", r); vk_device_lost_check(r, "the submit of an upload batch"); From e3e4b2d6507c3a5371c64dc7f768ff382a9984bb Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:21:43 -0500 Subject: [PATCH 10/15] Log what keeps the released textures above their size bound. The bound on the released textures' memory only takes entries that are old enough and that no lookup reaches. The texture pool stamps every GPU-owned entry that a slot's last fill resolved to in each submission, a finalized slot's released entry included, so such an entry never counts as idle: in a session at 5120x2160 the released entries held 9.2 and then 10.1 GiB against a bound of 4,008 MiB, and its overwritten render targets were logged as "last used 1 presents ago". Nothing said which rule held how much. When the released entries stay a quarter over the bound after the drop, one line now splits them: released too lately, sampled lately, shown by a pool slot and not sampled (and of those, the ones another target was drawn over), reached by another lookup, next to go. No rule changes. Co-Authored-By: Claude Fable 5.2 --- runtime/vk_texture.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 2c1302ef..714c53d7 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -3068,6 +3068,41 @@ static void commit_evict(void) { // tex_lock held, the commit is low and the r if (freed < COMMIT_EVICT_STEP) evict_disused(COMMIT_EVICT_STEP - freed); } +// The released entries hold more than their bound (released_cap_bytes) after drop_stale took what it may: one line that +// says what keeps the rest, at most every REL_CENSUS_SUBS submissions and REL_CENSUS_LINES times a run. The bound is +// no bound for an entry a lookup still reaches, and the pool stamps every GPU-owned entry that a slot's last fill +// resolved to in each submission (pool_touch), a finalized slot's released one too: at 5120x2160 a session's log had +// 989 and then 2,172 released entries holding 9.2 and 10.1 GiB against a bound of 4,008 MiB, and its overwritten +// render targets "last used 1 presents ago". Each entry counts once, under the first reason that holds. +#define REL_CENSUS_SUBS 3600 +#define REL_CENSUS_LINES 24 +static void rel_census(uint64_t cap) { // tex_lock held + static uint32_t last, lines; + if (lines >= REL_CENSUS_LINES || (last && submit_epoch - last < REL_CENSUS_SUBS)) return; + last = submit_epoch ? submit_epoch : 1, lines++; + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + enum { YOUNG, SLOT, SLOT_OVER, SAMPLED, LOOKED, NEXT, KINDS }; + uint64_t by[KINDS] = {0}, gpu = 0; + int n[KINDS] = {0}, ngpu = 0; + for (const Tex *e = rel_head; e; e = e->rel_next) { + if (!e->bytes) continue; + int shown = e->ph && pref[e->ph - 1].refs; + int k = now - e->rel_frame < stale_after(e) ? YOUNG + : now - e->samp_frame < STALE_FRAMES ? SAMPLED + : shown ? (e->overwritten ? SLOT_OVER : SLOT) + : !e->overwritten && !idle(e, now) ? LOOKED : NEXT; + by[k] += e->bytes, n[k]++; + if (!comes_back_intact(e)) gpu += e->bytes, ngpu++; + } + const double M = 1048576.0; + rt_log("vk: released textures over their bound: %d entries, %.0f MiB of %.0f MiB (%d of them, %.0f MiB, hold content " + "only the GPU made): %d, %.0f MiB, released too lately to go; %d, %.0f MiB, sampled or drawn into lately; %d, " + "%.0f MiB, shown by a texture pool slot and not sampled, and %d more, %.0f MiB, that another target was drawn " + "over; %d, %.0f MiB, reached by another lookup; %d, %.0f MiB, next to go", nreleased, rel_bytes / M, cap / M, + ngpu, gpu / M, n[YOUNG], by[YOUNG] / M, n[SAMPLED], by[SAMPLED] / M, n[SLOT], by[SLOT] / M, n[SLOT_OVER], + by[SLOT_OVER] / M, n[LOOKED], by[LOOKED] / M, n[NEXT], by[NEXT] / M); +} + static void sweep_pressure(void) { // tex_lock held if (vram_pressure < 0) { const char *s = cfg_get("NX2NAVI_TEX_VRAM_PRESSURE"); @@ -3128,6 +3163,7 @@ static void sweep_pressure(void) { // tex_lock held drop_overwritten(); uint64_t cap = released_cap_bytes(); if (rel_bytes > cap) drop_stale(cap, &n_cap_drops, DROP_CAP); + if (cap != UINT64_MAX && rel_bytes > cap + cap / 4) rel_census(cap); // well over it still: what holds the rest if (rel_bytes > COMMIT_LOW_KEEP && low && !held) more = drop_stale(COMMIT_LOW_KEEP, &n_commit_drops, DROP_COMMIT) == PRESSURE_STEP; } From 724a9d98cea9c53f0002c589aa121879bb9fea84 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:31:25 -0500 Subject: [PATCH 11/15] Size the game's render-target heap for the scene's share of the picture, so a scene at the picture's full size fits it. The heap the engine's render targets live in is made for a 2560x1440 picture whose scene is three quarters of it at most. It grew only with the picture. With DLAA, or FSR at native size, the scene is the whole picture: at the game's own 1440 lines that is 2560x1440 targets where the game never passes 1920x1080, in a heap that had not grown at all. The render-target manager's search for a free block then takes its out-of-memory path and reads through a null block: a player's game ended there (main+0x3ed00, reading address 0x48) each time a dialogue closed in Castle Town on 0.8.0 with dlss = dlaa (#605), the same function that failed at 5120x2160 before the heap followed the picture. The heap and the application pool now grow by the scene's pixels over the game's most as well: 238 MiB at 1440 lines with the upscaler setting on, nothing without it, and nothing at 1080 lines, where a full scene is the game's own size. The share is decided at the start, from the settings; an upscaler first switched on in the menu of a run that began without one keeps the scene at three quarters until a restart, and the log says so. Compiled; not played. Co-Authored-By: Claude Fable 5.2 --- runtime/widescreen.c | 54 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/runtime/widescreen.c b/runtime/widescreen.c index 48f3d7e0..fcea6aeb 100644 --- a/runtime/widescreen.c +++ b/runtime/widescreen.c @@ -88,11 +88,23 @@ static unsigned picture_height(void) { static double dlss_share(void); double wide_dlss_share(void) { return dlss_share(); } // vk_dlaa.c: the mode's share, 0 off +static double heap_share(void); + static double dlss_share(void) { if (!vk_dlaa_wanted() || !cfg_bool("NX2NAVI_DLAA") || (!vk_dlaa_fsr3() && !vk_dlaa_available())) return 0; const char *m = cfg_value("NX2NAVI_DLSS"); - return !strcmp(m, "quality") ? 2.0 / 3 : !strcmp(m, "balanced") ? 0.58 : !strcmp(m, "performance") ? 0.5 - : !strcmp(m, "ultra_performance") ? 1.0 / 3 : 1.0; + double s = !strcmp(m, "quality") ? 2.0 / 3 : !strcmp(m, "balanced") ? 0.58 : !strcmp(m, "performance") ? 0.5 + : !strcmp(m, "ultra_performance") ? 1.0 / 3 : 1.0; + // Never more than the texture heap was made for at the start (heap_share): only an upscaler switched on in the + // menu of a run that began without one can ask for more, and it gets the whole picture after a restart. + if (s > heap_share()) { + static int said; + if (!said++) + rt_log("widescreen: the scene stays at %.0f%% of the picture until a restart: the game's render-target heap " + "was sized at the start, without an upscaler, and %.0f%% does not fit it", heap_share() * 100, s * 100); + s = heap_share(); + } + return s; } // Without DLSS or FSR 3.1 (FSR 1, bilinear) the game's dynamic resolution took the scene down to a third of the picture @@ -109,6 +121,25 @@ static double scene_share(void) { return s > 0 ? s : plain; } +// The largest share of the picture the scene can have in this run, which the texture heap is made for +// (wide_heap_extra_kib). Decided once, at the first call, which is the start (memory.c sizes the application pool +// from it before any device exists): the whole picture when the upscaler setting is on, whatever its mode, since +// the menu changes the mode in play and DLAA and FSR at native size render all of it; else the game's own most, +// PLAIN_SHARE, or NX2NAVI_SCENE_SHARE when that asks for more. From the settings, not from what runs: DLSS is known +// to run only once the device is up. +static double heap_share(void) { + static double share = -1; + if (share < 0) { + const char *e = cfg_get("NX2NAVI_SCENE_SHARE"); + double s = e && *e ? atof(e) : PLAIN_SHARE; + if (s < PLAIN_SHARE) s = PLAIN_SHARE; + if (s > 1) s = 1; + if (rt_patch_set("res_fixed") >= 0 && cfg_bool("NX2NAVI_DLAA")) s = 1; + share = s; + } + return share; +} + // The game turns the scale at +240 into a scene size against its own 1440-line picture while the picture is taller // (a 2160-line picture at 1.0 gave a 1440-line scene), so the share goes there times the picture's height over 1440. // Under a 1080-line picture the game measures against that picture instead (a share of 1.0 times 1080/1440 gave an @@ -152,15 +183,20 @@ static int picture_size(unsigned *ow, unsigned *oh, int quiet) { // The texture heap: the engine's render targets live in "DynamicTextureAllocatorMem" of the HeapDef table // (System/HeapSize/Product.Nin_Ounce_NVN2.HeapDef.byml.zs; 312320 KiB, carved from "Graphics"), made for the 2560x1440 -// picture. Larger pictures' targets did not fit: the render-target manager's search for a free block (main+0x3ed00) -// fell to its out-of-memory path and read through a null block when the pause menu's blur needed one at 5120x2160. -// The heap and its parent grow by the shipped size times the picture's pixels over 2560x1440 (its targets are the -// picture's size or a share of it), in whole MiB; resolution.c rewrites the table (res_heapdef_build) and memory.c adds -// the amount to the application pool, so no other heap shrinks. 0 without the widescreen sets or for the game's size. +// picture with the scene at three quarters of it at most. Larger pictures' targets did not fit: the render-target +// manager's search for a free block (main+0x3ed00) fell to its out-of-memory path and read through a null block when +// the pause menu's blur needed one at 5120x2160. So did a larger scene in the game's own picture: with DLAA the scene +// is all of the picture, 2560x1440 where the game never goes past 1920x1080, and a player's game ended in that same +// function, reading address 0x48, each time a dialogue closed in Castle Town (0.8.0, 1440 lines, dlss = dlaa). +// The heap and its parent grow by the shipped size times the picture's pixels over 2560x1440 and times the scene's +// pixels over the game's most (heap_share: its targets are the picture's size or the scene's), in whole MiB; +// resolution.c rewrites the table (res_heapdef_build) and memory.c adds the amount to the application pool, so no +// other heap shrinks. 0 for the game's picture and scene, and for a larger picture without the widescreen sets. uint64_t wide_heap_extra_kib(void) { unsigned ow, oh; - if (!wide_available() || !picture_size(&ow, &oh, 1)) return 0; - double grow = (double)ow * oh / ((double)GAME_W * GAME_H) - 1; + if (!wide_available() || !picture_size(&ow, &oh, 1)) ow = GAME_W, oh = GAME_H; + double k = heap_share() / PLAIN_SHARE; + double grow = (double)ow * oh / ((double)GAME_W * GAME_H) * k * k - 1; return grow > 0 ? (uint64_t)ceil(GAME_DTA_KIB * grow / 1024) * 1024 : 0; } From 07557fe06caf889ce6d86f3388454943eab799b0 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:31:25 -0500 Subject: [PATCH 12/15] Give an empty slot's handle for a texture id out of range instead of ending the game. nvnDeviceGetTextureHandle ended the game when the texture id was negative or past the table. The game passes -1 when it has no texture to give, as when its render-target heap is full; the handle of the last slot, which no game registers, draws the stand-in like any slot without a texture. Logged up to eight times. Co-Authored-By: Claude Fable 5.2 --- runtime/nvn_api_core.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 3945b419..caad7a04 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -236,9 +236,19 @@ NVN_API(nvnDeviceRegisterFastClearDepth) { ret_x(c, 0); } NVN_API(nvnDeviceGetImageHandle) { ret_x(c, (uint64_t)(int64_t)(int)arg_x(c, 1)); } // NVNtextureHandle nvnDeviceGetTextureHandle(const NVNdevice *d, int texture_id, int sampler_id): // texture id (20 bits) | sampler id << 20, the encoding the replay decodes (NVN_HANDLE_TEX / NVN_HANDLE_SAMP) +// An id out of range (-1: the game had no texture to give, as when its render-target heap is full) gets the handle of +// the last texture slot, which no game registers (its pool has 32,768), so a draw through it samples the stand-in as +// for any slot without a texture, and sampler 0. Ending the game here turned one missing texture into a crash. NVN_API(nvnDeviceGetTextureHandle) { int tid = (int)arg_x(c, 1), sid = (int)arg_x(c, 2); - if (tid < 0 || tid >= (1 << 20) || sid < 0 || sid >= (1 << 12)) rt_fatal("nvn: texture handle ids out of range %d/%d", tid, sid); + if (tid < 0 || tid >= (1 << 20) || sid < 0 || sid >= (1 << 12)) { + static int said; + if (said < 8 && ++said) + rt_log("nvn: texture handle asked for with ids out of range (texture %d, sampler %d): the handle of an " + "empty slot is given, which draws the stand-in", tid, sid); + if (tid < 0 || tid >= (1 << 20)) tid = (1 << 20) - 1; + if (sid < 0 || sid >= (1 << 12)) sid = 0; + } ret_x(c, (uint64_t)tid | (uint64_t)sid << 20); } // NVNtextureHandle nvnDeviceGetTexelFetchHandle(const NVNdevice *d, int texture_id) From bc256496cb4291983dbcb3da0d3b5d6596c4492c Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:31:56 -0500 Subject: [PATCH 13/15] Build task and mesh shader pipelines without the driver's optimiser on AMD's driver for RDNA 1 and 2, unless vk_noopt says otherwise. That driver loses the device in play ("No fault detected", a timeout) on RX 6600 to RX 6950 XT cards in every version from 0.6.0 to 0.9.2, with the pipeline library on or off and the lazy write-back on or off (#591, #602, #659, #683, #684, #689, #635, #719, #279, and fourteen of twenty sessions in a player's launcher log of an RX 6800). Where the pass marks name a pass the GPU was in, twelve logs of five reporters, its first draw is a mesh pipeline: mesh shader ba4302f1 with fragment shader f64a96dd in nine, the one-draw pass of 1bbdb17e with 6cc3a476 in three. The five logs of 0.9.1 and 0.9.2 stop between the compute dispatch before such a pass and the pass. vk_noopt = mesh was there as a test switch for this suspicion and no reporter had run with it. It is now what an unset vk_noopt means on that driver branch; every other driver keeps 0, and vk_noopt = 0 gives the optimised pipelines back. The picture is the same; the mesh and task shaders run slower, by how much is not measured, and those pipelines are built once more because the flag is part of the driver's cache key. That the optimiser is the cause is not shown: no AMD card ran this. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 6 ++++++ runtime/cfg.c | 2 +- runtime/vk_pipeline.c | 23 ++++++++++++++++++----- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 2f107f78..ac545fef 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -854,6 +854,12 @@ # last one it finished. Unset: on AMD GPUs (VK_AMD_buffer_marker). 1: also on NVIDIA GPUs # (VK_NV_device_diagnostic_checkpoints), which NVIDIA says costs a little. 0: off. # pass_crumbs = 1 +# Pipelines built without the video driver's shader optimiser: the same picture, slower shaders in those pipelines. +# mesh: only the pipelines with task and mesh shaders (terrain, foliage). 1: all. 0: none. +# Unset (default): mesh on AMD's Windows driver for the RX 5000 and RX 6000 series and older, where the device is lost +# ("driver timeout") in render passes that start with such a pipeline; 0 on every other driver. On those cards 0 gives +# the optimised pipelines back. +# vk_noopt = mesh # GPU draw marks, for a GPU crash (device lost) that comes back in the same place on an AMD GPU: inside each render pass # a mark is written wherever the shaders change, and after the crash the log names the shaders of the draws the GPU had # not finished, not only the pass. It costs a little frame rate, so it is off unless a crash is being tracked down. diff --git a/runtime/cfg.c b/runtime/cfg.c index c50354f3..aeba5924 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -344,7 +344,7 @@ static const CfgMeta metas[] = { {"VK_RECONVERGE", "1", "0|1", RESTART}, // vk_pipeline.c spirv_reconverge {"VK_NAN_PRESERVE", "", "0|1", RESTART}, // vk_pipeline.c spirv_nan_preserve; unset: every driver but NVIDIA's {"VK_NAN_BITS", "", "0|1", RESTART}, // vk_pipeline.c spirv_nan_bits; unset: AMD's driver for RDNA 1 and 2 - {"VK_NOOPT", "0", "0|1|mesh", RESTART}, // vk_pipeline.c noopt_flag_for + {"VK_NOOPT", "", "0|1|mesh", RESTART}, // vk_pipeline.c noopt_flag_for; unset: mesh on AMD's driver for RDNA 1 and 2 // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2|3", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 36040e6e..185969f5 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -3171,14 +3171,27 @@ static int is_src1(unsigned f) { return f >= NVN_BLEND_FUNC_SRC1_COLOR && f <= N // which NX2navi never sets (0 of 876 pipelines in the field and fight saves). // NX2NAVI_VK_NOOPT=1: pipelines built without the driver's optimiser, for a driver whose shader compiler crashes on // one (the test switch for such a report); slower shaders, the same picture. mesh: only pipelines with task and mesh -// stages, the foliage's, whose optimiser is the suspect in device losses some minutes into a forest on AMD GPUs. +// stages, the terrain's and the foliage's. 0: none. +// Unset: mesh on AMD's Windows driver for RDNA 1 and 2 (vk_caps.amd_sc), 0 elsewhere. That driver loses the device +// ("No fault detected", a timeout) minutes into play on RX 6600 to RX 6950 XT cards in every version from 0.6.0 to +// 0.9.2, with the pipeline library on or off and the lazy write-back on or off. Where the reports' pass marks name a +// pass the GPU was in, twelve logs of five reporters, it is a pass whose first draw is a mesh pipeline (mesh shader +// ba4302f1 with fragment shader f64a96dd in nine, the one-draw pass of 1bbdb17e with 6cc3a476 in three); the five +// logs of 0.9.1 and 0.9.2 stop between the compute dispatch before such a pass and the pass. No reporter ran with +// the optimiser off, so that it is the cause is not shown: it is the one suspect the marks leave, it costs shader +// speed in those pipelines and not the picture, and NX2NAVI_VK_NOOPT=0 gives the optimised ones back. The flag is +// part of what the driver's cache keys a pipeline by, so those pipelines are built once more after the change. static VkPipelineCreateFlags noopt_flag_for(int mesh) { static int on = -1; if (on < 0) { - const char *v = cfg_value("NX2NAVI_VK_NOOPT"); - on = !strcmp(v, "mesh") ? 2 : strcmp(v, "0") != 0; - if (on) rt_log("vk_pipeline: %s built without the driver's optimiser (NX2NAVI_VK_NOOPT)", - on == 2 ? "pipelines with task and mesh stages" : "pipelines"); + const char *v = cfg_get("NX2NAVI_VK_NOOPT"); + int by_driver = !v || !*v; + on = by_driver ? (vk_caps.amd_sc ? 2 : 0) : !strcmp(v, "mesh") ? 2 : strcmp(v, "0") != 0; + if (on) rt_log("vk_pipeline: %s built without the driver's optimiser (%s)", + on == 2 ? "pipelines with task and mesh stages" : "pipelines", + by_driver ? "the default on AMD's driver for RDNA 1 and 2, where the device is lost in passes " + "that start with such a pipeline; NX2NAVI_VK_NOOPT=0 builds them optimised" + : "NX2NAVI_VK_NOOPT"); } return on == 1 || (on == 2 && mesh) ? VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT : 0; } From 016c90fafbb78a56dab6711b7c7eaa4327d4dad5 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 15:31:56 -0500 Subject: [PATCH 14/15] Wait at the start for every recorded pipeline when the pipeline cache was written by another device or driver. After a driver update pipeline-cache.bin is dropped and every recorded pipeline is built from nothing. With the pipeline library the boot waited 20 s and left the rest to be built beside the game. In a player's log of an 8-core integrated Radeon that was 2,141 of 8,122 pipelines, finished 296 s after the start, with the 42,231 prepared variants being built behind them: the game drew fewer than 600 frames in its first 11 minutes, which reads as a freeze. A start that finds such a file now waits for all recorded pipelines, with their count on the boot screen, as a start without the library does; NX2NAVI_PREWARM_FULL=0 keeps the 20 s. The prepared variants still follow in the background. Checked with --probe-prewarm on a cache copy whose header names another device: the line is logged and all 5,775 pipelines are built before the stage ends. Co-Authored-By: Claude Fable 5.2 --- runtime/vk_pipeline.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 185969f5..477430d1 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -761,6 +761,7 @@ static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) { // (vk_cache_file.h cache_file_read): a file past 2 GB is read with the limit raised or off, and refused by // its real size otherwise. static int cache_over_cap; // the last file cache_data_read was asked for is over the limit +static int cache_foreign; // pipeline-cache.bin was written by another device or driver (a driver update) static void *cache_data_read(VkCtx *c, const char *path, size_t *size) { *size = 0; cache_over_cap = 0; @@ -784,6 +785,7 @@ static void *cache_data_read(VkCtx *c, const char *path, size_t *size) { h.deviceID != c->props.deviceID || memcmp(h.pipelineCacheUUID, c->props.pipelineCacheUUID, VK_UUID_SIZE)) { rt_log("vk_pipeline: %s (%.1f MB) was written by another device or driver; starting empty", path, len / 1048576.0); + if (path == P.cache_path) cache_foreign = 1; free(data); return NULL; } @@ -5458,7 +5460,17 @@ static void prewarm_run(void) { __atomic_store_n(&pw_pipe_total, (int64_t)queued, __ATOMIC_RELAXED); __atomic_store_n(&pw_phase, 2, __ATOMIC_RELEASE); const char *full = cfg_get("NX2NAVI_PREWARM_FULL"); - K.until = P.gpl && !(full && !strcmp(full, "1")) ? now_ms() + PREWARM_PIPE_MS : 0; // before the builders start + // A cache file of another device or driver (a driver update: cache_data_read) leaves every recorded pipeline to be + // built from nothing, and the ones the boot left after PREWARM_PIPE_MS were built beside the game: on an 8-core + // integrated GPU that start had 2,141 of 8,122 left after 20 s and took 296 s to finish them, with the game + // drawing fewer than 600 frames in its first 11 minutes. The boot waits for all of them then, with its count on + // the boot screen, as it does without the library; NX2NAVI_PREWARM_FULL=0 keeps the 20 s. + int all = (full && !strcmp(full, "1")) || (cache_foreign && queued && !(full && !strcmp(full, "0"))); + if (P.gpl && all && cache_foreign && !(full && !strcmp(full, "1"))) + rt_log("vk_pipeline: prewarm: the pipeline cache was written by another device or driver, so the start waits " + "for all %u recorded pipelines and not %d s (NX2NAVI_PREWARM_FULL=0: the rest beside the game)", queued, + (int)(PREWARM_PIPE_MS / 1000)); + K.until = P.gpl && !all ? now_ms() + PREWARM_PIPE_MS : 0; // before the builders start PlatThread *t[64]; int nt = 0; if (queued) warm_builder((void *)1); // the first alone (warm_builder) From cd0b1a387d06bbce5d7896e20a4da25d2a34b77d Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 20:03:12 -0500 Subject: [PATCH 15/15] Do the whole warm on the boot screen at the first start after a driver change. A start that finds the pipeline cache written by another device or driver has nothing built, and everything was compiled beside the game: an 8-core integrated GPU drew fewer than 600 frames in its first 11 minutes. Such a start now waits on the "Preparing shaders" screen, with counts, for every recorded pipeline, for the compute pipelines on every core at normal priority, and, when a sample says the driver's own cache lacks the prepared stage libraries, for all of them on every core. The memory gate still holds for the builds, and the library builds write the preparation's journal while the start waits, so a variant the driver's compiler dies on is skipped from the next start on. boot-warm-owed.txt beside the caches names the GPU and driver: a start closed during the warm is followed by one that goes on with it, three at most, and "done" keeps later starts ordinary while the cache file still carries the earlier driver's header. When the boot stops waiting (prewarm_wait_ms, or a minute without progress) the builders step down to the background's thread counts and idle priority. prewarm_full = 0 turns the warm off. With nothing in the driver's cache the probe took 425 s on an RTX 5090 with 16 CPU threads: 87 s for 5,775 recorded pipelines, 19 s for 4,075 compute pipelines and 317 s for 42,231 prepared variants. Co-Authored-By: Claude Fable 5.2 --- AUDIT.md | 6 +- docs/architecture.md | 10 +- docs/building.md | 8 ++ nx2navi.ini.example | 12 +- runtime/boot.c | 9 +- runtime/boot_warm.h | 39 ++++++ runtime/ui/menu.cpp | 7 +- runtime/ui/menu.h | 5 +- runtime/ui_text.c | 4 + runtime/ui_text.h | 3 + runtime/vk.h | 9 +- runtime/vk_pipeline.c | 289 +++++++++++++++++++++++++++++++++------ tests/boot_warm_check.c | 61 +++++++++ tests/boot_warm_check.py | 40 ++++++ 14 files changed, 443 insertions(+), 59 deletions(-) create mode 100644 runtime/boot_warm.h create mode 100644 tests/boot_warm_check.c create mode 100644 tests/boot_warm_check.py diff --git a/AUDIT.md b/AUDIT.md index bfa900a8..a7346cea 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -301,12 +301,12 @@ platform's seven levels; "normal" when nothing sets one. |---|---|---|---|---| | "pipe worker" | `vk_pipeline.c` `pipe_worker` (~3533) | `workers_start` (~3623), first use | `plat_cpu_usable()/4` clamped 2..`MAX_WORKERS` (4). Below normal for prebuild jobs, normal otherwise | Never | | "pipe lib worker" | `vk_pipeline.c` `lib_worker` (~3500) | `workers_start` (~3638), with GPL and `NX2NAVI_PIPE_WAIT` not `draw` | Pipe-worker count or 2 | Never | -| "pipe prewarm" | `vk_pipeline.c` `warm_builder` (~4015) | `prewarm_run` (~4186) on the prewarm thread | min(cpus, 64) | Joined; ends at `PREWARM_PIPE_MS` (20 s) unless `NX2NAVI_PREWARM_FULL=1` | +| "pipe prewarm" | `vk_pipeline.c` `warm_builder` (~4015) | `prewarm_run` (~4186) on the prewarm thread | min(cpus, 64) | Joined; ends at `PREWARM_PIPE_MS` (20 s) unless `NX2NAVI_PREWARM_FULL=1` or the whole warm after a driver change (`bf_decide`) | | "pipe list" | `warm_builder` (~5942) | `seed_pipelines`, preparation modes only | min(cpus, 64) | Joined | | "big compute" | `vk_pipeline.c` `bg_main` (~5140) | `bg_start` (~5252) by the last `lb_worker`; `NX2NAVI_LIB_PREBUILD_BIG_BG` (default 1) | 1, below normal; raised to normal when a dispatch waits for it | Joined in `lb_stop` | -| "lib prebuild" | `vk_pipeline.c` `lb_worker` (~5418) | `lb_start` (~5707) from the boot prewarm; `NX2NAVI_LIB_PREBUILD` | `NX2NAVI_LIB_PREBUILD_THREADS` or 2 (max 16). Idle priority, normal around `P.lock` and shader loads | When the list is done | +| "lib prebuild" | `vk_pipeline.c` `lb_worker` (~5418) | `lb_start` (~5707) from the boot prewarm; `NX2NAVI_LIB_PREBUILD` | `NX2NAVI_LIB_PREBUILD_THREADS` or 2 (max 16). Idle priority, normal around `P.lock` and shader loads. In the whole warm after a driver change (`bf_wait`): one per core (max 16) at normal priority while the start waits | When the list is done | | "lib warm check" | `vk_pipeline.c` `lw_main` (~5572) | `lw_start` (~5649); `NX2NAVI_LIB_WARM` (default 1) | 1, below normal | Joined in `lw_stop` | -| "lib warm" | `vk_pipeline.c` `lw_worker` (~5532) | `lw_main` when the driver cache is cold | cpus/4 clamped 2..6, idle | Joined in `lw_stop` | +| "lib warm" | `vk_pipeline.c` `lw_worker` (~5532) | `lw_main` when the driver cache is cold | cpus/4 clamped 2..6, idle. In the whole warm after a driver change (`bf_wait`): one per core (max 32) at normal priority while the start waits | Joined in `lw_stop` | | "lib warm" | `vk_pipeline.c` `lb_warm_worker` (~5733) | `lb_warm_prepared` (~5794), preparation modes only | min(cpus, 64) | Joined | | "pipe autosave" | `vk_pipeline.c` `autosave_main` (~6390) | `vk_pipeline_autosave` (~6402) from the present thread, once | 1, detached, below normal, 500 ms loop | Never; not stopped by `vk_pipeline_shutdown` (lead L-15) | diff --git a/docs/architecture.md b/docs/architecture.md index 23c3f05a..dbf84309 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -114,8 +114,14 @@ keeps per-core data without locks and relies on that, so the host models it (`ru `menu_boot_frame`). With the graphics pipeline library the boot builds recorded pipelines in record order for at most 20 s after the shaders are ready and leaves the rest to the pipeline workers below normal priority; a draw that meets one before it is built gets a fast-linked pipeline (`vk_pipeline.c` `PREWARM_PIPE_MS`). - `NX2NAVI_PREWARM_FULL=1`, and drivers without the library, wait for every pipeline. `NX2NAVI_PREWARM_WAIT_MS` caps - the whole wait, after which the rest finishes in the background. + `NX2NAVI_PREWARM_FULL=1`, and drivers without the library, wait for every pipeline. A start that finds the + pipeline cache written by another GPU or driver waits for the whole warm on that screen, since nothing is built + then and all of it would compile beside the game: every recorded pipeline, the compute pipelines, and the + prepared shader variants built into the driver's own cache when a sample says it lacks them (`vk_pipeline.c` + `bf_decide`, `bf_wait`). `boot-warm-owed.txt` next to the caches lets a start go on with a warm that an earlier + one began and did not end, three starts at most, and says when the warm for a GPU and driver is done + (`runtime/boot_warm.h`); `NX2NAVI_PREWARM_FULL=0` turns it off. + `NX2NAVI_PREWARM_WAIT_MS` caps the whole wait, after which the rest finishes in the background. - Translator updates: every cache key names the translator, so a new one has no translations. The cache carries the previous translator's (`NX2NAVI_SHADER_CARRY`, `runtime/gfx_shader.c` "Carry"): `shader-cache/identities.txt` lists the translators it holds translations of, and a shader the new translator has not translated yet uses the diff --git a/docs/building.md b/docs/building.md index 825112c1..f8b37ab2 100644 --- a/docs/building.md +++ b/docs/building.md @@ -472,6 +472,14 @@ A change to the start's memory warning (`runtime/mem_need.h`, `runtime/memory.c` `python tests\mem_need_check.py []`, which builds the header alone and checks when the game's need counts as short. With the executable it also runs `--probe-memneed` (it prints the machine's commit and the answer and starts nothing) with `mem_need_mb` at 0, at 1 and at a value no machine has, and reads the warning line of the last. +A change to when a start does the whole warm on the boot screen (`runtime/boot_warm.h`, `runtime/vk_pipeline.c` +`bf_decide`) passes `python tests\boot_warm_check.py`, which builds the header alone: the warm after a driver change, +its continuation by a later start, the end of the tries, and no second warm once one is done. The warm itself shows +in `--probe-prewarm` on a copy of an install's caches whose `pipeline-cache.bin` has another driver's header (bytes +8 to 15 of the file are the vendor and device ids, the 16 after them the driver's cache id): the log names the whole +warm, the compute pipelines on every core and the library warm's answer, and `boot-warm-owed.txt` ends as "done". +Run under an exe name the driver has not seen, the probe compiles everything, as after a real driver update. What +the boot screen shows, and the builders' step down when the wait is cut short, need a start of the game. A change to how `pipeline-cache.bin` is read (`runtime/vk_cache_file.h`, `runtime/vk_pipeline.c` `cache_load`) passes `python tests\pipeline_cache_file_check.py`, which builds the header alone and reads a sparse file just over 2 GB through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU; diff --git a/nx2navi.ini.example b/nx2navi.ini.example index ac545fef..6336bd9a 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -220,9 +220,15 @@ # draw, from the pipeline cache, and built at the start from then on. # 1: always built at the start. 0: never; a first visit to an area can then hold a frame for a moment. # pipe_seed_boot = auto -# 1: the start waits until every known pipeline is built. Default 0: with the graphics pipeline library the start -# builds them for at most 20 seconds and the rest in the background while you play. -# prewarm_full = 0 +# How long the start waits for known pipelines on the "Preparing shaders" screen. +# auto (default): with the graphics pipeline library the start builds them for at most 20 seconds and the rest in the +# background while you play. The first start after a graphics driver update (or with another GPU) waits for +# everything instead, because nothing is built then and the game would crawl while it all compiles beside it: every +# known pipeline, the compute pipelines and the driver's own shader cache. That takes minutes, once per driver; a +# start closed during it goes on with it the next time (three starts at most). +# 1: every start waits until every known pipeline is built. +# 0: never more than the 20 seconds, also after a driver update. +# prewarm_full = auto # Threads that translate known shaders at start (1 to 64; default: half the CPU threads, at most 16). # prewarm_xlate = 8 # After an update that brings a new shader translator, 1 keeps using the shader cache's translations from the earlier diff --git a/runtime/boot.c b/runtime/boot.c index 76f7b984..12203c1c 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -380,6 +380,7 @@ static void boot_wait(PlatThread *warm, uint64_t cap_ms) { if ((cap_ms && now - t0 >= cap_ms) || now - moved >= BOOT_STALL_MS) { plat_thread_detach(warm); vk_pipeline_prewarm_marker_clear(); + vk_pipeline_prewarm_detach(); rt_log("boot: the prewarm goes on in the background after %llu ms (%s)", (unsigned long long)(now - t0), cap_ms && now - t0 >= cap_ms ? "NX2NAVI_PREWARM_WAIT_MS" : "no progress for 60 s"); break; @@ -423,9 +424,11 @@ void rt_boot(void) { uint64_t held = plat_now_ms(); // The guest waits for all of the prewarm (boot_wait): a warm start is ready in about half a second; a cold one (a new // translator, GPU or driver) translated 2,987 shaders in about 11 s and then builds the recorded pipelines, with the - // boot screen showing its progress. Started early instead, the game would meet those shaders in play and wait for - // each one there. NX2NAVI_PREWARM_WAIT_MS caps the wait (0, the default: none); past it the prewarm goes on in the - // background. A --shot image waits for all of it without the screen, so that it does not depend on build timing. + // boot screen showing its progress; after a driver change also the compute pipelines and the driver's cache of + // stage libraries (vk_pipeline.c bf_decide). Started early instead, the game would meet those shaders in play and + // wait for each one there. NX2NAVI_PREWARM_WAIT_MS caps the wait (0, the default: none); past it the prewarm goes + // on in the background. A --shot image waits for all of it without the screen, so that it does not depend on build + // timing. const char *pw = cfg_get("NX2NAVI_PREWARM_WAIT_MS"); uint64_t wait_ms = pw && *pw ? strtoull(pw, NULL, 10) : 0; if (warm && rt_config()->shot) plat_thread_join(warm); diff --git a/runtime/boot_warm.h b/runtime/boot_warm.h new file mode 100644 index 00000000..dc65a9d7 --- /dev/null +++ b/runtime/boot_warm.h @@ -0,0 +1,39 @@ +// Whether a start does the whole warm on the boot screen (vk_pipeline.c bf_decide, where the reasons are). Own header +// so tests/boot_warm_check.c checks it alone. +// +// /boot-warm-owed.txt is " " or " done". +// The start that finds the pipeline cache written by another device or driver writes it with 1; a start that finds +// its own GPU and driver in it with a count goes on with the warm and counts up; the start whose warm ends writes +// "done". The pipeline cache's file keeps the earlier driver's header for some starts after that (what a run compiles +// goes to the play cache until that is folded in), and "done" is what keeps those starts ordinary ones. After +// BOOT_WARM_TRIES starts that began the warm and did not end it the file stays as it is, and starts on that GPU and +// driver are ordinary ones too. +#pragma once + +#include +#include +#include + +#define BOOT_WARM_TRIES 3 + +// NO: nothing owed (a file there is another GPU's or driver's, or not one: the caller deletes it). YES: this start +// does the warm. GAVE_UP and DONE: an ordinary start, and the file stays. +enum { BOOT_WARM_NO, BOOT_WARM_YES, BOOT_WARM_GAVE_UP, BOOT_WARM_DONE }; + +// text: the file's content, NULL without the file. key: this GPU and driver, without spaces. foreign: the pipeline +// cache was written by another device or driver. *tries: the count to write with BOOT_WARM_YES, 0 otherwise. +static inline int boot_warm_decide(const char *text, const char *key, int foreign, int *tries) { + char was[64] = "", tok[16] = "", *end = tok; + long n = -1; + *tries = 0; + if (text && sscanf(text, "%63s %15s", was, tok) == 2 && !strcmp(was, key)) { + if (!strcmp(tok, "done")) return BOOT_WARM_DONE; + n = strtol(tok, &end, 10); + if (end == tok || *end || n < 0) n = -1; + } + if (n < 0 && !foreign) return BOOT_WARM_NO; + if (n < 0) n = 0; // this GPU and driver's first start + if (n >= BOOT_WARM_TRIES) return BOOT_WARM_GAVE_UP; + *tries = (int)n + 1; + return BOOT_WARM_YES; +} diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 14acb19b..71b02de8 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -2222,8 +2222,9 @@ extern "C" int menu_boot_frame(const MenuTarget *t) { int phase = g_boot_phase.load(std::memory_order_acquire); long long done = g_boot_done.load(std::memory_order_relaxed), total = g_boot_total.load(std::memory_order_relaxed); char step[160]; - if (phase == 1 || phase == 2) - snprintf(step, sizeof step, ui_text(phase == 1 ? UI_BOOT_SHADERS : UI_BOOT_PIPELINES), done, total); + static const UiTextId steps[] = {UI_BOOT_SHADERS, UI_BOOT_PIPELINES, UI_BOOT_COMPUTE, UI_BOOT_DRIVER_CACHE}; + if (phase >= 1 && phase <= 4) + snprintf(step, sizeof step, ui_text(steps[phase - 1]), done, total); else snprintf(step, sizeof step, "%s", ui_text(UI_BOOT_STARTING)); const char *title = ui_text(UI_BOOT_TITLE); @@ -2234,7 +2235,7 @@ extern "C" int menu_boot_frame(const MenuTarget *t) { ImFont *head = g_title_font ? g_title_font : body; float head_px = g_title_font ? g_title_font->FontSize * io.FontGlobalScale : body_px * 1.2f; // as GetFontSize scales float frac = -1; - if ((phase == 1 || phase == 2) && total > 0) frac = done <= 0 ? 0 : done >= total ? 1 : (float)done / (float)total; + if (phase >= 1 && phase <= 4 && total > 0) frac = done <= 0 ? 0 : done >= total ? 1 : (float)done / (float)total; float s = (float)fmod((now - g_boot_t0) / 1e9 * 0.8, 2.0); skin_boot_panel(ImGui::GetForegroundDrawList(), head, head_px, body, body_px, title, step, frac, s < 1 ? s : 2 - s, (float)t->width, (float)t->height, scale); diff --git a/runtime/ui/menu.h b/runtime/ui/menu.h index cbcf2f47..06aae8b5 100644 --- a/runtime/ui/menu.h +++ b/runtime/ui/menu.h @@ -61,8 +61,9 @@ int menu_overlay_frame(uint64_t frame, const MenuTarget *t, const char **shot); // Records the frame menu_overlay_frame laid out into cb, which is inside dynamic rendering on the image. void menu_overlay_draw(VkCommandBuffer cb); // The boot screen while boot.c waits for the prewarm: phase as VkPrewarmProgress.phase (0 starting, 1 shaders, -// 2 pipelines) with its count. menu_boot_frame lays out the screen for t like menu_overlay_frame (1: draw it with -// menu_overlay_draw); only before the guest runs, from the thread that presents then. +// 2 pipelines, 3 compute pipelines, 4 the driver's cache) with its count. menu_boot_frame lays out the screen for t +// like menu_overlay_frame (1: draw it with menu_overlay_draw); only before the guest runs, from the thread that +// presents then. void menu_boot_set(int phase, long long done, long long total); int menu_boot_frame(const MenuTarget *t); void menu_vk_release(void); // before the device goes away (gfx_window_release_vk) diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 808bc33e..5349e42b 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -766,6 +766,7 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_COOP_ACCEPTED_AS] = "a key you accepted as %s", [UI_BOOT_TITLE] = "Preparing shaders", [UI_BOOT_STARTING] = "Starting", [UI_BOOT_SHADERS] = "Shaders: %lld of %lld", [UI_BOOT_PIPELINES] = "Pipelines: %lld of %lld", + [UI_BOOT_COMPUTE] = "Compute pipelines: %lld of %lld", [UI_BOOT_DRIVER_CACHE] = "Driver cache: %lld of %lld", [UI_MENU_UPSCALING] = "Upscaling", [UI_MENU_UPSCALING_DESC] = "How the picture is smoothed and scaled to your window.", [UI_MENU_UPSCALING_DESC_FSR] = "How the picture is smoothed and scaled to your window.", @@ -1466,6 +1467,7 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_COOP_ACCEPTED_AS] = "ключ, который вы приняли как %s", [UI_BOOT_TITLE] = "Подготовка шейдеров", [UI_BOOT_STARTING] = "Запуск", [UI_BOOT_SHADERS] = "Шейдеры: %lld из %lld", [UI_BOOT_PIPELINES] = "Конвейеры: %lld из %lld", + [UI_BOOT_COMPUTE] = "Вычислительные конвейеры: %lld из %lld", [UI_BOOT_DRIVER_CACHE] = "Кэш драйвера: %lld из %lld", [UI_MENU_CHEATS_REBUILD] = "Пересобрать игру сейчас", [UI_MENU_CHEATS_REBUILD_DESC] = "Некоторые читы меняют код игры и ещё не собраны в неё: они новые или изменены, " "либо прошлая пересборка не удалась или была отменена. Закрывает игру; установщик " @@ -2152,6 +2154,7 @@ static const char *const text_pt_br[UI_TEXT_COUNT] = { [UI_MENU_COOP_ACCEPTED_AS] = "uma chave que você aceitou como %s", [UI_BOOT_TITLE] = "Preparando shaders", [UI_BOOT_STARTING] = "Iniciando", [UI_BOOT_SHADERS] = "Shaders: %lld de %lld", [UI_BOOT_PIPELINES] = "Pipelines: %lld de %lld", + [UI_BOOT_COMPUTE] = "Pipelines de computação: %lld de %lld", [UI_BOOT_DRIVER_CACHE] = "Cache do driver: %lld de %lld", [UI_MENU_CHEATS_REBUILD] = "Recompilar o jogo agora", [UI_MENU_CHEATS_REBUILD_DESC] = "Alguns cheats alteram o código do jogo e ainda não foram compilados nele: são " "novos ou editados, ou a última recompilação falhou ou foi cancelada. Fecha o " @@ -2898,6 +2901,7 @@ static const char *const text_zh_hans[UI_TEXT_COUNT] = { [UI_MENU_COOP_ACCEPTED_AS] = "你曾以“%s”之名接受的身份指纹", [UI_BOOT_TITLE] = "正在准备着色器", [UI_BOOT_STARTING] = "正在启动", [UI_BOOT_SHADERS] = "着色器:%lld / %lld", [UI_BOOT_PIPELINES] = "管线:%lld / %lld", + [UI_BOOT_COMPUTE] = "计算管线:%lld / %lld", [UI_BOOT_DRIVER_CACHE] = "驱动缓存:%lld / %lld", [UI_MENU_CHEATS_REBUILD] = "立即重新构建游戏", [UI_MENU_CHEATS_REBUILD_DESC] = "有些金手指会修改游戏代码,但尚未构建进游戏:它们是新增或修改的,或上次重新构建失败或被取消。将关闭游戏;安装程序会把它们构建进去(几分钟)并重新启动游戏。上次存档之后的进度会丢失。", [UI_MENU_MODS_EXEFS_UNSUPPORTED] = "此模组修改游戏代码(exefs)的方式本移植无法应用,因此整个模组保持关闭。%s", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 4de41427..7a2352d8 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -310,6 +310,9 @@ typedef enum { // the boot screen while the boot prewarm prepares shaders and pipelines (boot.c, ui/menu.cpp menu_boot_frame) UI_BOOT_TITLE, UI_BOOT_STARTING, UI_BOOT_SHADERS, UI_BOOT_PIPELINES, // printf: "%lld" done, "%lld" total + // after a driver change (vk_pipeline.c bf_decide): the compute pipelines, then the prepared shader variants built + // into the driver's own cache; printf as above + UI_BOOT_COMPUTE, UI_BOOT_DRIVER_CACHE, // the settings menu's Picture rows for this game (ui/menu.cpp tab_screen): one upscaling choice (bilinear, FSR 1 // or a DLSS mode) and the picture's resolution UI_MENU_UPSCALING, UI_MENU_UPSCALING_DESC, UI_MENU_FRAMEGEN, UI_MENU_FRAMEGEN_DESC, diff --git a/runtime/vk.h b/runtime/vk.h index 871233e8..6fe33822 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -943,10 +943,15 @@ void vk_pipeline_prewarm(void); // Ends the boot prebuild's crash marker (vk_pipeline.c, "boot prebuild crash marker"): boot_wait calls it when it lets // the game start while the prewarm goes on, so a crash in play is not taken for one in the boot's builds. void vk_pipeline_prewarm_marker_clear(void); +// boot_wait calls it when it lets the game start while the prewarm goes on: the builders of the whole warm after a +// driver change (vk_pipeline.c bf_decide) go on as in play, at idle priority and in the background's numbers. +void vk_pipeline_prewarm_detach(void); typedef struct { - int phase; // 0 starting (device, key file), 1 shaders, 2 pipelines, 3 done + // 0 starting (device, key file), 1 shaders, 2 pipelines, 5 done. Only in the whole warm after a driver change + // (vk_pipeline.c bf_decide): 3 compute pipelines, 4 the prepared variants built into the driver's cache. + int phase; int64_t sh_done, sh_total, sh_translated; // gfx_shader_prewarm_progress - int64_t pipe_done, pipe_total; // pipelines built or failed of those queued + int64_t pipe_done, pipe_total; // pipelines built or failed of those queued; phases 3 and 4: their counts } VkPrewarmProgress; void vk_pipeline_prewarm_progress(VkPrewarmProgress *p); // any thread, while vk_pipeline_prewarm runs or after VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 477430d1..bf69d66b 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -46,6 +46,7 @@ #include "mesh_emu_spirv.h" #include "gfx.h" #include "host.h" +#include "boot_warm.h" #include "mem_need.h" #include "nvn_hmap.h" #include "platform/platform.h" @@ -5285,7 +5286,7 @@ static int pm_skipped(void); static void pm_set(void); // vk_pipeline_prewarm_progress's numbers: the phase, and the pipelines finished (built or failed) of those queued, copied -// while prewarm_run waits for its builders. +// while prewarm_run waits for its builders; in phases 3 and 4 (bf_wait) the compute pipelines and the prepared variants. static int32_t pw_phase; static int64_t pw_pipe_done, pw_pipe_total; @@ -5298,19 +5299,39 @@ void vk_pipeline_prewarm_progress(VkPrewarmProgress *p) { static void prewarm_run(void); +// The whole warm on the boot screen after a driver change (comment at bf_decide). +static struct { + int on; // this start waits for all of it + int lw; // lb_start left the library warm's start to bf_wait + volatile int32_t game; // the wait is over: the warm ended, or the boot let the game start beside it + char owed[PLAT_PATH_MAX * 2 + 48]; // boot-warm-owed.txt +} BF; + +// 1 while the start waits for the builders of the whole warm: they run at normal priority on every core then, and as +// in play (idle priority, the background's thread counts) from the moment the game starts. +static int bf_waiting(void) { + return BF.on && !__atomic_load_n(&BF.game, __ATOMIC_ACQUIRE); +} + +void vk_pipeline_prewarm_detach(void) { + __atomic_store_n(&BF.game, 1, __ATOMIC_RELEASE); +} + +static void bf_decide(void); +static void bf_wait(void); + // The boot's share of the pipeline prebuild with the graphics pipeline library. The boot builds the recorded // pipelines in record order, so those met first in play (the title screen, the first places) come first, for at most // this long after the shaders are ready; the rest stay queued for the pool workers, which build them below normal // priority while the game runs. A draw that meets one of them before it is built gets a pipeline linked from stage // libraries at once (gpl_link), as for any new pipeline in play. After a driver update, when every pipeline is built -// from nothing, the boot thus waits this long instead of for all of them (15 minutes and more with a long play -// history). Without the library a draw would have to wait for a whole build, so the boot waits for all; so it does -// with NX2NAVI_PREWARM_FULL=1. +// from nothing, the boot waits for all of them and for the rest of the warm (bf_decide). Without the library a draw +// would have to wait for a whole build, so the boot waits for all; so it does with NX2NAVI_PREWARM_FULL=1. #define PREWARM_PIPE_MS 20000.0 void vk_pipeline_prewarm(void) { prewarm_run(); - __atomic_store_n(&pw_phase, 3, __ATOMIC_RELEASE); + __atomic_store_n(&pw_phase, 5, __ATOMIC_RELEASE); } // NX2NAVI_PIPE_SEED_BOOT (auto, the default; 1; 0): whether the boot builds the pipelines of the shared list that this @@ -5404,6 +5425,7 @@ static void prewarm_run(void) { return; } pm_set(); + bf_decide(); uint32_t queued = 0, dropped = 0, known = 0, listed = 0; int at_boot = seed_boot(); plat_mutex_lock(&P.lock); @@ -5462,14 +5484,12 @@ static void prewarm_run(void) { const char *full = cfg_get("NX2NAVI_PREWARM_FULL"); // A cache file of another device or driver (a driver update: cache_data_read) leaves every recorded pipeline to be // built from nothing, and the ones the boot left after PREWARM_PIPE_MS were built beside the game: on an 8-core - // integrated GPU that start had 2,141 of 8,122 left after 20 s and took 296 s to finish them, with the game - // drawing fewer than 600 frames in its first 11 minutes. The boot waits for all of them then, with its count on - // the boot screen, as it does without the library; NX2NAVI_PREWARM_FULL=0 keeps the 20 s. - int all = (full && !strcmp(full, "1")) || (cache_foreign && queued && !(full && !strcmp(full, "0"))); - if (P.gpl && all && cache_foreign && !(full && !strcmp(full, "1"))) - rt_log("vk_pipeline: prewarm: the pipeline cache was written by another device or driver, so the start waits " - "for all %u recorded pipelines and not %d s (NX2NAVI_PREWARM_FULL=0: the rest beside the game)", queued, - (int)(PREWARM_PIPE_MS / 1000)); + // integrated GPU that start had 2,141 of 8,122 left after 20 s and took 296 s to finish them. The whole warm + // (bf_decide) waits for all of them, with its count on the boot screen, as a start without the library does. + int all = (full && !strcmp(full, "1")) || (BF.on && queued); + if (P.gpl && all && BF.on && !(full && !strcmp(full, "1"))) + rt_log("vk_pipeline: prewarm: the start waits for all %u recorded pipelines and not %d s (the whole warm after a " + "driver change)", queued, (int)(PREWARM_PIPE_MS / 1000)); K.until = P.gpl && !all ? now_ms() + PREWARM_PIPE_MS : 0; // before the builders start PlatThread *t[64]; int nt = 0; @@ -5512,6 +5532,7 @@ static void prewarm_run(void) { vk_mem_stage_log("the recorded pipelines' prebuild"); vk_pipeline_prewarm_marker_clear(); // the boot's share is done; what the pool builds later runs during play lb_start(pr_hist, fs_hist); + if (BF.on) bf_wait(); } // ---- boot prebuild crash marker @@ -5576,6 +5597,65 @@ void vk_pipeline_prewarm_marker_clear(void) { if (plat_xchg32(&pm_on, 0)) plat_file_delete(PM_run); } +// ---- the whole warm on the boot screen after a driver change +// A start that finds pipeline-cache.bin written by another device or driver (cache_data_read: a driver update, another +// GPU) has nothing built: not the recorded pipelines, not the compute pipelines, and the driver's own cache of stage +// libraries is empty as well. Left to the background as on any other start, all of it was compiled beside the game: +// an 8-core integrated GPU drew fewer than 600 frames in its first 11 minutes, with the recorded pipelines done after +// 296 s and the compute pipelines after 570 s, the game's own compute dispatches compiling on the replay thread until +// then. Such a start waits for all of it on the boot screen instead, with its counts (boot.c boot_wait): the recorded +// pipelines (prewarm_run, phase 2), then the compute pipelines on up to LB_MAX_THREADS threads at normal priority +// (lb_start, phase 3), then, when a sample says the driver's cache does not hold the prepared stage libraries +// (lw_check), every prepared variant on up to LW_BOOT_THREADS threads (lw_worker, phase 4). The memory gate +// (gate_wait) holds for all of them. The large compute shaders (bg_main) are not waited for: their one thread runs +// beside the library phase. +// It takes minutes (with nothing in the driver's cache, an RTX 5090 and 16 CPU threads: 87 s for 5,775 recorded +// pipelines, 19 s for 4,075 compute pipelines, 317 s for 42,231 prepared variants), once per driver: +// /boot-warm-owed.txt names the GPU and driver and counts the starts that began the warm (boot_warm.h). A start +// that finds its own GPU and driver there with a count goes on with a warm that an earlier start did not end; what +// was built comes back from the caches in a moment. The start whose warm ends writes "done" there, which keeps the +// next starts ordinary ones: pipeline-cache.bin keeps the earlier driver's header until the play cache is folded +// into it (cache_load), some starts later. After BOOT_WARM_TRIES starts that did not end the warm the file stays as +// well, and starts on this GPU and driver are ordinary ones again. While the boot waits, the library builds write +// the preparation's journal (journal_mark), so a variant the driver's compiler dies on is skipped from the next +// start on (skip_init) instead of ending every start. +// NX2NAVI_PREWARM_FULL=0 turns it off. NX2NAVI_PREWARM_WAIT_MS and a minute without progress end the wait as for any +// prewarm (boot_wait); what is left then runs as in play (vk_pipeline_prewarm_detach). Not for a --shot image, which +// no build timing may change and which nobody watches. +static void bf_decide(void) { + const char *full = cfg_get("NX2NAVI_PREWARM_FULL"); + char dir[PLAT_PATH_MAX * 2], key[32], line[100]; + if ((full && !strcmp(full, "0")) || rt_config()->shot || !cache_dir(dir, sizeof dir)) return; + snprintf(BF.owed, sizeof BF.owed, "%s/boot-warm-owed.txt", dir); + gpu_key(key, sizeof key); + uint64_t n = 0; + char *f = (char *)plat_file_read(BF.owed, &n); + if (f) snprintf(line, sizeof line, "%.*s", (int)(n < sizeof line - 1 ? n : sizeof line - 1), f); + int tries = 0, what = boot_warm_decide(f ? line : NULL, key, cache_foreign, &tries); + if (what == BOOT_WARM_NO) { + if (f) plat_file_delete(BF.owed); // another GPU's or driver's + } else if (what == BOOT_WARM_DONE) { + if (cache_foreign) + rt_log("vk_pipeline: the whole warm for this GPU and driver was done by an earlier start (%s); the " + "pipeline cache's file keeps the earlier driver's header until the play cache is folded into it", + BF.owed); + } else if (what == BOOT_WARM_GAVE_UP) { + rt_log("vk_pipeline: the whole warm after a driver change did not end in %d starts on this GPU and driver (%s): " + "this start builds in the background like any other. Delete %s to have it tried again", BOOT_WARM_TRIES, + key, BF.owed); + } else { + int len = snprintf(line, sizeof line, "%s %d\n", key, tries); + if (!plat_file_write_atomic(BF.owed, line, (size_t)len)) rt_log("vk_pipeline: cannot write %s", BF.owed); + BF.on = 1; + rt_log("vk_pipeline: the whole warm on the boot screen (%s, start %d of %d at most): the start waits for the " + "recorded pipelines, the compute pipelines and the driver's cache of stage libraries. " + "NX2NAVI_PREWARM_FULL=0 leaves them to the background", + cache_foreign ? "the pipeline cache was written by another device or driver" : "an earlier start began it", + tries, BOOT_WARM_TRIES); + } + free(f); +} + // ensure_init's hook: without a prewarm (NX2NAVI_PIPE_PREBUILD=0, or the device came up first elsewhere) the records // are not built. Caller holds no lock. static void prebuild_start(void) { @@ -5805,19 +5885,27 @@ static struct { int32_t keep_n; int idle; // build at PLAT_PRIO_IDLE (boot), normal (preparation) int tail_one; // automatic boot policy: only worker 0 takes the warming tail + int bg_threads; // boot: the workers that go on beside the game (lb_next) PlatMutex stat; } LB = {.stat = PLAT_MUTEX_INIT}; static __thread int lb_me = -1; // on a boot prebuild thread (lb_worker): its index in LB.th +// The boot's builders run beside the game, at idle priority; not while the start waits for them (bf_waiting). +static int lb_idle(void) { + return LB.idle && !bf_waiting(); +} + // Claim a boot job without dropping one when the extra workers finish the recent-pipeline prefix. Recent pipelines // retain parallel creation; a single worker continues the complete tail. Explicit thread counts override this // policy, and installer preparation uses its own queue at full concurrency. The CAS makes the prefix boundary // exact even when two workers reach it together. No pipeline creation or dispatch waits on this queue lock. +// The whole warm (bf_wait) starts more workers than go on beside a game: those end here once the game runs. static size_t lb_next(int worker) { int64_t next = __atomic_load_n(&LB.next, __ATOMIC_RELAXED); for (;;) { - if (LB.stop || (size_t)next >= LB.n || - (LB.idle && LB.tail_one && worker > 0 && (size_t)next >= LB.n_keep)) return LB.n; + int bg = lb_idle(); + if (LB.stop || (size_t)next >= LB.n || (bg && worker >= LB.bg_threads) || + (bg && LB.tail_one && worker > 0 && (size_t)next >= LB.n_keep)) return LB.n; if (__atomic_compare_exchange_n(&LB.next, &next, next + 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) return (size_t)next; } @@ -5838,11 +5926,13 @@ static size_t lb_next(int worker) { #define JR_SLOTS 64 // lb_warm_prepared's thread cap static struct { - PlatFile *journal; // while lb_warm_prepared builds + PlatFile *journal; // while lb_warm_prepared builds, and while the start waits for the library warm (bf_wait) + PlatMutex jlock; // journal: bf_wait and the process's end close it while workers write + char jpath[PLAT_PATH_MAX * 2 + 48]; // the boot's journal, "" without one uint8_t (*iid)[32]; // the skip list's variants for this GPU and driver size_t n; int loaded; -} SK; +} SK = {.jlock = PLAT_MUTEX_INIT}; static int cache_dir(char *dir, size_t n) { // the folder of pipeline-cache.bin and shader-prepared.bin snprintf(dir, n, "%s", P.cache_path); @@ -5901,8 +5991,9 @@ static void skip_init(const char *dir) { out[len] = 0; // parsed below as the list's text } if (added && plat_file_write_atomic(sp, out, len)) { - rt_log("vk_pipeline: the last shader preparation died while the driver built %d variants; %s now lists them " - "and preparation skips them on this GPU and driver (%s)", added, sp, key); + rt_log("vk_pipeline: the last shader preparation, or the last start's warm on the boot screen, died while " + "the driver built %d variants; %s now lists them and they are skipped ahead of play on this GPU and " + "driver (%s)", added, sp, key); free(s), s = out, sn = len, out = NULL; } else if (added) { rt_log("vk_pipeline: cannot write %s; %s stays for the next run", sp, jp); @@ -5945,10 +6036,44 @@ static void skip_count(void) { // Journal slot `slot`: the variant e its worker hands to the driver next, or zeros (e NULL) when the build returned. static void journal_mark(int slot, const GfxWarmEnt *e) { - if (!SK.journal || slot < 0 || slot >= JR_SLOTS) return; + if (slot < 0 || slot >= JR_SLOTS) return; uint8_t rec[JR_SLOT] = {0}; if (e) memcpy(rec, e->iid, 32), rec[32] = e->stage; - plat_file_pwrite(SK.journal, rec, sizeof rec, (uint64_t)slot * JR_SLOT); + plat_mutex_lock(&SK.jlock); + if (SK.journal) plat_file_pwrite(SK.journal, rec, sizeof rec, (uint64_t)slot * JR_SLOT); + plat_mutex_unlock(&SK.jlock); +} + +// The boot's journal, for the library builds the start waits for (bf_wait). skip_init has run (lb_start) and has +// dealt with a leftover one. +static void journal_open(void) { + char dir[PLAT_PATH_MAX * 2]; + if (!cache_dir(dir, sizeof dir)) return; + snprintf(SK.jpath, sizeof SK.jpath, "%s/shader-prepare-journal.bin", dir); + PlatFile *j = plat_fs_create(SK.jpath, NULL); + if (!j) { + rt_log("vk_pipeline: cannot create %s: a driver crash in this start's library warm will not be recorded", SK.jpath); + SK.jpath[0] = 0; + return; + } + plat_mutex_lock(&SK.jlock); + SK.journal = j; + plat_mutex_unlock(&SK.jlock); +} + +// Ends the boot's journal: when the library warm is done, when the game starts beside it (a crash in play says +// nothing about the variant a background builder has in hand) and when the process ends in order during the wait +// (vk_pipeline_save_exit), so that only a process that died while the start waited leaves one. The builds still in +// flight write nothing afterwards (journal_mark). +static void journal_close(void) { + if (!SK.jpath[0]) return; + plat_mutex_lock(&SK.jlock); + PlatFile *j = SK.journal; + SK.journal = NULL; + plat_mutex_unlock(&SK.jlock); + if (!j) return; + plat_file_close(j); + plat_file_delete(SK.jpath); } // Compute shaders whose pipeline a thread is compiling now (P.lock): a cold driver takes about 2 s for one of Ocarina's @@ -6323,7 +6448,7 @@ static void lb_lock(void) { static void lb_unlock(void) { plat_mutex_unlock(&P.lock); - if (LB.idle) plat_thread_priority(PLAT_PRIO_IDLE); + if (lb_idle()) plat_thread_priority(PLAT_PRIO_IDLE); } static void lb_count(int kind, int what, double ms) { // what: 0 built, 1 already there, 2 failed, 3 cache only @@ -6614,10 +6739,10 @@ static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep) { // at idle priority under load held that frame for seconds. The dispatch may have come between the unlock and the // call below, which would undo its raise, hence the look at the slot after it. Without a slot or a handle nobody // could raise the thread, so it compiles at normal priority. - if (LB.idle && flight && me) { + if (lb_idle() && flight && me) { plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); if (cs_flight_is_wanted(cs)) plat_thread_priority(PLAT_PRIO_NORMAL); - } else if (LB.idle) { + } else if (lb_idle()) { plat_thread_priority(PLAT_PRIO_NORMAL); } VkPipeline pipe = compute_create(cs, l, big); @@ -6737,7 +6862,7 @@ static void lb_report(const char *how) { // cache) and kept libraries. static void lb_worker(void *arg) { lb_me = (int)(intptr_t)arg; // its handle is LB.th[lb_me] - plat_thread_priority(PLAT_PRIO_IDLE); + if (lb_idle()) plat_thread_priority(PLAT_PRIO_IDLE); for (;;) { size_t i = lb_next(lb_me); if (i >= LB.n) break; @@ -6769,7 +6894,7 @@ static void lb_worker(void *arg) { plat_thread_priority(PLAT_PRIO_NORMAL); // gfx_shader's cache lock, disk reads const GfxShader *sh = keep || LB.e[i].stage != GFX_STAGE_CS ? gfx_shader_load_input(LB.e[i].iid, 0) : gfx_shader_load_private(LB.e[i].iid, &owned); - plat_thread_priority(PLAT_PRIO_IDLE); + if (lb_idle()) plat_thread_priority(PLAT_PRIO_IDLE); if (!sh) { plat_mutex_lock(&LB.stat); LB.noshader++; @@ -6828,14 +6953,17 @@ static void lb_state(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { #define LW_SAMPLE 12 #define LW_COLD_MS 15.0 #define LW_MAX_THREADS 6 +#define LW_BOOT_THREADS 32 // while the start waits for the warm (bf_wait): every core, as the preparation static struct { GfxWarmEnt *e; size_t n; volatile int64_t next, done; volatile int32_t running, stop, started; volatile int32_t short_mem; // the memory gate refused a build for GATE_WAIT_MS: the warm ended for this run - PlatThread *th[LW_MAX_THREADS + 1]; + volatile int32_t checked; // lw_main has returned: the workers it started, if any, are counted in running + PlatThread *th[LW_BOOT_THREADS + 1]; int threads, strikes; + int bg_threads; // the workers that go on beside the game (lw_worker) uint32_t vendor; // the GPU's vendor id, for what a cold cache's log line adds (vk_warm_hint.h) double t0; char stamp[PLAT_PATH_MAX * 2 + 32], id[64]; @@ -6864,10 +6992,14 @@ static double lw_build(const GfxWarmEnt *e) { } static void lw_worker(void *arg) { - (void)arg; - plat_thread_priority(PLAT_PRIO_IDLE); + int me = (int)(intptr_t)arg, idle = 0; // me: its journal slot while the start waits (bf_wait) no_cache = 1; // into the driver's own cache only (see no_cache) for (;;) { + // Beside the game: idle priority, and of the workers the whole warm started only the background's count. + if (!bf_waiting()) { + if (me >= LW.bg_threads) break; + if (!idle) plat_thread_priority(PLAT_PRIO_IDLE), idle = 1; + } size_t i = (size_t)plat_inc64(&LW.next) - 1; if (i >= LW.n || LW.stop) break; // The memory gate (gate_wait): without room the warm waits, and after GATE_WAIT_MS it ends for this run. @@ -6882,7 +7014,9 @@ static void lw_worker(void *arg) { } break; } + journal_mark(me, &LW.e[i]); // from here to its clearing, a process that dies names this variant lw_build(&LW.e[i]); + journal_mark(me, NULL); gate_leave(held); int64_t d = plat_inc64(&LW.done); if (d % 8000 == 0) @@ -6891,7 +7025,7 @@ static void lw_worker(void *arg) { } if (!plat_dec32(&LW.running) && !LW.stop && !LW.short_mem) { lw_stamp("done", LW.strikes); - rt_log("vk_pipeline: library warm: done, %zu prepared variants in %.0f s on %d idle threads; new pipelines " + rt_log("vk_pipeline: library warm: done, %zu prepared variants in %.0f s on %d threads; new pipelines " "link from the driver's cache from here on", LW.n, (now_ms() - LW.t0) / 1000, LW.threads); } } @@ -6917,9 +7051,8 @@ static double lw_probe(void) { return ms[got / 2]; } -static void lw_main(void *arg) { - (void)arg; - plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); +static void lw_check(void) { + if (!bf_waiting()) plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); char was[16] = "", id[64] = ""; int strikes = 0; uint64_t size = 0; @@ -6950,9 +7083,18 @@ static void lw_main(void *arg) { } LW.strikes = resume ? strikes : same && !strcmp(was, "done") ? strikes + 1 : 0; lw_stamp("warming", LW.strikes); - int cpus = (int)plat_cpu_usable(), t = cpus / 4; + int cpus = (int)plat_cpu_usable(), t = cpus / 4, boot = bf_waiting(); t = t < 2 ? 2 : t > LW_MAX_THREADS ? LW_MAX_THREADS : t; - if (resume) + LW.bg_threads = t; + if (boot) t = cpus < 1 ? 1 : cpus > LW_BOOT_THREADS ? LW_BOOT_THREADS : cpus; + if (boot) { + char why[64]; + if (resume) snprintf(why, sizeof why, "an earlier run began the warm and did not end it"); + else snprintf(why, sizeof why, "a sample's median %.0f ms", median); + rt_log("vk_pipeline: library warm: the driver's cache does not have the prepared libraries (%s): building %zu " + "prepared variants into it on %d threads while the start waits (the whole warm after a driver change)", + why, LW.n, t); + } else if (resume) rt_log("vk_pipeline: library warm: going on with the warm an earlier run began: %zu prepared variants on %d " "idle threads", LW.n, t); else // cold right after a finished warm on this driver says more than a first cold (vk_warm_hint.h) @@ -6968,11 +7110,17 @@ static void lw_main(void *arg) { LW.running = t; int started = 0; for (int i = 0; i < t; i++) - if ((LW.th[1 + started] = plat_thread_new(lw_worker, NULL, "lib warm"))) started++; + if ((LW.th[1 + started] = plat_thread_new(lw_worker, (void *)(intptr_t)i, "lib warm"))) started++; LW.threads = started; if (plat_add32(&LW.running, started - t) == 0 && !started) rt_log("vk_pipeline: library warm: no thread started"); } +static void lw_main(void *arg) { + (void)arg; + lw_check(); + plat_xchg32(&LW.checked, 1); +} + // Boot, once the library state is known (lb_state): the prepared list is read and the check starts on its own thread. static void lw_start(void) { const char *mode = cfg_value("NX2NAVI_LIB_WARM"); @@ -6998,7 +7146,7 @@ static void lw_start(void) { static void lw_stop(void) { plat_xchg32(&LW.stop, 1); - for (int i = 0; i <= LW_MAX_THREADS; i++) + for (int i = 0; i <= LW_BOOT_THREADS; i++) if (LW.th[i]) plat_thread_join(LW.th[i]), LW.th[i] = NULL; } @@ -7010,7 +7158,8 @@ static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { const char *be = cfg_get("NX2NAVI_LIB_PREBUILD_BIG"); // SPIR-V words; LB_BIG_WORDS when unset, 0 no limit if (!A.on || !cfg_bool("NX2NAVI_LIB_PREBUILD")) return; lb_state(pr_hist, fs_hist); // the libraries' state, also for lw_start - lw_start(); + if (BF.on) BF.lw = 1; // after the compute pipelines, with the start waiting (bf_wait) + else lw_start(); LB.big_words = be && *be ? strtoull(be, NULL, 10) : LB_BIG_WORDS; char dir[PLAT_PATH_MAX * 2]; if (cache_dir(dir, sizeof dir)) skip_init(dir); @@ -7052,6 +7201,10 @@ static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { int t = te && *te ? atoi(te) : (cpus < 4 ? 1 : 2); // idle priority; a cold driver takes ~2 s per large compute shader LB.tail_one = !(te && *te); t = t < 1 ? 1 : t > LB_MAX_THREADS ? LB_MAX_THREADS : t; + // The whole warm (bf_wait): no game runs beside these builds until they are done, so every core takes them, as + // in the preparation; should the game start first, the count above goes on (lb_next). + LB.bg_threads = t; + if (bf_waiting() && LB.tail_one) t = cpus < 1 ? 1 : cpus > LB_MAX_THREADS ? LB_MAX_THREADS : cpus; if ((size_t)t > n) t = (int)n; LB.t0 = now_ms(); LB.running = t; @@ -7066,11 +7219,12 @@ static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { P.cache_control ? "only from the pipeline caches" : "left to their first dispatch"); rt_log("vk_pipeline: stage library prebuild: %zu compute pipelines, of which %zu stay (the shaders dispatched most " "recently, NX2NAVI_COMPUTE_KEEP) and the others are built into the pipeline cache and destroyed, to be created " - "at their first dispatch; %zu recorded variants' libraries (NX2NAVI_LIB_PREBUILD_KEEP %d); on %d idle-priority " + "at their first dispatch; %zu recorded variants' libraries (NX2NAVI_LIB_PREBUILD_KEEP %d); on %d %s " "threads; library state clamp %u, polygon mode %u, depth -w..w %u, samples %u; %s (NX2NAVI_LIB_PREBUILD_BIG)", - n_cs < n ? n_cs : n, LB.n_keep, n > n_cs ? n - n_cs : 0, LB.keep_n, started, LB.clamp, LB.poly, LB.minus_w, + n_cs < n ? n_cs : n, LB.n_keep, n > n_cs ? n - n_cs : 0, LB.keep_n, started, + bf_waiting() ? "normal-priority (the start waits for them)" : "idle-priority", LB.clamp, LB.poly, LB.minus_w, LB.samples, big); - if (LB.tail_one && started > 1) + if (LB.tail_one && started > 1 && !bf_waiting()) rt_log("vk_pipeline: stage library prebuild: after the %zu recent compute pipelines, one worker continues the " "remaining list; NX2NAVI_LIB_PREBUILD_THREADS overrides this background limit", LB.n_keep); } @@ -7084,6 +7238,58 @@ static void lb_stop(void) { if (BG.th) plat_thread_join(BG.th), BG.th = NULL; // after its current build, if one runs } +// The whole warm's wait (comment at bf_decide), on the prewarm thread after the recorded pipelines: the compute +// pipelines lb_start has just started, then the library warm. The counts go to the boot screen +// (vk_pipeline_prewarm_progress). Should the boot stop waiting (vk_pipeline_prewarm_detach), this thread still sees +// the warm to its end, which the builders then reach as in play. +static void bf_wait(void) { + double t0 = now_ms(); + if (LB.n && LB.threads) { + __atomic_store_n(&pw_pipe_done, 0, __ATOMIC_RELAXED); + __atomic_store_n(&pw_pipe_total, (int64_t)LB.n, __ATOMIC_RELAXED); + __atomic_store_n(&pw_phase, 3, __ATOMIC_RELEASE); + while (__atomic_load_n(&LB.running, __ATOMIC_ACQUIRE) > 0 && !LB.stop) { + plat_sleep_ms(100); + plat_mutex_lock_shared(&LB.stat); + uint64_t d = LB.done; + plat_mutex_unlock_shared(&LB.stat); + __atomic_store_n(&pw_pipe_done, (int64_t)d, __ATOMIC_RELAXED); + } + } + double t1 = now_ms(); + int libs = 0; + if (BF.lw && !LB.stop) { + if (bf_waiting()) journal_open(); + lw_start(); + if (LW.th[0]) { + libs = 1; + __atomic_store_n(&pw_pipe_done, 0, __ATOMIC_RELAXED); + __atomic_store_n(&pw_pipe_total, (int64_t)LW.n, __ATOMIC_RELAXED); + __atomic_store_n(&pw_phase, 4, __ATOMIC_RELEASE); + while ((!LW.checked || LW.running > 0) && !LW.stop) { + plat_sleep_ms(100); + __atomic_store_n(&pw_pipe_done, (int64_t)LW.done, __ATOMIC_RELAXED); + if (!bf_waiting()) journal_close(); // the game runs + } + } + journal_close(); + } + int stopped = LB.stop || LW.stop, beside = __atomic_load_n(&BF.game, __ATOMIC_ACQUIRE); + rt_log("vk_pipeline: the whole warm %s: %zu compute pipelines in %.0f ms, the driver's cache of stage libraries %s in " + "%.0f ms (%lld of %zu prepared variants built)%s", stopped ? "stopped with the process" : "is done", LB.n, + t1 - t0, !libs ? "not warmed (no prepared list, or the graphics pipeline library or NX2NAVI_LIB_WARM is off)" + : LW.done ? "warmed" : "left as it is (the check found it warm, or gave the warm up)", + now_ms() - t1, (long long)LW.done, LW.n, + beside ? "; the boot had stopped waiting, so the game ran beside the end of it" : ""); + if (!stopped) { // done for this GPU and driver (boot_warm.h) + char key[32], line[64]; + gpu_key(key, sizeof key); + int len = snprintf(line, sizeof line, "%s done\n", key); + if (!plat_file_write_atomic(BF.owed, line, (size_t)len)) rt_log("vk_pipeline: cannot write %s", BF.owed); + } + vk_pipeline_prewarm_detach(); // nothing waits from here on: later builders are the background's +} + // Preparation: one prepared variant at a time into the driver's disk cache (private shader copy, nothing kept). arg is // the worker's journal slot. static void lb_warm_worker(void *arg) { @@ -7775,6 +7981,7 @@ void vk_pipeline_save(void) { // The save as the process ends (window closed, the game's exit, a timed run's end): new builds wait while it waits up to // SAVE_WAIT_MS for the running ones to finish (cache_saving_at_exit). void vk_pipeline_save_exit(void) { + journal_close(); // an orderly end during the whole warm is not a driver crash save_with(SAVE_WAIT_MS, 1); } diff --git a/tests/boot_warm_check.c b/tests/boot_warm_check.c new file mode 100644 index 00000000..480c4f03 --- /dev/null +++ b/tests/boot_warm_check.c @@ -0,0 +1,61 @@ +// runtime/boot_warm.h: whether a start does the whole warm on the boot screen (runtime/vk_pipeline.c bf_decide). +// Built and run by tests/boot_warm_check.py; prints each failed check and exits 1 on any. +#include + +#include "boot_warm.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +int main(void) { + const char *key = "1002:15bf:00806001", *other = "1002:15bf:00805ffe"; + char text[128]; + int tries = -1; + + // An ordinary start: no file, the pipeline cache is this driver's. + CHECK(boot_warm_decide(NULL, key, 0, &tries) == BOOT_WARM_NO && tries == 0, "a warm without a driver change"); + + // The first start after a driver update: the warm, and the file gets 1. + CHECK(boot_warm_decide(NULL, key, 1, &tries) == BOOT_WARM_YES && tries == 1, "no warm after a driver change"); + + // That start was closed during the warm. The next one goes on with it and counts up, whether the pipeline cache's + // file still has the earlier driver's header or not. + for (int n = 1; n < BOOT_WARM_TRIES; n++) { + snprintf(text, sizeof text, "%s %d\n", key, n); + CHECK(boot_warm_decide(text, key, 0, &tries) == BOOT_WARM_YES && tries == n + 1, "an owed warm not taken up at %d", n); + CHECK(boot_warm_decide(text, key, 1, &tries) == BOOT_WARM_YES && tries == n + 1, "the count lost at %d", n); + } + + // BOOT_WARM_TRIES starts began it and none ended it: no more, with or without a foreign cache, and nothing to write. + snprintf(text, sizeof text, "%s %d\n", key, BOOT_WARM_TRIES); + CHECK(boot_warm_decide(text, key, 0, &tries) == BOOT_WARM_GAVE_UP && tries == 0, "a fourth try"); + CHECK(boot_warm_decide(text, key, 1, &tries) == BOOT_WARM_GAVE_UP && tries == 0, "a fourth try on a foreign cache"); + + // The warm ended. The cache file keeps the earlier driver's header until the play cache is folded in: those + // starts are ordinary ones, and so are the ones after. + snprintf(text, sizeof text, "%s done\n", key); + CHECK(boot_warm_decide(text, key, 1, &tries) == BOOT_WARM_DONE && tries == 0, "a second warm on the same driver"); + CHECK(boot_warm_decide(text, key, 0, &tries) == BOOT_WARM_DONE && tries == 0, "a done file read as stale"); + + // The file of an earlier driver: stale without a foreign cache, a fresh count with one, whether the earlier + // driver's warm was done or given up. + snprintf(text, sizeof text, "%s %d\n", other, BOOT_WARM_TRIES); + CHECK(boot_warm_decide(text, key, 0, &tries) == BOOT_WARM_NO, "another driver's file taken for this one's"); + CHECK(boot_warm_decide(text, key, 1, &tries) == BOOT_WARM_YES && tries == 1, "another driver's count carried over"); + snprintf(text, sizeof text, "%s done\n", other); + CHECK(boot_warm_decide(text, key, 0, &tries) == BOOT_WARM_NO, "another driver's done taken for this one's"); + CHECK(boot_warm_decide(text, key, 1, &tries) == BOOT_WARM_YES && tries == 1, "another driver's done stops this one's warm"); + + // A file that is not one: as without it. + CHECK(boot_warm_decide("", key, 0, &tries) == BOOT_WARM_NO, "an empty file owes a warm"); + CHECK(boot_warm_decide("garbage", key, 1, &tries) == BOOT_WARM_YES && tries == 1, "a broken file stops the warm"); + snprintf(text, sizeof text, "%s -4\n", key); + CHECK(boot_warm_decide(text, key, 0, &tries) == BOOT_WARM_NO, "a negative count owes a warm"); + snprintf(text, sizeof text, "%s x\n", key); + CHECK(boot_warm_decide(text, key, 0, &tries) == BOOT_WARM_NO, "a count that is no number owes a warm"); + snprintf(text, sizeof text, "%s 2x\n", key); + CHECK(boot_warm_decide(text, key, 0, &tries) == BOOT_WARM_NO, "a count with a tail owes a warm"); + + printf("boot_warm_check: %s\n", fails ? "FAILED" : "passed"); + return fails ? 1 : 0; +} diff --git a/tests/boot_warm_check.py b/tests/boot_warm_check.py new file mode 100644 index 00000000..f5c7950a --- /dev/null +++ b/tests/boot_warm_check.py @@ -0,0 +1,40 @@ +"""Build and run tests/boot_warm_check.c: runtime/boot_warm.h, whether a start does the whole warm on the boot screen +after a driver change (the decision alone; no GPU, no game files). + +Usage: boot_warm_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests' / 'diff')) +try: + from common import MSYS_BIN, compiler # noqa: E402 +except ImportError: # a Python without the differential tests' packages: the same compiler choice without them + MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + def compiler(): + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'boot_warm' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('boot_warm_check.exe' if os.name == 'nt' else 'boot_warm_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'boot_warm_check.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main())