From 0101a8707904a82f51654df39e7a8eafe221ca4e Mon Sep 17 00:00:00 2001 From: chrg6 Date: Mon, 9 Nov 2026 09:55:25 -0500 Subject: [PATCH] Raise the frame rate: fewer copies into the VRAM mirrors, earlier GPU starts, render target images reused. From the local fixes package (fix 11), measured in Kokiri Forest at 4K on an RTX 4090 over PCIe 3.0: 40.5 to 62.6 fps native and 45.6 to 74.9 fps with DLSS Quality, the mirror copies from 6.5 to 0.3 ms of GPU time a frame. - Pages a VRAM mirror wrote back into guest memory are not copied up into it again: the mirror sync reads page stamps of their own that a mirror's write-back leaves alone (vk_gpu_written_back, decided at record time by vk_mem_mirrored). This replaces the lookup table NX2NAVI_WB_NO_REUP had, keeps its name, and is now on by default. - The game's flushes of mirrored memory are not copied where the write watch covers the region, as every page the game wrote there is stamped already (47-57 MiB a frame); NX2NAVI_FLUSH_UPLOAD=1 copies them as before. - Written storage ranges up to NX2NAVI_WB_MERGE_MAX (now 32 MiB) are merged in 4 MiB pieces, so a range is no longer bound to what one dispatch covers; the snapshot pool keeps its budget-scaled size with 128 slots. - A submission's first part goes to the GPU at the first pass boundary while the GPU is idle (split_idle), long passes are split too (split_in_pass), and the part's time limit reads the clock less often. - A render target that needs an image of its own takes the image of a released one with the same parameters, released two or more frames before (tex_reuse); memory requirements are kept per parameter set; a released entry another descriptor's entry replaced at its address is kept and taken back. - The memory check reads a page's write-watch bit and stamp under the lock that keeps them together. - The launcher reads the gamepad only while its own window is in front, and does not start a second copy of the game. - make_game.py says why a Python package did not load, not only that it is missing. The fix's per-pass GPU timing is left out: NX2NAVI_GPU_PASSES does the same in main. Co-Authored-By: Claude Opus 5.5 --- launcher/ui/app.py | 6 +- launcher/winsys.py | 29 ++++- nx2navi.ini.example | 29 +++-- runtime/cfg.c | 20 ++-- runtime/vk.h | 5 + runtime/vk_exec.c | 59 ++++++++-- runtime/vk_exec_draw.c | 8 +- runtime/vk_exec_xfer.c | 7 +- runtime/vk_memory.c | 215 +++++++++++++++++----------------- runtime/vk_texture.c | 257 +++++++++++++++++++++++++++++++++-------- tools/make_game.py | 5 +- 11 files changed, 442 insertions(+), 198 deletions(-) diff --git a/launcher/ui/app.py b/launcher/ui/app.py index 305d1578..9e6202c7 100644 --- a/launcher/ui/app.py +++ b/launcher/ui/app.py @@ -319,7 +319,7 @@ class App: def poll_gamepad(self): try: - for b in self.gamepad.poll(): + for b in self.gamepad.poll(winsys.in_front(self.root)): {'up': lambda: self.move(-1), 'down': lambda: self.move(1), 'left': lambda: self.side(-1), 'right': lambda: self.side(1), 'a': self.activate, 'b': self.back, 'y': lambda: self.check_now(force=True)}.get(b, lambda: None)() @@ -793,6 +793,10 @@ class App: return if self.game_proc and self.game_proc.poll() is None: return + if saves.game_running(gd): # a copy this launcher did not start (another launcher, a shortcut, a test run) + self.message = 'The game is already running.' + self.render() + return try: self.game_proc = winsys.start_game(exe) except OSError as e: diff --git a/launcher/winsys.py b/launcher/winsys.py index 460a0eb5..174e493c 100644 --- a/launcher/winsys.py +++ b/launcher/winsys.py @@ -273,9 +273,32 @@ BUTTONS = {0x0001: 'up', 0x0002: 'down', 0x0004: 'left', 0x0008: 'right', 0x1000 0x4000: 'x', 0x0010: 'start'} +def in_front(root): + """Whether the window the user is typing into belongs to this process. XInput reads the pad whichever window is in + front, so without this check the launcher, left open behind the game, took the presses meant for the game: A + activated its focused row (it started the game five times in five minutes and tried a rebuild), the stick moved + its focus and Y checked for updates. True where it cannot tell.""" + if os.name != 'nt': + return True + try: + user32 = ctypes.windll.user32 + user32.GetForegroundWindow.restype = ctypes.c_void_p + user32.GetWindowThreadProcessId.restype = ctypes.c_uint + user32.GetWindowThreadProcessId.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint)] + hwnd = user32.GetForegroundWindow() + if not hwnd: + return False # no window in front (the desktop is switching, a lock screen): nothing is meant for us + pid = ctypes.c_uint(0) + user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) + return pid.value == os.getpid() + except (AttributeError, OSError): + return True + + class Gamepad: """Polls the first XInput pad; poll() returns the buttons pressed since the last call (edges), the left stick - counted as the d-pad.""" + counted as the d-pad. poll(False) still reads the pad but returns nothing: a button held while another window was + in front does not count as pressed when this one comes back.""" def __init__(self): self.dll = None @@ -288,7 +311,7 @@ class Gamepad: except (AttributeError, OSError): continue - def poll(self): + def poll(self, active=True): if not self.dll: return set() st = _XInputState() @@ -309,7 +332,7 @@ class Gamepad: break except OSError: return set() - pressed = down - self.held + pressed = down - self.held if active else set() self.held = down return pressed diff --git a/nx2navi.ini.example b/nx2navi.ini.example index fed282fe..47592b26 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -133,6 +133,9 @@ # With astc = gpu: 1 stages a texture's ASTC blocks in the shared staging ranges instead of a buffer of their own # (0 = a buffer each; off until a play test measures it). # astc_arena = 0 +# A render target that needs an image of its own takes the image of a released one of the same size and format +# instead of making a new one (0 = always a new image). +# tex_reuse = 1 # Host zstd decoder in place of the game's (0 = the game's), and a check that decodes every call both ways and logs # the differences (1 = on; slow). # zstd_native = 1 @@ -164,8 +167,14 @@ # dyn_delta = 1 # Submit GPU work in parts every N draws between passes so the GPU starts earlier (0 to 1000; 0 = off), each part at # least split_us microseconds long (0 to 20000). +# Submit GPU work in parts every N draws so the GPU starts earlier (0 to 1000; 0 = off), each part at least split_us +# microseconds long (0 to 20000). split_in_pass = 1 also splits inside a long pass (it ends there and begins again in the +# next part); 0 splits between passes only. split_idle = 1 sends a submission's first part at the first pass boundary +# while the GPU has nothing else to run. # split_draws = 64 # split_us = 1000 +# split_in_pass = 1 +# split_idle = 1 # Vulkan. # Vulkan validation layer: checks every call and slows the game down a lot. Needs the layer installed (Vulkan SDK). @@ -511,11 +520,15 @@ # the next refresh and show their stand-in until then. Lower spreads streaming over more frames (smaller hitches, a # frame or two of pop-in); higher loads sooner. # pool_create_budget = 16 -# Written storage-buffer ranges up to this size in MiB (1 to 16) go back to game memory as a merge: copied within video -# memory and only the words that changed written over PCIe. Larger ones are copied back whole at the end of every -# submission. Raising it from 4 to 16 took a 5.5 MiB range off the plain copies (about 46 MiB a frame in the field). -# wb_merge_max = 16 -# 1: pages of a VRAM-mirrored pool that the GPU wrote and copied back to game memory are not copied up to the mirror -# again for that write, which already holds them (a write by the game still is). Outdoors this removed about 57 MiB of -# copies a frame. Default: 0. -# wb_no_reup = 0 +# Written storage-buffer ranges up to this size in MiB (1 to 256) go back to game memory as a merge, in 4 MiB pieces: +# copied within video memory and only the words that changed written over PCIe. Larger ones are copied back whole at +# the end of every submission. Raising it from 4 to 16 took a 5.5 MiB range off the plain copies (about 46 MiB a frame +# in the field), and 32 the 4-15 MiB ranges of Kokiri Forest. +# wb_merge_max = 32 +# 1: pages a VRAM mirror wrote back to game memory are not copied up into it again, as it already holds them (a write by +# the game still is). Outdoors this removed about 57 MiB of copies a frame. 0: they are copied up again. +# wb_no_reup = 1 +# 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every +# page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in +# Kokiri Forest. +# flush_upload = 0 diff --git a/runtime/cfg.c b/runtime/cfg.c index e165fd4e..873dd95f 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -42,7 +42,7 @@ static const char *const known[] = { "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", - "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FREEZE_IDLE", "FREE_CORES", + "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FREEZE_IDLE", "FREE_CORES", "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST", "HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", @@ -58,9 +58,9 @@ static const char *const known[] = { "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_CACHE_FLUSH", "SHADER_MEMO", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SVC_SLOW_MS", - "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", + "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NOOPT", "VK_NO_FLOAT64", + "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -109,12 +109,12 @@ static const struct { const char *key, *section; } sections[] = { {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, - {"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, + {"TEX_ALIAS", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, - {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, + {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, }; @@ -206,7 +206,7 @@ static const CfgMeta metas[] = { {"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART}, // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, - {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_ALIAS", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "90", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, {"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART}, {"ASTC_SCRATCH_KEEP", "0", "0|1", RESTART}, {"ASTC_ARENA", "0", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena // render fixes @@ -215,7 +215,7 @@ static const CfgMeta metas[] = { {"BIND_REUSE", "1", "0|1", RESTART}, {"DYN_DELTA", "1", "0|1", RESTART}, {"DESC_TEMPLATE", "1", "0|1", RESTART}, {"PUSH_SKIP", "1", "0|1", RESTART}, {"SPLIT_DRAWS", "64", "0..1000", RESTART}, - {"SPLIT_US", "1000", "0..20000", RESTART}, + {"SPLIT_US", "1000", "0..20000", RESTART}, {"SPLIT_IDLE", "1", "0|1", RESTART}, {"SPLIT_IN_PASS", "1", "0|1", RESTART}, // Vulkan {"VK_VALIDATION", "0", "0|1", RESTART}, {"VK_DYN_VERTEX", "", "0|1", RESTART}, {"VK_DYN_A2C", "", "0|1", RESTART}, {"VK_SUBGROUP_PIN", "1", "0|1", RESTART}, @@ -223,8 +223,8 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "0", "0|1", RESTART}, - {"WB_MERGE_MAX", "16", "1..16", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, + {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/vk.h b/runtime/vk.h index 159d0ca6..bb7d0be6 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -315,6 +315,11 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb); void vk_mem_shadow_sync(void); void vk_mem_gpu_pending(uint64_t addr, uint64_t size); void vk_gpu_written(uint64_t addr, uint64_t size); +// The same for a GPU write into a VRAM mirror, which went back to guest memory with the submission: the texture checks +// see it, the mirror's next sync does not copy it back up. vk_mem_mirrored at record time, right after the write's +// buffer was picked (VK_MEM_USE / VK_MEM_PICK): 1 when [addr, addr+size) lies in one pool whose mirror is on. +void vk_gpu_written_back(uint64_t addr, uint64_t size); +int vk_mem_mirrored(uint64_t addr, uint64_t size); int vk_mem_flush_add_probe(void); // --probe-vkexec: the flush range list; 0 or the number of the failed check uint64_t vk_mem_mirror_uploaded(void); // --probe-vkexec: bytes copied into mirrors since the last summary line int vk_mem_mirror_probe(uint64_t addr); // --probe-vkexec: the pool at addr gets its mirror as on a first buffer use; diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 67ed665b..93ef89c2 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -9,6 +9,7 @@ // until they change and skips clears while rendering is disabled. A pass has GENERAL-layout attachments, LOAD/STORE and // a render area from the smallest attachment, with a full barrier before it. // Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md). +#include #include #include #include @@ -872,9 +873,25 @@ static void set_rt(VkExec *ex, const p_set_rt *r) { // A part also spans at least NX2NAVI_SPLIT_US (1000) of replay time: a split costs about 40 us of the render thread // (vkQueueSubmit2, a command buffer, the mirror prologue), and a fight frame of 2,000 draws made 15 parts on the 7900X; // a millisecond of recording is a fraction of a millisecond of GPU work, so the GPU still starts early. -static int split_n = -1, split_draws, part_start; +// Neither limit holds the first part of a submission back while the GPU has run everything submitted (NX2NAVI_SPLIT_IDLE, +// on): it goes at the first pass boundary after a draw or dispatch. Each frame the game's thread waits for the GPU to run +// the frame's first submission (nvn_api_core.c nvnQueueSubmitCommands, NX2NAVI_SUBMIT_FINISH), so the GPU is idle +// when the next one replays, and waited for the first 64 draws and 1 ms of it in Kokiri Forest (2026-11-09). +// Both limits also split inside a pass (NX2NAVI_SPLIT_IN_PASS, on): the pass ends there and the next draw begins it +// again in the new part, as after a barrier (attachments load and store; occlusion queries continue in a new segment). +// Kokiri Forest's two scene passes of ~500 draws take 2-4 ms each to replay and most of the frame's GPU time; split only +// between passes, the GPU waited for each whole pass, idle up to 3.6 ms a frame (2026-11-09). +// After the time limit held a part back, the clock is read again only 8 draws later (split_at): read after every +// command, it was 4% of the replay in Kokiri Forest. +static int split_n = -1, split_idle, split_in_pass, split_draws, split_at, part_start; static uint64_t split_ns, part_t0; -static uint64_t n_parts, n_subs; +static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits; + +static int split_wanted(void) { + if (split_draws >= split_at) return 1; + // the first part only: later parts of the submission run on the GPU, while done and issued stay as they were + return split_idle && split_draws && !part_start && vk_sync_fence_done() >= vk_sync_issued(); +} static void maybe_split(VkExec *ex) { if (split_n < 0) { @@ -882,13 +899,29 @@ static void maybe_split(VkExec *ex) { if (split_n < 0) split_n = 0; int us = cfg_int("NX2NAVI_SPLIT_US"); split_ns = us > 0 ? (uint64_t)us * 1000 : 0; - rt_log("vk: replay command buffer split every %d draws between passes, parts of at least %d us%s", split_n, - us > 0 ? us : 0, split_n ? "" : " (off)"); + split_idle = split_n && cfg_bool("NX2NAVI_SPLIT_IDLE"); + split_in_pass = split_n && cfg_bool("NX2NAVI_SPLIT_IN_PASS"); + rt_log("vk: replay command buffer split every %d draws %s, parts of at least %d us%s%s", split_n, + split_in_pass ? "(inside passes too)" : "between passes", us > 0 ? us : 0, split_n ? "" : " (off)", + split_idle ? "; a first part at once while the GPU is idle" : ""); + split_at = split_n ? split_n : INT_MAX; + } + if (!split_n) return; + // the first part while the GPU is idle: at a pass boundary only, where a split costs no pass restart + int idle = !ex->in_pass && split_idle && split_draws && !part_start && vk_sync_fence_done() >= vk_sync_issued(); + if (!idle) { + if (ex->in_pass && !split_in_pass) return; + if (split_draws < split_n) return; + if (split_ns && kern_now_ns() - part_t0 < split_ns) { // split_draws stays: a later draw asks again + split_at = split_draws + 8; + return; + } } - if (!split_n || ex->in_pass || split_draws < split_n) return; - if (split_ns && kern_now_ns() - part_t0 < split_ns) return; // split_draws stays: the next draw asks again split_draws = 0; + split_at = split_n; + if (idle && ex->recorded != part_start) n_idle_parts++; if (ex->recorded == part_start) return; + if (ex->in_pass) vk_exec_pass_end(ex), n_pass_splits++; // the next draw begins the pass again, in the new part ex->cb = vk_sync_split(); part_start = ex->recorded; part_t0 = kern_now_ns(); @@ -1182,7 +1215,7 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { default: rt_fatal("vk: op %s has no replay handler", nvn_op_name(h->op)); } if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); - if (split_draws >= split_n) maybe_split(ex); + if (split_draws && split_wanted()) maybe_split(ex); h = nvn_op_next(h); } } @@ -1243,6 +1276,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_frame.submits++; vk_exec_submit_seq++; split_draws = part_start = 0; + split_at = split_n < 0 ? 0 : split_n ? split_n : INT_MAX; // 0 until maybe_split reads the settings part_t0 = kern_now_ns(); x_here = 1; int evo = evo_enabled() && gfx_submit_queue; @@ -1296,12 +1330,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t ex->cb = VK_NULL_HANDLE; if (n) n_subs++; if ((rt_trace & RT_TRACE_PERF) && n_subs >= 240) { - rt_log("perf: replay command buffer parts per submission %.2f; event order: held %llu, resumed %llu (%llu " - "without their signal), switches %llu, held now %d", (double)(n_subs + n_parts) / (double)n_subs, - (unsigned long long)evo_held, (unsigned long long)evo_resumed, (unsigned long long)evo_forced, - (unsigned long long)evo_switches, evo_pending); + rt_log("perf: replay command buffer parts per submission %.2f (%.2f first parts sent while the GPU was idle, %.2f " + "splits inside a pass); event order: held %llu, resumed %llu (%llu without their signal), switches %llu, " + "held now %d", (double)(n_subs + n_parts) / (double)n_subs, (double)n_idle_parts / (double)n_subs, + (double)n_pass_splits / (double)n_subs, (unsigned long long)evo_held, (unsigned long long)evo_resumed, + (unsigned long long)evo_forced, (unsigned long long)evo_switches, evo_pending); evo_held = evo_resumed = evo_forced = evo_switches = 0; - n_subs = n_parts = 0; + n_subs = n_parts = n_idle_parts = n_pass_splits = 0; } return ex->recorded - part_start; // what the last part holds } diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index b975d1ee..4c687600 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -115,7 +115,7 @@ typedef struct { uint32_t aspect, level0, levels, layer0, layers, swz; } ViewKey; _Static_assert(sizeof(ViewKey) == 40, "ViewKey is hashed as bytes: no padding"); -typedef struct { uint64_t addr, size; } Range; +typedef struct { uint64_t addr, size; int back; } Range; // back: written into a VRAM mirror (vk_mem_mirrored) typedef struct { Chunk *chunks; int nchunks, cchunks; @@ -222,7 +222,9 @@ static uint64_t bind_gen = 1; // bumped by every bind op that writes static void cb_done(void *arg) { // completion thread CbRes *r = arg; VkCtx *c = vk_ctx(); - for (int i = 0; i < r->nwritten; i++) vk_gpu_written(r->written[i].addr, r->written[i].size); + for (int i = 0; i < r->nwritten; i++) + if (r->written[i].back) vk_gpu_written_back(r->written[i].addr, r->written[i].size); + else vk_gpu_written(r->written[i].addr, r->written[i].size); for (int i = 0; i < r->npools; i++) { vkResetDescriptorPool(c->device, r->pools[i], 0); plat_mutex_lock(&spare_lock); @@ -319,7 +321,7 @@ static void note_written(VkExec *ex, uint64_t addr, uint64_t size) { for (int i = 0; i < r->nwritten; i++) if (r->written[i].addr == addr && r->written[i].size == size) return; if (r->nwritten == r->cwritten) r->written = grow(r->written, &r->cwritten, sizeof *r->written); - r->written[r->nwritten++] = (Range){addr, size}; + r->written[r->nwritten++] = (Range){addr, size, vk_mem_mirrored(addr, size)}; vk_mem_gpu_pending(addr, size); } diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 440123e4..a9f45483 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -60,11 +60,12 @@ static uint32_t max1(uint32_t v) { return v ? v : 1; } // ---- completion callbacks (completion thread) -typedef struct { uint64_t addr, size; } Range; +typedef struct { uint64_t addr, size; int back; } Range; // back: written into a VRAM mirror (vk_mem_mirrored) static void guest_written_cb(void *arg) { Range *r = arg; - vk_gpu_written(r->addr, r->size); + if (r->back) vk_gpu_written_back(r->addr, r->size); + else vk_gpu_written(r->addr, r->size); free(r); } @@ -74,7 +75,7 @@ static void note_gpu_write(VkExec *ex, uint64_t addr, uint64_t size) { if (nvn_gpu_only(addr)) return; Range *r = malloc(sizeof *r); if (!r) rt_fatal("vk: out of memory"); - *r = (Range){addr, size}; + *r = (Range){addr, size, vk_mem_mirrored(addr, size)}; vk_mem_gpu_pending(addr, size); vk_completion_add(ex->fence, guest_written_cb, r); } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 9fae8ede..33c57ae2 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -710,6 +710,12 @@ static uint64_t gen_pages[1024]; // sync skips regions with nothing newer than its pool's mirror. #define REG_SHIFT 21 static uint32_t reg_max[RT_ADDRESS_LIMIT >> REG_SHIFT]; +// The same two tables without the stamps of a mirror's own write-back (vk_gpu_written_back): what the CPU and the host +// paths wrote. The mirror sync reads these, so the bytes a mirror wrote back into guest memory are not copied up into it +// again at its next sync: the storage buffers the game's compute shaders write went down and came back up every frame, +// 20 MiB a frame over PCIe in Kokiri Forest (2026-11-09). +static uint32_t *cgen_chunk[GEN_CHUNKS]; +static uint32_t creg_max[RT_ADDRESS_LIMIT >> REG_SHIFT]; // Every write-watch query of the generation table (gen_lock held exclusively): calls, bytes and time since the last // mirror summary, all consumers together; the queries that found written pages (they also flush TLBs) apart. @@ -741,26 +747,34 @@ static uint32_t *fcand; // gen_lock: regions to try at the static int nfcand, cfcand; static uint64_t n_frz_refused; // gen_lock: candidates mem_freeze refused -static uint32_t *gen_slot(uint64_t page_addr, int create) { +static uint32_t *slot_in(uint32_t **tab, uint64_t page_addr, int create) { uint64_t ci = page_addr >> GEN_CHUNK_SHIFT; if (ci >= GEN_CHUNKS) return NULL; - if (!gen_chunk[ci]) { + if (!tab[ci]) { if (!create) return NULL; - gen_chunk[ci] = calloc(GEN_PER_CHUNK, sizeof(uint32_t)); - if (!gen_chunk[ci]) rt_fatal("vk: out of memory for page generations"); + tab[ci] = calloc(GEN_PER_CHUNK, sizeof(uint32_t)); + if (!tab[ci]) rt_fatal("vk: out of memory for page generations"); } - return &gen_chunk[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)]; + return &tab[ci][(page_addr >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)]; } -// gen_lock held exclusively; g is the newest generation handed out. -static void stamp_page(uint64_t a, uint32_t g) { +static uint32_t *gen_slot(uint64_t page_addr, int create) { return slot_in(gen_chunk, page_addr, create); } + +// gen_lock held exclusively; g is the newest generation handed out. back: a mirror's write-back, which leaves the +// CPU-side tables alone. +static void stamp_page_as(uint64_t a, uint32_t g, int back) { uint32_t *s = gen_slot(a, 1); if (!s) return; *s = g; reg_max[a >> REG_SHIFT] = g; reg_last[a >> REG_SHIFT] = ww_ep; + if (back) return; + uint32_t *c = slot_in(cgen_chunk, a, 1); + if (c) *c = g, creg_max[a >> REG_SHIFT] = g; } +static void stamp_page(uint64_t a, uint32_t g) { stamp_page_as(a, g, 0); } + static void stamp_range(uint64_t lo, uint64_t hi, uint32_t g) { for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page(a, g); } @@ -1084,73 +1098,25 @@ void vk_mem_gpu_pending(uint64_t addr, uint64_t size) { plat_mutex_unlock(&shadow_lock); } -// NX2NAVI_WB_NO_REUP=1: a mirrored pool's pages the GPU wrote are not copied up to the mirror again for that write. -// vk_gpu_written stamps the pages the GPU wrote, at the fence, so the texture cache re-reads them. The same stamps made -// mirror_sync copy those pages back up into the mirror the write-back had read them from: outdoors about 57 of the 60 MiB -// copied up a frame were pages unchanged since the GPU wrote them back, and the copies took about 5 ms of GPU time a -// frame (RTX 4080 SUPER, 2026-11-08). Each stamp of such a write is noted with the page's generation before it; -// mirror_sync skips a page whose newest stamp is that one while the stamp before it was already in the mirror. A CPU -// write stamps the page anew (a later generation), so it is still copied. -typedef struct { uint64_t page; uint32_t g, old; } GwEnt; -#define GW_BITS 20 -static GwEnt *gw_tab; // gen_lock (written exclusively; mirror_sync reads it on the replay thread) -static volatile int64_t gw_skipped; // pages mirror_sync left out since the last summary -static int gw_on = -1; +// NX2NAVI_WB_NO_REUP (default 1; 0 = off): the pages a VRAM mirror wrote back into guest memory are not copied up into +// it again at its next sync. vk_gpu_written stamps the pages the GPU wrote, at the fence, so the texture cache re-reads +// them; the same stamps made mirror_sync copy them back up into the mirror the write-back had read them from: the +// storage buffers the game's compute shaders write went down and came back up every frame, about 57 of the 60 MiB copied +// up a frame outdoors (RTX 4080 SUPER, 2026-11-08) and 20 MiB a frame in Kokiri Forest (RTX 4090, PCIe 3.0, 2026-11-09). +// A write into a mirror (vk_mem_mirrored, noted at record time) stamps only the texture tables (vk_gpu_written_back); +// mirror_sync reads the CPU-side ones (stamp_page_as), which a CPU write or a host path still stamps. +static int wb_reup_off = -1; static int wb_no_reup(void) { - if (gw_on < 0) { - int on = rt_knob("WB_NO_REUP", "0") == 1; - if (on && !(gw_tab = calloc((size_t)1 << GW_BITS, sizeof *gw_tab))) on = 0; - rt_log("mem: pages the GPU wrote back %s (NX2NAVI_WB_NO_REUP)", - on ? "are not copied up to their mirror again" : "are copied up to their mirror again"); - __atomic_store_n(&gw_on, on, __ATOMIC_RELEASE); + if (wb_reup_off < 0) { + int on = rt_knob("WB_NO_REUP", "1") == 1; + rt_log("mem: pages a VRAM mirror wrote back %s (NX2NAVI_WB_NO_REUP)", + on ? "are not copied up into it again" : "are copied up into it again"); + __atomic_store_n(&wb_reup_off, on, __ATOMIC_RELEASE); } - return gw_on; -} -static GwEnt *gw_find(uint64_t page) { // gen_lock held: the entry of page or NULL - uint64_t m = ((uint64_t)1 << GW_BITS) - 1, i = (page >> GEN_PAGE_SHIFT) * 0x9e3779b97f4a7c15ull >> (64 - GW_BITS); - for (int n = 0; n < 32; n++, i = (i + 1) & m) { - if (gw_tab[i].page == page) return &gw_tab[i]; - if (!gw_tab[i].page) return NULL; - } - return NULL; -} -static void gw_put(uint64_t page, uint32_t g, uint32_t old) { // gen_lock held exclusively - uint64_t m = ((uint64_t)1 << GW_BITS) - 1, i0 = (page >> GEN_PAGE_SHIFT) * 0x9e3779b97f4a7c15ull >> (64 - GW_BITS); - uint64_t i = i0; - for (int n = 0; n < 32; n++, i = (i + 1) & m) - if (!gw_tab[i].page || gw_tab[i].page == page) { gw_tab[i] = (GwEnt){page, g, old}; return; } - gw_tab[i0] = (GwEnt){page, g, old}; // a full run: the evicted page is copied up as before -} -// mirror_sync, gen_lock held: the page's newest stamp g is a GPU write the mirror already holds. -static int gw_skip(uint64_t page, uint32_t g, uint32_t since) { - if (gw_on != 1) return 0; - const GwEnt *e = gw_find(page); - return e && e->g == g && e->old <= since; -} -// vk_gpu_written: stamps [addr, addr+size) as vk_guest_written does, noting the stamps when the range lies in a mirrored -// pool's imported range. 0: not such a range (the caller stamps it plainly). -static int gw_note(uint64_t addr, uint64_t size) { - if (!wb_no_reup() || nvn_gpu_only(addr)) return 0; - plat_mutex_lock_shared(&pool_lock); - const pidx_ent *e = pidx_find(&pool_index, addr, size); - const PoolRec *r = e ? e->buf : NULL; - int ok = r && r->mstate == MS_ON && addr >= r->import_lo && addr + size <= r->import_hi; - plat_mutex_unlock_shared(&pool_lock); - if (!ok) return 0; - uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; - plat_mutex_lock(&gen_lock); - uint32_t g = ++gen_now; - for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) { - uint32_t *sl = gen_slot(a, 1); - if (!sl) continue; - gw_put(a, g, *sl); - stamp_page(a, g); - } - plat_mutex_unlock(&gen_lock); - return 1; + return wb_reup_off; } -void vk_gpu_written(uint64_t addr, uint64_t size) { +static void gpu_written(uint64_t addr, uint64_t size, int back) { if (shadow_on && size) { plat_mutex_lock_shared(&pool_lock); const pidx_ent *e = pidx_find(&pool_index, addr, size); @@ -1166,8 +1132,23 @@ void vk_gpu_written(uint64_t addr, uint64_t size) { plat_mutex_unlock(&shadow_lock); plat_mutex_unlock_shared(&pool_lock); } - if (size && gw_note(addr, size)) return; - vk_guest_written(addr, size); + if (!size || nvn_gpu_only(addr)) return; + uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; + back = back && wb_no_reup(); // read before gen_lock: its first call logs + plat_mutex_lock(&gen_lock); + uint32_t g = ++gen_now; + for (uint64_t a = lo; a < hi; a += 1ull << GEN_PAGE_SHIFT) stamp_page_as(a, g, back); + plat_mutex_unlock(&gen_lock); +} + +void vk_gpu_written(uint64_t addr, uint64_t size) { gpu_written(addr, size, 0); } +void vk_gpu_written_back(uint64_t addr, uint64_t size) { gpu_written(addr, size, 1); } + +int vk_mem_mirrored(uint64_t addr, uint64_t size) { + VkMemSpan sp; + if (!vk_mirror_on || !size || !vk_mem_resolve(addr, size, &sp) || !sp.pool) return 0; + const PoolRec *r = sp.pool; + return r->mstate == MS_ON && addr >= r->import_lo && addr + size <= r->import_hi; } static void shadow_log(uint64_t frame) { @@ -1314,16 +1295,19 @@ static void dirty_frame(uint64_t frame) { // the CPU reads and writes; textures and every host path that reads or writes pool bytes keep using it. // - CPU -> mirror: the first access of a submission to a mirror syncs it (mirror_sync): the 2 MiB write-watch // snapshots of its regions for this epoch (snap_regions, shared with the texture checks: the OS bits keep one -// consumer, the shared page generations) and every page stamped after the mirror's mgen goes into a copy; so do the -// nvnMemoryPoolFlushMappedRange ranges, each on its own: one span from the lowest to the highest flush of a 252 MiB -// pool copied up to the whole pool for a few MiB flushed, 30-50 ms of GPU time over PCIe 3.0 x8 in crowded towns -// (NX2NAVI_FLUSH_SPAN=1 restores it). Writes between the ranges need no copy: the write watch sees them. The first use -// copies the whole range. The copies run in a prologue command +// consumer, the shared page generations) and every page the CPU-side tables (cgen_chunk) stamped after the mirror's +// mgen goes into a copy. Where a region of the pool is not write-watched, so do the nvnMemoryPoolFlushMappedRange +// ranges, each on its own: one span from the lowest to the highest flush of a 252 MiB pool copied up to the whole +// pool for a few MiB flushed, 30-50 ms of GPU time over PCIe 3.0 x8 in crowded towns (NX2NAVI_FLUSH_SPAN=1 restores +// it). Where the write watch covers the pool, a flushed page the CPU wrote is a written page already, and the game's +// flushes of whole pools it had not written cost 47-57 MiB of copies a frame in Kokiri Forest (2026-11-09); +// NX2NAVI_FLUSH_UPLOAD=1 copies them anyway. The first use copies the whole range. The copies run in a prologue command // buffer that vk_sync.c submits right before the part that accessed the mirror first, between full barriers. // vk_mem_mirror_begin snaps the regions of the pools the previous submission accessed in a few merged queries. // - GPU -> guest: what the replay writes into a mirror (storage buffers its shaders write, buffer copies, fills, // uniform updates, texture-to-buffer copies: vk_mem_gpu_wrote) goes back into the imported buffer at the end of the -// submission, exact byte ranges, before the fence signals (vk_mem_mirror_end). +// submission, exact byte ranges, before the fence signals (vk_mem_mirror_end). At the fence those pages get a new +// generation for the texture checks (vk_gpu_written_back) but not in the CPU-side tables: the mirror holds the bytes. // - NX2NAVI_MEMORY=verify: every submission also copies up to VERIFY_BYTES of mirror pages, round robin, into host memory; // at its fence the pages the CPU has not written since their sync are compared with guest memory. // A pool stays imported when its mirror would not fit the VRAM budget, its range is not write-watched, it overlaps @@ -1348,12 +1332,14 @@ static VkBufferCopy *mregions; // replay: vkCmdCopyBuffer regions static int cregions; static PlatMutex flush_lock = PLAT_MUTEX_INIT; static int flush_cap = FLUSH_RANGES; // ranges a pool keeps; 1 with NX2NAVI_FLUSH_SPAN=1 +static int flush_upload; // NX2NAVI_FLUSH_UPLOAD=1: flush ranges are copied where the write watch sees them too typedef struct { uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges; uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes; uint64_t snap_bytes, merge_bytes, merge_dispatches, snap_allocs; // write-back merge uint64_t page_bytes, flush_n, flush_bytes, flush_calls, flush_sum; // what mirror_sync queues, by source + uint64_t flush_skipped; // flush ranges not copied: the write watch covers them } MirrorAcc; static MirrorAcc mrep; // replay: this submission static MirrorAcc macc; // mirror_stat_lock: since the last summary @@ -1382,6 +1368,8 @@ static void mirror_mode(int discrete) { flush_cap = 1; rt_log("mem: a pool's flushes copy one span from the lowest to the highest (NX2NAVI_FLUSH_SPAN=1)"); } + flush_upload = rt_knob("FLUSH_UPLOAD", "0") == 1; + if (flush_upload) rt_log("mem: flushed ranges are copied into the mirrors as well as written pages (NX2NAVI_FLUSH_UPLOAD=1)"); const char *fi = cfg_get("NX2NAVI_FREEZE_IDLE"); freeze_idle = fi && *fi ? atoi(fi) : 60; if (freeze_idle < 0) freeze_idle = 0; @@ -1691,16 +1679,17 @@ static void mirror_sync(PoolRec *r) { mrep.sync_queries += snap_regions(r0, r1, mepoch); } uint32_t since = r->mgen; + int untracked = 0; for (uint64_t k = r0; k <= r1; k++) { - if (reg_max[k] <= since) continue; + untracked |= snap_untracked[k]; + if (creg_max[k] <= since) continue; uint64_t a = k << SNAP_SHIFT, b = a + (1ull << SNAP_SHIFT); if (a < lo) a = lo; if (b > hi) b = hi; - const uint32_t *ch = gen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk + const uint32_t *ch = cgen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk if (!ch) continue; for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since) { - if (gw_skip(x, ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)], since)) { plat_inc64(&gw_skipped); continue; } range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT), mrep.page_bytes += 1ull << GEN_PAGE_SHIFT; r->st_pages += 1ull << GEN_PAGE_SHIFT; } @@ -1720,6 +1709,10 @@ static void mirror_sync(PoolRec *r) { for (int i = 0; i < n; i++) { uint64_t a = fl[i].lo > lo ? fl[i].lo : lo, b = fl[i].hi < hi ? fl[i].hi : hi; if (a >= b) continue; + if (!untracked && !flush_upload) { // the write watch saw every write in them: the pages above + mrep.flush_skipped += b - a; + continue; + } range_add(&pend, &npend, &cpend, r, a - lo, b - a); mrep.flush_n++, mrep.flush_bytes += b - a, r->st_flush += b - a; if ((rt_trace & RT_TRACE_PERF) && b - a >= 16u << 20) @@ -1785,20 +1778,23 @@ int vk_mem_mirror_probe(uint64_t addr) { #define SNAP_CHUNK (1u << 20) #define MERGE_SPAN (64ull << 20) // the bytes of one pool a merge dispatch binds at most #define MERGE_WORDS (65535u * 64u) // the words one merge dispatch covers at most -// Larger written ranges stay plain copies (NX2NAVI_WB_MERGE_MAX, MiB, default 16, at most what one dispatch covers). -// A plain copy writes the whole range over PCIe at the end of each submission even when the shader changed a few words; -// a merge copies it within video memory and writes only the words that changed. With a 4 MiB limit the field ran about -// 46 MiB of plain write-back a frame in ranges of about 5.5 MiB, just above it; at 16 the same outdoor route went from a -// median 47.5 to 67.3 fps and from 11.3 to 7.2 ms of GPU time a frame (RTX 4080 SUPER, 120 fps limit, 2026-11-08). -#define MERGE_MAX_CAP ((uint64_t)MERGE_WORDS * 4u) +// Larger written ranges stay plain copies (NX2NAVI_WB_MERGE_MAX, MiB, default 32). A plain copy writes the whole range +// over PCIe at the end of each submission even when the shader changed a few words; a merge copies it within video +// memory and writes only the words that changed. With a 4 MiB limit the field ran about 46 MiB of plain write-back a +// frame in ranges of about 5.5 MiB, just above it; at 16 the same outdoor route went from a median 47.5 to 67.3 fps and +// from 11.3 to 7.2 ms of GPU time a frame (RTX 4080 SUPER, 120 fps limit, 2026-11-08). Kokiri Forest writes seven +// storage buffer ranges of 4-15 MiB a frame; at 32, merged in MERGE_PIECE parts so that no dispatch covers more than +// MERGE_WORDS, the plain copies of about 4 ms of GPU time a frame on PCIe 3.0 went (RTX 4090, 2026-11-09). +#define MERGE_PIECE (4ull << 20) +#define MERGE_MAX_LIMIT (256ull << 20) // the snapshot pool's largest size (snap_pool_cap) static uint64_t merge_max(void) { static uint64_t m; if (!m) { const char *e = cfg_get("NX2NAVI_WB_MERGE_MAX"); - uint64_t mib = e && *e ? (uint64_t)strtoull(e, NULL, 10) : 16; + uint64_t mib = e && *e ? (uint64_t)strtoull(e, NULL, 10) : 32; m = mib << 20; if (!m) m = 1; - if (m > MERGE_MAX_CAP) m = MERGE_MAX_CAP; + if (m > MERGE_MAX_LIMIT) m = MERGE_MAX_LIMIT; rt_log("mem: write-back merge covers ranges up to %.1f MiB (NX2NAVI_WB_MERGE_MAX)", m / 1048576.0); } return m; @@ -1812,9 +1808,11 @@ static SnapChunk *chunks; // replay: this submission's snapshot chunks static int nchunks, cchunks; // Chunks a finished submission used (snap_free) wait here for chunk_new, so a frame with merges creates no buffer once // the pool holds the frames in flight: a VMA allocation that needs a new memory block costs 8-11 ms on the replay -// thread. Chunk sizes are powers of two so that one fits the next request; what the pool cannot hold is destroyed. +// thread. Chunk sizes are powers of two so that one fits the next request; what the pool cannot hold is destroyed. The +// pool holds about three frames of Kokiri Forest's snapshots (65 MiB a frame); at 16 MiB it held less than one frame's +// and two buffers a frame were created and destroyed again (2026-11-09). #define SNAP_LIST_MIN (64u << 10) -#define SNAP_POOL_N 64 +#define SNAP_POOL_N 128 // The pool holds up to 1/64 of the VRAM budget, between 64 and 256 MiB: room for merges of multi-MiB ranges across the // submissions in flight. Its chunks prefer device memory and are not evictable, so a small card keeps less of them. static uint64_t snap_pool_cap(void) { @@ -2035,12 +2033,14 @@ static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; r->vd_wrote_sub = msub, r->vd_wrote_n++; // A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies. - if (shader && !(sp->offset & 3) && !(size & 3) && size <= merge_max() && merging() && snap_add(r, sp->offset, size)) { - r->st_merge += size; - return; - } - r->st_wb += size; - range_add(&wbl, &nwbl, &cwbl, r, sp->offset, size); + uint64_t o = 0; + if (shader && !(sp->offset & 3) && !(size & 3) && size <= merge_max() && merging()) + for (uint64_t n; o < size; o += n) + if (!snap_add(r, sp->offset + o, n = size - o < MERGE_PIECE ? size - o : MERGE_PIECE)) break; + r->st_merge += o; + if (o == size) return; + r->st_wb += size - o; + range_add(&wbl, &nwbl, &cwbl, r, sp->offset + o, size - o); // what found no snapshot room } void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 0); } void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 1); } @@ -2219,7 +2219,12 @@ static void verify_cb(void *arg) { // completion thread const uint8_t *m = j->map + s->dst + o, *g = (const uint8_t *)(uintptr_t)a; pages++; if (!memcmp(m, g, 1ull << GEN_PAGE_SHIFT)) continue; - if (page_dirty(a, s->mgen)) { skipped++; continue; } + // gen_lock: a write-watch query clears the page's bit and stamps it in one hold (vk_mem_changed, mirror_sync), + // so the check sees one or the other; without it a page between the two counted as a mismatch. + plat_mutex_lock_shared(&gen_lock); + int dirty = page_dirty(a, s->mgen); + plat_mutex_unlock_shared(&gen_lock); + if (dirty) { skipped++; continue; } bad++; if (plat_inc32(&logged) <= VERIFY_LOG) { int k = 0, e = (1 << GEN_PAGE_SHIFT) - 1; @@ -2369,13 +2374,6 @@ static void mirror_pool_log(uint64_t frame) { // One summary line: with NX2NAVI_TRACE=perf every 60 frames, else at frames 600 and 1800. static void mirror_log(uint64_t frame) { - if (gw_on == 1) { - static uint64_t gw_last; - double f = frame > gw_last ? (double)(frame - gw_last) : 1.0; - gw_last = frame; - rt_log("mem: write-back frame %llu: %.2f MiB a frame of GPU-written pages not copied up again (NX2NAVI_WB_NO_REUP)", - (unsigned long long)frame, plat_xchg64(&gw_skipped, 0) * (double)(1ull << GEN_PAGE_SHIFT) / 1048576.0 / f); - } static uint64_t last, ww_last; plat_mutex_lock(&gen_lock); uint64_t wc = ww_calls, wb = ww_bytes, wt = ww_ticks, dc = ww_dirty_calls, dt = ww_dirty_ticks, wp = ww_pages; @@ -2422,8 +2420,9 @@ static void mirror_log(uint64_t frame) { qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, (long long)m_evicted_n, m_evicted_bytes / M, (long long)m_retries, ver); rt_log("mem: mirror frame %llu: upload sources per frame, overlaps counted twice: written pages %.3f MiB; flush ranges " - "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB", (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, - a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M); + "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; flushed and not copied (write-watched) %.3f MiB", + (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, + a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M, a.flush_skipped / f / M); } // After vram_log: every heap's usage as the driver reports it (VK_EXT_memory_budget) beside VMA's share of it, VMA by @@ -3040,6 +3039,6 @@ void vk_mem_shutdown(void) { nphys = 0; plat_mutex_unlock(&pool_lock); plat_mutex_lock(&gen_lock); - for (uint64_t i = 0; i < GEN_CHUNKS; i++) free(gen_chunk[i]), gen_chunk[i] = NULL; + for (uint64_t i = 0; i < GEN_CHUNKS; i++) free(gen_chunk[i]), gen_chunk[i] = NULL, free(cgen_chunk[i]), cgen_chunk[i] = NULL; plat_mutex_unlock(&gen_lock); } diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 4d4c6273..6f67578c 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -34,6 +34,10 @@ typedef struct { uint8_t has_layout, layout[8], pad[7]; } TexKey; +// What makes two images interchangeable: the parameters create_tex makes them with (img_key_of). Image reuse and the +// memory requirement cache match on it. +typedef struct { uint32_t flags, type, format, w, h, d, levels, layers, samples, usage; } ImgKey; + typedef struct Tex { VkTexImage pub; // first: vk_texture_get hands out &pub TexKey key; @@ -66,6 +70,14 @@ typedef struct Tex { uint64_t chash; // content_hash of the storage at its last upload (small textures, CHASH_MAX) uint8_t chash_ok; // chash holds it struct Tex *rel_prev, *rel_next; // while released: the list of released entries, oldest first + ImgKey ik; // the image's creation parameters + uint8_t reusable; // its image may go to a new texture once it is released (own memory, GPU-only content) + uint8_t reused; // its image may still be in use from before (a released entry's, or its own restarted by + // start_at): the first barrier waits for all earlier work + uint8_t ru_on; // in the reuse list of ik + uint8_t superseded; // released, and another descriptor's entry was made at its address since (start_at) + uint32_t rel_epoch; // submit_epoch when it was released + struct Tex *ru_prev, *ru_next; // the reuse list of ik, newest release first } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the @@ -301,21 +313,69 @@ static int format_compressed(VkFormat f) { // vkCmdClearColorImage takes no bl (f >= VK_FORMAT_ASTC_4x4_SFLOAT_BLOCK && f <= VK_FORMAT_ASTC_12x12_SFLOAT_BLOCK); } +// Image reuse (NX2NAVI_TEX_REUSE, default on): a GPU-only texture that needs an image of its own takes the image of a +// released entry made with the same parameters (ImgKey) instead of a new one, which then starts over as a new image +// would (init_image: undefined content, cleared). Ocarina places its per-frame render targets with a linear allocator +// behind small targets whose size follows an effect on screen (77x77 to 97x97), so every few frames the targets after +// them move to new addresses, new descriptors whose images were made again: vkCreateImage, the memory requirements and +// the binding of a dedicated allocation cost 8% of the replay in Kokiri Forest. The released entries of each parameter +// set are kept in a list, newest release first; create_tex takes the one nearest in address of those released at least +// two frames earlier, never during the current submission's replay, whose later parts may still use it. +static nvn_hmap reuse = {.klen = sizeof(ImgKey)}; // ImgKey -> the newest released reusable entry +static int tex_reuse = -1; +static uint64_t n_reused, n_reuse_new; // since the last perf line: images taken from a released entry, made anew +static uint64_t n_restarts; // superseded entries taken back (start_at) + +static int tex_reuse_on(void) { + if (tex_reuse < 0) { + tex_reuse = cfg_bool("NX2NAVI_TEX_REUSE"); + rt_log("vk: texture image reuse %s (NX2NAVI_TEX_REUSE)", tex_reuse ? "on" : "off"); + } + return tex_reuse; +} + +static void destroy_tex(Tex *e); + +// Whether an image with these requirements can lie in the pool memory at e's address (*sp: that memory). +static int pool_fits(VkCtx *c, const Tex *e, const VkMemoryRequirements *req, uint64_t guest_size, VkMemSpan *sp) { + if ((e->f.depth || e->f.stencil) && req->size > guest_size) return 0; + if (!req->size) return 0; // an image the device could not lay out (image_size) + if (!vk_mem_resolve(e->pub.addr, req->size, sp) || !sp->alloc || sp->alloc_offset % req->alignment) return 0; + VmaAllocationInfo info; + vmaGetAllocationInfo(c->vma, sp->alloc, &info); + return req->memoryTypeBits >> info.memoryType & 1; +} + +// The memory requirements of each parameter set (ImgKey) an image was made with; images made with the same parameters +// have the same requirements. With them create_tex makes no image that pool_fits would turn down: Ocarina's moving +// render targets mostly miss the device's alignment, and making the image only to read its requirements +// (vkGetImageMemoryRequirements does most of the driver's layout work) cost 5% of the replay in Kokiri Forest. +static nvn_hmap img_reqs = {.klen = sizeof(ImgKey)}; // ImgKey -> VkMemoryRequirements (malloc'd) +static uint64_t n_bind_skips; // since the last perf line + static int bind_pool_memory(VkCtx *c, Tex *e, VkImage img, uint64_t guest_size) { VkMemoryRequirements req; vkGetImageMemoryRequirements(c->device, img, &req); - if ((e->f.depth || e->f.stencil) && req.size > guest_size) return 0; + if (!nvn_hm_get(&img_reqs, &e->ik)) { + VkMemoryRequirements *q = malloc(sizeof *q); + if (!q) rt_fatal("vk: out of memory"); + *q = req; + nvn_hm_put(&img_reqs, &e->ik, q); + } VkMemSpan sp; - if (!req.size) return 0; // an image the device could not lay out (image_size) - if (!vk_mem_resolve(e->pub.addr, req.size, &sp) || !sp.alloc || sp.alloc_offset % req.alignment) return 0; - VmaAllocationInfo info; - vmaGetAllocationInfo(c->vma, sp.alloc, &info); - if (!(req.memoryTypeBits >> info.memoryType & 1)) return 0; + if (!pool_fits(c, e, &req, guest_size, &sp)) return 0; if (vmaBindImageMemory2(c->vma, sp.alloc, sp.alloc_offset, img, NULL) != VK_SUCCESS) return 0; e->bound_to = sp.alloc; return 1; } +static void img_key_of(const VkImageCreateInfo *ci, ImgKey *k) { + memset(k, 0, sizeof *k); + k->flags = ci->flags, k->type = ci->imageType, k->format = ci->format, k->w = ci->extent.width; + k->h = ci->extent.height, k->d = ci->extent.depth, k->levels = ci->mipLevels, k->layers = ci->arrayLayers; + k->samples = ci->samples, k->usage = ci->usage; +} + // agl's BC4 encoder draws a BC texture's blocks through an RGBA16UI view, one texel per 8-byte block. // Vulkan does it with an uncompressed // view of a BLOCK_TEXEL_VIEW_COMPATIBLE image; EXTENDED_USAGE lets the BC image carry the COLOR_ATTACHMENT usage that @@ -346,8 +406,12 @@ static VkDeviceSize image_size(const VkImageCreateInfo *ci) { // UNDEFINED -> GENERAL; new depth / stencil images start cleared (LAS textures.c: the game copies depth // without stencil, and the stencil of those images must not be whatever the memory held). +// A reused image (create_tex) was another texture's: the transition waits for every earlier command, which may still +// read or write it. static void init_image(Tex *e, VkCommandBuffer cb) { - VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_NONE, + VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .srcStageMask = e->reused ? VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT : VK_PIPELINE_STAGE_2_NONE, + .srcAccessMask = e->reused ? VK_ACCESS_2_MEMORY_WRITE_BIT : 0, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, @@ -371,6 +435,22 @@ static void init_image(Tex *e, VkCommandBuffer cb) { } } +// A new entry's first transition. A reused image goes through it at the replay's current position: earlier commands +// of the part being recorded may still use the image as the texture it was, and the upload batch runs before the part. +static void init_new(Tex *e) { + if (e->reused) { + int reopen; + VkCommandBuffer cb = vk_exec_inline_begin(&reopen); + if (cb) { + init_image(e, cb); + vk_exec_inline_end(reopen); + return; + } + } + init_image(e, vk_upload_begin()); + vk_upload_end(); +} + // NX2NAVI_SHADOW_SCALE (1 to 4): the game's shadow maps are made this many times larger on each side. NX2navi renders its // sun cascades into a 960x960 DEPTH16 2D array of 4 layers and its static shadows into 1440x1440 and 720x720 arrays of 2, // whatever the output resolution, and shrinks the cascades under load (872, 848, 800, 480 and so on). Those are made @@ -540,9 +620,16 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { e->t = *t; e->f = f; e->pub.addr = k->addr; + img_key_of(&ci, &e->ik); VkImage img = VK_NULL_HANDLE; if (scale == 1 && nvn_gpu_only(k->addr)) { // storage in a physical or virtual pool: alias the pool memory (section 3.2) - if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS && !bind_pool_memory(c, e, img, L.size)) { + const VkMemoryRequirements *q = nvn_hm_get(&img_reqs, &e->ik); + VkMemSpan sp; + if (q && !pool_fits(c, e, q, L.size, &sp)) { // it would not bind there (img_reqs) + n_bind_skips++; + log_limited(&n_unbound, "texture %#llx (%ux%u fmt %#x) cannot alias its pool memory; own memory instead", + (unsigned long long)k->addr, w, h, d->format); + } else if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS && !bind_pool_memory(c, e, img, L.size)) { vkDestroyImage(c->device, img, NULL); img = VK_NULL_HANDLE; log_limited(&n_unbound, "texture %#llx (%ux%u fmt %#x) cannot alias its pool memory; own memory instead", @@ -578,7 +665,33 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { img = VK_NULL_HANDLE; } } + int gpu_only = !tex_mirrored(t); + if (!img && gpu_only && tex_reuse_on()) { // the image of a released entry with these parameters (reuse, above) + // Not one released in this frame or the one before: Ocarina releases its render targets and registers them + // again every frame, and DLSS's scene target (2560x1440 at a fixed address in Kokiri Forest) lost its image + // to a moving target of the same size in between, and DLSS its history (vk_dlaa.c evaluate), several times a + // second. Not one at this address either: its own descriptor comes back for it (start_at). Of the others the + // one nearest in address, so a target the allocator moves keeps the image it had. + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + Tex *a = NULL; + uint64_t best = UINT64_MAX; + int n = 0; + for (Tex *q = nvn_hm_get(&reuse, &e->ik); q && n < 128; q = q->ru_next, n++) { + if (q->rel_epoch == submit_epoch || now - q->rel_frame < 2 || q->key.addr == k->addr) continue; + uint64_t dist = q->key.addr > k->addr ? q->key.addr - k->addr : k->addr - q->key.addr; + if (dist < best) best = dist, a = q; + } + if (a) { + img = a->pub.image, e->alloc = a->alloc; + vk_exec_image_gone(img); // the views and records of the texture it was + a->pub.image = VK_NULL_HANDLE, a->alloc = NULL; + destroy_tex(a); + e->reused = 1; + n_reused++; + } + } if (!img) { + if (gpu_only) n_reuse_new++; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); if (r != VK_SUCCESS) { @@ -593,6 +706,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { vmaGetAllocationInfo(c->vma, e->alloc, &own); e->bytes = own.size; } + e->reusable = e->alloc && gpu_only; VkTexImage *p = &e->pub; p->image = img; p->format = f.vk; @@ -619,10 +733,7 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { p->fb_usage = (uint8_t)fb_usage; p->scale = (uint8_t)scale; - if (!defer_init) { - init_image(e, vk_upload_begin()); - vk_upload_end(); - } + if (!defer_init) init_new(e); return e; } @@ -667,6 +778,24 @@ static void own_remove(Tex *e) { static uint64_t rel_bytes; // own memory of the released entries (Tex.bytes) +static void ru_link(Tex *e) { + Tex *head = nvn_hm_get(&reuse, &e->ik); + e->ru_prev = NULL, e->ru_next = head; + if (head) head->ru_prev = e; + nvn_hm_put(&reuse, &e->ik, e); + e->ru_on = 1; +} + +static void ru_unlink(Tex *e) { + if (!e->ru_on) return; + if (e->ru_prev) e->ru_prev->ru_next = e->ru_next; + else if (e->ru_next) nvn_hm_put(&reuse, &e->ik, e->ru_next); + else nvn_hm_del(&reuse, &e->ik); + if (e->ru_next) e->ru_next->ru_prev = e->ru_prev; + e->ru_prev = e->ru_next = NULL; + e->ru_on = 0; +} + static void rel_append(Tex *e) { e->rel_next = NULL; e->rel_prev = rel_tail; @@ -674,9 +803,11 @@ static void rel_append(Tex *e) { else rel_head = e; rel_tail = e; rel_bytes += e->bytes; + if (e->reusable && tex_reuse_on()) ru_link(e); } static void rel_unlink(Tex *e) { + ru_unlink(e); rel_bytes -= e->bytes; if (e->rel_prev) e->rel_prev->rel_next = e->rel_next; else rel_head = e->rel_next; @@ -699,7 +830,7 @@ static void destroy_tex(Tex *e) { if (p->next_at == e) { p->next_at = e->next_at; break; } } if (e->released) nreleased--, rel_unlink(e); - vk_exec_image_gone(e->pub.image); + if (e->pub.image) vk_exec_image_gone(e->pub.image); // none when another texture took it (create_tex, reuse) vk_defer_image(e->pub.image, e->alloc); free(e->snap); free(e); @@ -1467,6 +1598,51 @@ static void create_failed(const TexKey *k) { failed_hold[i].k = *k, failed_hold[i].until_ns = plat_now_ns() + 2000000000ull; } +// The start of e's content at its address, for a new entry and for a superseded one taken back (taken_back): another +// descriptor's released entry there is superseded, and a live one holding GPU-written content is the alias source, +// preferring the one that differs only in swizzle (same_but_swizzle). A superseded entry is kept for its own +// descriptor, which takes it back and starts over here as a new entry would (alias copy or init_image): Ocarina +// alternates descriptors at some addresses every frame (a 2560x1440 target and a 480x270 one in Kokiri Forest), and +// destroying the released one made both images again each frame. Released mirrored entries are still destroyed: their +// content is the storage's, uploaded when an entry is made. +static void start_at(Tex *e, const TexKey *k, int taken_back) { + int alias = 0; + Tex *src = NULL; + for (Tex *p = nvn_hm_get(&by_addr, &k->addr), *nx; p; p = nx) { + nx = p->next_at; + if (p == e) continue; + if (p->released) { + if (p->pub.mirrored) destroy_tex(p); + else p->superseded = 1; + continue; + } + alias = 1; + if (tex_alias && (!src || (same_but_swizzle(&p->key, k) && !same_but_swizzle(&src->key, k))) && + alias_compatible(e, p) && alias_has_gpu_content(p)) + src = p; + } + if (alias) perf_count(PERF_CNT_TEX_ALIAS); + if (!tex_alias) { // create_tex initialised a new entry + if (taken_back) init_new(e); + return; + } + if (src && alias_copy(e, src)) { + n_alias_copied++; + if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes + vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen); + e->uploaded = 1; + e->pub.gpu_owned = 1; + } + e->chash_ok = 0; + } else { + init_new(e); + if (alias) { + if (e->pub.mirrored) n_alias_upload++; + else n_alias_uninit++; + } + } +} + static Tex *entry_of(const tex_t *t) { TexKey k; make_key(t, &k); @@ -1480,37 +1656,7 @@ static Tex *entry_of(const tex_t *t) { } if (!(e = create_tex(t, &k, tex_alias))) return create_failed(&k), NULL; perf_count(PERF_CNT_TEX_NEW); - // released entries of another descriptor at this address are replaced by the new one; a live one holding - // GPU-written content is the alias source, preferring the one that differs only in swizzle (same_but_swizzle) - int alias = 0; - Tex *src = NULL; - for (Tex *p = nvn_hm_get(&by_addr, &k.addr), *nx; p; p = nx) { - nx = p->next_at; - if (p->released) { destroy_tex(p); continue; } - alias = 1; - if (tex_alias && (!src || (same_but_swizzle(&p->key, &k) && !same_but_swizzle(&src->key, &k))) && - alias_compatible(e, p) && alias_has_gpu_content(p)) - src = p; - } - if (alias) perf_count(PERF_CNT_TEX_ALIAS); - if (tex_alias) { - if (src && alias_copy(e, src)) { - n_alias_copied++; - if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes - vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen); - e->uploaded = 1; - e->pub.gpu_owned = 1; - } - e->chash_ok = 0; - } else { - init_image(e, vk_upload_begin()); - vk_upload_end(); - if (alias) { - if (e->pub.mirrored) n_alias_upload++; - else n_alias_uninit++; - } - } - } + start_at(e, &k, 0); e->next_at = nvn_hm_get(&by_addr, &k.addr); nvn_hm_put(&by_addr, &k.addr, e); nvn_hm_put(&by_key, &k, e); @@ -1518,7 +1664,14 @@ static Tex *entry_of(const tex_t *t) { tex_gen_bump(k.addr); vk_texstat_time(VK_TS_CREATE, c0); } - if (e->released) e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_bump(k.addr); + if (e->released) { + e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_bump(k.addr); + if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work + e->superseded = 0, e->reused = 1; + n_restarts++; + start_at(e, &k, 1); + } + } e->t = *t; touch(e); return e; @@ -1611,6 +1764,7 @@ void vk_texture_release(const tex_t *t) { Tex *e = nvn_hm_get(&by_key, &k); if (e && !e->released) { e->released = 1; + e->rel_epoch = submit_epoch; e->rel_serial = reg_serial; e->rel_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); nreleased++; @@ -1745,7 +1899,7 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { // access violation in tex_bl_copy from the pool refresh, Kokiri Forest). It resolves to nothing; a slot // registered again after the release is live and revives the entry. if (!e && any) e = any; - else if (!e && !cpu_slot) e = own; + else if (!e && !cpu_slot) e = own && own->superseded ? NULL : own; // a superseded one: entry_of takes it back else if (!e && own && (uint32_t)(sinfo[s - tslots] >> 16) <= own->rel_serial) stale = 1; } if (!e && s->t.mem && !stale) { @@ -1894,11 +2048,14 @@ void vk_texture_new_submit(void) { (unsigned long long)untr); n_checks = 0; rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, " - "under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB)", + "under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB); images " + "reused %llu, made %llu, pool bindings not tried %llu, superseded taken back %llu", (unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops, (unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_evict, - n_evict_bytes / 1048576.0); + n_evict_bytes / 1048576.0, (unsigned long long)n_reused, (unsigned long long)n_reuse_new, + (unsigned long long)n_bind_skips, (unsigned long long)n_restarts); n_cap_drops = n_pressure_drops = n_commit_drops = n_evict = n_evict_bytes = 0; + n_reused = n_reuse_new = n_bind_skips = n_restarts = 0; } unlock(); } @@ -2707,6 +2864,10 @@ void vk_texture_shutdown(void) { free(by_key.keys), free(by_key.vals), free(by_addr.keys), free(by_addr.vals), free(samplers.keys), free(samplers.vals); by_key = (nvn_hmap){.klen = sizeof(TexKey)}; by_addr = (nvn_hmap){.klen = 8}; + for (uint32_t i = 0; i < img_reqs.cap; i++) free(img_reqs.vals[i]); + free(img_reqs.keys), free(img_reqs.vals), free(reuse.keys), free(reuse.vals); // reuse: emptied by destroy_tex + img_reqs = (nvn_hmap){.klen = sizeof(ImgKey)}; + reuse = (nvn_hmap){.klen = sizeof(ImgKey)}; samplers = (nvn_hmap){.klen = sizeof(samp_par)}; free(tslots), free(sslots); tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0, pool_slots = 0; diff --git a/tools/make_game.py b/tools/make_game.py index cdbca71a..17b746fd 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1355,9 +1355,10 @@ def main(): for pip_name, mod in packages.items(): try: __import__(mod) - except ImportError: + except ImportError as e: # its own message too: an installed package can fail to load (a DLL, a clash) extra = ' fonttools pillow' if prompt_art_dir() else '' - fail(f'the Python package {pip_name} is missing: pip install numpy lz4 zstandard zbic{extra}') + fail(f'the Python package {pip_name} is missing or does not load ({e}): ' + f'pip install numpy lz4 zstandard zbic{extra}') cache.mkdir(parents=True, exist_ok=True) reset = state.get('version') != STATE_VERSION or state.get('input') != src.identity if reset: