From 1241efa458b8c2d14d5277933d466f4a6f2a6b9a Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 16:43:48 -0500 Subject: [PATCH 1/2] Give back only the touched piece of a lazily written range, and say in the log why ranges are written back. With the lazy write-back on, a village and Hyrule Field still merged 20 to 29 MiB a frame back into guest memory at the end of every submission, inside the GPU work the game's fence waits for. The new log line (mem: lazy write-back at frame N: written back and not armed since the last line) names the reason by bytes: none of it lay in frozen regions or on pages that could not be armed; it lay in regions held after a touch. The game's CPU reads a few words at the start of buffers its shaders fill (12 bytes into ranges of 6.3 and 15.2 MiB among others), about once a second. Each read gave back the whole armed range and held every 2 MiB region it reached for 1 s, doubling to 32 s, and nothing was armed in a held region. A touch now gives back the 64 KiB around its address, fetched alone, and holds only that piece; the rest of the range stays armed (wb_lazy_chunk, default 1; 0 is the old rule). More than four touches in one region within a second are taken as a reader going through the range: that range goes whole and its regions are held as before, and the region's touches go that way at once while it keeps being touched within a minute. Each of the first eight touches of a log window is logged under the perf trace with its thread, address, range and what was given back. Measured on one machine (NVIDIA, uncapped, frame generation off, scripted walks, medians over the windows of 46 s of world clock): - village: 121.8, 124.1 and 124.4 fps before; 138.0 and 135.4 with this change; GPU busy 4.8 to 4.15 ms a frame - Hyrule Field: 101.1 and 113.1 fps before; 131.5 and 131.3 after; GPU busy 6.5 to 4.7 ms - merged a frame: 20 to 29 MiB before, 1.7 to 2.0 MiB after - faults on armed pages in 1,000 frames: 14 and 16 before, 17 to 18 after in the village; 19 and 23 before, 17 to 21 after in Hyrule Field; time spent fetching the same (4 to 5 ms in 1,000 frames) A run with synthetic reads of four armed ranges per log window (wb_lazy_touch = 4, 769 faults) ended without an error. The picture of every presented frame of a village walk and a field walk (thumbnails, 10,131 and 9,531 frames) had no black frame. The village walk has single frames whose brightness jumps: 14 with the change and 28 without it in about 10,000 frames each; the two looked at, one of each, are jumps of the camera. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 5 ++ runtime/cfg.c | 6 +- runtime/memory.c | 199 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 195 insertions(+), 15 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 017f5de3..dfa3c119 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -741,6 +741,11 @@ # "mem: lazy write-back" line says which is in effect and why). 0 elsewhere. With shadow storage (Linux, Wine) it # stays off whatever is set. # wb_lazy = 1 +# With wb_lazy: 1: when the game's CPU code touches such memory, only the 64 KiB around the place it touched are copied +# back and kept up to date from then on; the rest of that buffer stays on the video card. The game reads a few words +# at the start of buffers of several MiB about twice a second. 0: the whole buffer comes back and everything near it +# is copied back every frame for up to 32 seconds, as before (a tenth fewer frames a second in a village). +# wb_lazy_chunk = 1 # Barriers between the game's draw passes and compute dispatches. 1: only where the game itself asks for one, or where # the work changes between drawing and compute; this takes about a tenth off the graphics card's time for a frame. # 0: a full barrier before every pass and every dispatch. Default: 1 on NVIDIA graphics cards, 0 on all others, where it diff --git a/runtime/cfg.c b/runtime/cfg.c index 09b79cef..7e60364e 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -66,7 +66,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_ALIAS", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", @@ -127,7 +127,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; @@ -274,7 +274,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/memory.c b/runtime/memory.c index 425df439..5f57ff9b 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -123,7 +123,8 @@ static void unprobe_locked(uint64_t r) { // the mirror (mem_lazy_skip marks it stale). The first read or write of an armed page faults: lazy_fault has the // back end copy the range's bytes down now (lz_fetch, a GPU copy after everything submitted so far), gives the pages // back and runs the access again, and the range stays out for a while (lz_hot: 1 s, doubling to 32 s while it keeps -// being touched), written back every submission as before. Every thaw (before the kernel writes or reads guest +// being touched), written back every submission as before; by default only the piece around the address goes that +// way and the rest of the range stays armed (NX2NAVI_WB_LAZY_CHUNK, below). Every thaw (before the kernel writes or reads guest // memory, and before mapping changes) releases the armed ranges of its regions the same way. // A region with armed pages is not frozen (freeze_locked wants every page read-write), and a frozen one is not armed. // A fetch brings what the GPU work handed over so far wrote. The part of a submission that is being recorded has left @@ -146,6 +147,42 @@ static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size); // mirror's bytes over the whole range, so a CPU write the mirror does not have yet would be lost to it. static int (*lz_clean)(uint64_t lo, uint64_t hi, void *ctx); static uint64_t lz_unclean; // frz_lock: arms refused since the last report because the mirror lacked a CPU write +// frz_lock: bytes mem_lazy_arm left unarmed since the last report, by reason: the region is frozen, held after a touch +// (lz_hot), held after a thaw, has pages that are not read-write (or lacks a CPU write in the mirror), or the bytes lie +// in the partial pages at a range's ends. They are written back at every submission that writes them. +enum { LZ_NO_FROZEN, LZ_NO_HOT, LZ_NO_THAW, LZ_NO_PAGES, LZ_NO_EDGE, LZ_NO_N }; +static uint64_t lz_no[LZ_NO_N], lz_no_small, lz_arm_calls; // ranges without a whole page; ranges written back +// NX2NAVI_WB_LAZY_CHUNK (default 1; 0 = a touch releases its whole range and holds its regions, as before): the +// first touch of an armed range gives back only the LZ_CHUNK bytes around the address, fetched alone, and only that +// piece is held (lz_hots, 1 s doubling to 32 s like a region's hold); the rest of the range stays armed. More than +// LZ_HEAVY_N touches in one region within LZ_HEAVY_MS are a reader going through the range: its range goes whole +// and its regions are held, as before, and while the region is touched again within LZ_HEAVY_KEEP_MS its touches go +// that way at once, so such a reader pays LZ_HEAVY_N faults more, once. +// Ocarina's CPU reads a few words at the start of buffers its shaders fill (12 bytes into ranges of 6.3 and 15.2 MiB, +// among others), about twice a second in a village and in Hyrule Field. Each read gave back the whole range and held +// every 2 MiB region it reached for up to 32 s, in which nothing was armed: 22 to 26 MiB a frame went back to guest +// memory at the end of every submission although nothing read them, inside the GPU work the game's fence waits for. +// With the pieces 1.7 MiB a frame go back, on 17 to 18 faults in 1,000 frames where there were 14 to 16 and with the +// same time spent fetching (4 to 5 ms in 1,000 frames), and the village ran at 135 to 138 frames a second where it +// had run at 122 to 124. +#define LZ_CHUNK_SHIFT 16 +#define LZ_CHUNK (1ull << LZ_CHUNK_SHIFT) +#define LZ_HOTS 1024 // a power of two +#define LZ_HEAVY_N 4 +#define LZ_HEAVY_MS 1000 +#define LZ_HEAVY_KEEP_MS 60000 +typedef struct { uint64_t key; uint32_t until; uint8_t back; } LzHot; // key: chunk number + 1, 0 free; until as lz_hot +static LzHot lz_hots[LZ_HOTS]; // frz_lock: open addressing, a short linear probe +static int lz_nhots; // frz_lock: entries ever used (0: no lookup needed) +static uint32_t lz_touch_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 of the first piece of the region's run +static uint8_t lz_touch_n[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: pieces given back in that run +static uint32_t lz_heavy_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: ... of its last touch taken as a reader's +static int lz_chunk; // the setting, read in mem_init +static uint64_t lz_pieces, lz_piece_bytes, lz_heavy; // frz_lock, since the last report +// frz_lock: the first faults on armed pages since the last report, for its lines (trace perf) +#define LZ_NOTES 8 +static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale, hold; } lz_note[LZ_NOTES]; +static int lz_nnote; // frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps; static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else @@ -197,6 +234,73 @@ static void lazy_drop_locked(int i, int why) { } } +// frz_lock held. The hold entry of chunk c; make: 1 takes a free slot of the probe for it, or the one whose hold +// ended longest ago. +static LzHot *lazy_hot_locked(uint64_t c, int make) { + uint32_t h = (uint32_t)((c * 0x9e3779b97f4a7c15ull) >> 40); + LzHot *take = NULL; + for (int k = 0; k < 8; k++) { + LzHot *e = &lz_hots[(h + (uint32_t)k) & (LZ_HOTS - 1)]; + if (e->key == c + 1) return e; + if (!take || (take->key && (!e->key || (int32_t)(e->until - take->until) < 0))) take = e; + } + if (!make) return NULL; + *take = (LzHot){c + 1, 0, 0}; + lz_nhots++; + return take; +} + +static int lazy_chunk_held_locked(uint64_t c, uint32_t now) { + if (!lz_nhots) return 0; + const LzHot *e = lazy_hot_locked(c, 0); + return e && e->until && (int32_t)(e->until - now) > 0; +} + +static void lazy_push_locked(LzSeg s) { + if (lz_n == lz_cap) { + lz_cap = lz_cap ? lz_cap * 2 : 256; + lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap); + if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges"); + } + lzs[lz_n++] = s; +} + +// frz_lock held exclusively. A fault at a in segment i: the LZ_CHUNK piece around a goes (its bytes come down first +// when the mirror's are newer, then its pages are read-write again) and is held; what is left of the segment on +// either side stays armed. Returns the piece's size. +static uint64_t lazy_piece_locked(int i, uint64_t a, int write) { + LzSeg s = lzs[i]; + uint64_t c0 = a & ~(LZ_CHUNK - 1), c1 = c0 + LZ_CHUNK; + if (c0 < s.lo) c0 = s.lo; + if (c1 > s.hi) c1 = s.hi; + if (s.stale && lz_fetch) { + uint64_t t0 = plat_now_ns(); + __atomic_store_n(&lz_fetch_now, c1 - c0, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_tid, plat_thread_id(), __ATOMIC_RELEASE); + lz_fetch(s.h0, s.h1, (uint64_t)((int64_t)c0 + s.delta), c1 - c0); + __atomic_store_n(&lz_fetch_ms, (uint32_t)plat_now_ms() | 1, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_total, lz_fetch_total + 1, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_tid, 0, __ATOMIC_RELEASE); + lz_fetches++, lz_fetch_bytes += c1 - c0, lz_fetch_ns += plat_now_ns() - t0; + } + for (uint64_t x = c0, y; x < c1; x = y) + if (plat_mem_state(x, c1, &y) == PLAT_PROT_NONE && !plat_mem_protect(x, y - x, PLAT_PROT_RW)) + rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)x, + (unsigned long long)(y - x)); + lzs[i] = lzs[--lz_n]; + if (c0 > s.lo) lazy_push_locked((LzSeg){s.lo, c0, s.h0, s.h1, s.delta, s.stale}); + if (c1 < s.hi) lazy_push_locked((LzSeg){c1, s.hi, s.h0, s.h1, s.delta, s.stale}); + lazy_flags_locked(s.lo, s.hi); + lz_faults[write != 0]++, lz_pieces++, lz_piece_bytes += c1 - c0; + uint32_t now = (uint32_t)plat_now_ms(); + LzHot *e = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1); + int again = e->until && (int32_t)(now - e->until) < 60000; // touched again within a minute of its last hold + e->back = again ? (uint8_t)(e->back < 32 ? e->back * 2 : 32) : 1; + if (!e->back) e->back = 1; + e->until = (now + (uint32_t)e->back * 1000u) | 1; + return c1 - c0; +} + // frz_lock held exclusively: every segment that reaches into region r goes. static void lazy_drop_region_locked(uint64_t r, int why) { for (int i = lz_n; i-- > 0;) @@ -245,9 +349,15 @@ static int lazy_protect_locked(uint64_t x, uint64_t e, void *ctx) { uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off, void *ctx) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || a >= RT_ADDRESS_LIMIT || !watched(a, n)) return 0; uint64_t lo = (a + PAGE - 1) & ~(PAGE - 1), hi = (a + n) & ~(PAGE - 1), armed = 0; - if (hi <= lo) return 0; int64_t delta = (int64_t)off - (int64_t)a; plat_mutex_lock(&frz_lock); + lz_arm_calls++; + if (hi <= lo) { + lz_no[LZ_NO_EDGE] += n, lz_no_small++; + plat_mutex_unlock(&frz_lock); + return 0; + } + lz_no[LZ_NO_EDGE] += n - (hi - lo); if (lazy_covered_locked(lo, hi)) { plat_mutex_unlock(&frz_lock); return 0; @@ -257,9 +367,28 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t uint64_t r = x >> FRZ_SHIFT; e = (r + 1) << FRZ_SHIFT; if (e > hi) e = hi; - if (frz[r] || prb[r] || (lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0) || - (thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) || !lazy_protect_locked(x, e, ctx)) + if (lz_nhots) { // a held piece stays out: this turn arms what lies before it, the next ones what lies behind + uint64_t c = x >> LZ_CHUNK_SHIFT, y = x; + while (y < e && lazy_chunk_held_locked(c, now)) y = ++c << LZ_CHUNK_SHIFT; + if (y > x) { + if (y > e) y = e; + lz_no[LZ_NO_HOT] += y - x; + e = y; + continue; + } + for (c++; c << LZ_CHUNK_SHIFT < e; c++) + if (lazy_chunk_held_locked(c, now)) { + e = c << LZ_CHUNK_SHIFT; + break; + } + } + int no = frz[r] ? LZ_NO_FROZEN : prb[r] ? LZ_NO_PAGES : lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0 ? LZ_NO_HOT : + thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS ? LZ_NO_THAW : + !lazy_protect_locked(x, e, ctx) ? LZ_NO_PAGES : -1; + if (no >= 0) { + lz_no[no] += e - x; continue; + } uint64_t slo = x, shi = e; uint8_t stale = 0; for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer @@ -270,12 +399,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t lzs[j] = lzs[--lz_n]; j = -1; } - if (lz_n == lz_cap) { - lz_cap = lz_cap ? lz_cap * 2 : 256; - lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap); - if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges"); - } - lzs[lz_n++] = (LzSeg){slo, shi, h0, h1, delta, stale}; + lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale}); __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE); armed += e - x; } @@ -361,7 +485,29 @@ static int lazy_fault(uint64_t a, int write) { continue; } if (w) lz_gaps++; - lazy_drop_locked(i, write != 0); + int k = lz_nnote < LZ_NOTES ? lz_nnote++ : -1; + if (k >= 0) + lz_note[k].addr = a, lz_note[k].lo = lzs[i].lo, lz_note[k].hi = lzs[i].hi, lz_note[k].tid = me, + lz_note[k].write = write != 0, lz_note[k].stale = lzs[i].stale; + uint64_t r = a >> FRZ_SHIFT, out = lzs[i].hi - lzs[i].lo; + uint32_t now = (uint32_t)plat_now_ms(); + int reader = lz_heavy_ms[r] && (int32_t)(now - lz_heavy_ms[r]) < LZ_HEAVY_KEEP_MS; + if (lz_chunk && !reader) { + if (lz_touch_ms[r] && (int32_t)(now - lz_touch_ms[r]) < LZ_HEAVY_MS) { + if (++lz_touch_n[r] > LZ_HEAVY_N) reader = 1, lz_heavy++; + } else { + lz_touch_ms[r] = now | 1, lz_touch_n[r] = 1; + } + } + if (reader) lz_heavy_ms[r] = now | 1; + if (lz_chunk && !reader) { + out = lazy_piece_locked(i, a, write != 0); + if (k >= 0) lz_note[k].hold = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1)->back; + } else { + lazy_drop_locked(i, write != 0); + if (k >= 0) lz_note[k].hold = lz_back[r]; + } + if (k >= 0) lz_note[k].out = out; plat_mutex_unlock(&frz_lock); return 1; } @@ -403,10 +549,18 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; plat_mutex_lock(&frz_lock); uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes, - fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean; + fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean, no[LZ_NO_N]; int n = lz_n; for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo; lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = lz_gaps = lz_unclean = 0; + uint64_t small = lz_no_small, calls = lz_arm_calls; + int nnote = lz_nnote; + __typeof__(lz_note) note; + memcpy(note, lz_note, sizeof note); + lz_nnote = 0; + memcpy(no, lz_no, sizeof no); + memset(lz_no, 0, sizeof lz_no); + lz_no_small = lz_arm_calls = 0; plat_mutex_unlock(&frz_lock); rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the " "mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of " @@ -414,6 +568,26 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr, (unsigned long long)fw, (unsigned long long)gaps, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6, (unsigned long long)unclean); + uint64_t pieces = lz_pieces, piece_bytes = lz_piece_bytes, heavy = lz_heavy; + lz_pieces = lz_piece_bytes = lz_heavy = 0; + for (int i = 0; i < nnote && (rt_trace & RT_TRACE_PERF); i++) + rt_log("mem: lazy write-back: thread %u %s %#llx, %llu KiB into an armed range of %.2f MiB at %#llx; %llu KiB of " + "it given back%s and held for %u s", note[i].tid, note[i].write ? "wrote" : "read", + (unsigned long long)note[i].addr, (unsigned long long)((note[i].addr - note[i].lo) >> 10), + (note[i].hi - note[i].lo) / 1048576.0, (unsigned long long)note[i].lo, (unsigned long long)(note[i].out >> 10), + note[i].stale ? " (fetched)" : " (its bytes were current)", note[i].hold); + if (pieces || heavy) + rt_log("mem: lazy write-back at frame %llu: %llu touches gave back a piece only (%.2f MiB in all), %llu regions " + "were touched too often in a second and give back whole ranges for now", + (unsigned long long)frame, (unsigned long long)pieces, piece_bytes / 1048576.0, (unsigned long long)heavy); + if (no[LZ_NO_FROZEN] + no[LZ_NO_HOT] + no[LZ_NO_THAW] + no[LZ_NO_PAGES] + no[LZ_NO_EDGE]) + rt_log("mem: lazy write-back at frame %llu: written back and not armed since the last line: %.1f MiB in frozen " + "regions, %.1f MiB in regions the CPU touched lately, %.1f MiB in regions held after a thaw, %.1f MiB in " + "regions with pages that are not read-write or with a CPU write not in the mirror, %.2f MiB in the " + "partial pages at the ends of ranges; %llu ranges written back, %llu of them without a whole page", + (unsigned long long)frame, no[LZ_NO_FROZEN] / 1048576.0, no[LZ_NO_HOT] / 1048576.0, + no[LZ_NO_THAW] / 1048576.0, no[LZ_NO_PAGES] / 1048576.0, no[LZ_NO_EDGE] / 1048576.0, + (unsigned long long)calls, (unsigned long long)small); } // frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. @@ -673,6 +847,7 @@ void mem_init(void) { } else if (cfg_bool("NX2NAVI_WB_LAZY")) { // armed once the back end wants it and can fetch (mem_lazy_enable) plat_mem_access_hook(access_fault); lz_want = 1; + lz_chunk = cfg_bool("NX2NAVI_WB_LAZY_CHUNK"); } const char *pool = cfg_get("NX2NAVI_APP_POOL"); L = (RtGuestLayout){ From 892859cad0fb4aa33608d76262edc9b16d2b019b Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 16:43:48 -0500 Subject: [PATCH 2/2] Fill the records of a run of indirect mesh draws in one gap, so their pass is not cut at each of them. An indirect mesh draw reads its workgroup count through a ring entry that a buffer copy fills, and a copy cannot be recorded inside a pass: every such draw ended its pass, 45 times a frame in a village. The first draw of a run now fills the entries of the mesh draws that follow it with only state, draws, counter reports and render conditions between them, and those draws stay in the pass (mesh_ahead, default 1; 0 is the old way, ab switches in A/B blocks). An entry is taken only by the draw it was filled for, at the same record address, while no other op was replayed since. Counter reports count as staying in the pass because the replay closes their query inside it and writes the value from the completion thread; the game reports one after most of these draws, and without that only 38% of the draws were served. Measured in the village: 42 of the 45 draws a frame are served by an entry filled ahead; gaps with these copies 41 to 3 a frame (0.22 to 0.04 ms of GPU time); passes and dispatches 248 to 208 a frame (302 to 254 in Hyrule Field). With the piece release of the commit before, one run each: village 142.6 fps and GPU busy 3.87 ms (138.0 and 135.4 fps, 4.15 ms without this change), Hyrule Field 136.8 fps and 4.34 ms (131.5 and 131.3, 4.70 ms). In A/B blocks of one run with the narrower rule (counter reports still ending a run) the GPU's busy time went from 4.81 to 4.64 ms. Co-Authored-By: Claude Fable 5.2 --- docs/architecture.md | 4 ++- runtime/vk_exec.c | 67 ++++++++++++++++++++++++++++++++++++++++++ runtime/vk_exec.h | 3 ++ runtime/vk_exec_draw.c | 62 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 133 insertions(+), 3 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 7c3830dc..683d3638 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -128,7 +128,9 @@ keeps per-core data without locks and relies on that, so the host models it (`ru their stand-in, and the frame's later refreshes skip them (`runtime/vk_exec_draw.c` `pool_refill`). An area load spreads over several frames instead of holding one for 200-400 ms. - Mesh and task shaders run as `VK_EXT_mesh_shader`; indirect mesh draws read NVN's `{count, first}` record through a - ring of Vulkan `{x, y, z}` records filled on the GPU. + ring of Vulkan `{x, y, z}` records filled on the GPU. The copy cannot be recorded inside a pass, so the first such + draw of a pass fills the records of the mesh draws that follow it too, and the pass is not cut at each of them + (`runtime/vk_exec_draw.c` `mesh_ring_entry`). - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link at a new state, an optimised pipeline replacing it from a background thread. A draw whose pipeline is not ready is skipped when its pass is one the game renders frame after frame: one of its targets was rendered in the last few diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index f4877e70..740e2aba 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1310,6 +1310,7 @@ static void evo_resume(VkExec *ex) { if (!c || !cont_ready(c)) continue; progress = 1; evo_switches++; + vk_exec_brk++; vk_exec_pass_end(ex); vk_exec_barrier(ex); uint8_t *mine = malloc(ST_HI - ST_LO); @@ -1463,12 +1464,78 @@ void vk_exec_ahead(VkExec *ex) { } } +// Ops that only store state or draw: nothing they record runs outside a pass, and the walk goes on with the op after +// them. Counter reports and resets are among them: they close and open an occlusion query inside the pass, and the +// value reaches guest memory from the completion thread once the submission's fence has passed (vk_exec_xfer.c +// oq_cb, counter_cb), never from the GPU inside the run; so are the render conditions, which the replay decides on +// the CPU. Ocarina reports a counter after most of its indirect mesh draws. Any other op the walk replays bumps +// vk_exec_brk, and so does a held submission that goes on in between (evo_resume): what was prepared for the draws +// of a run of such ops (vk_exec_draw.c, the indirect mesh draws' records) holds only while it stays the same. +uint32_t vk_exec_brk; + +static int op_linear(uint32_t op) { + switch (op) { + case OP_BEGIN: case OP_JUMP: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_STENCIL: + case OP_POLY_OFFSET: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP: case OP_RASTER_DISCARD: case OP_PRIM_RESTART: + case OP_COPY_STRIDE: case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: + case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO: + case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC: case OP_DRAW_MESH: + case OP_DRAW_MESH_IND: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: + return 1; + default: return 0; + } +} + +#define MESH_RUN_OPS 2048 +static uint32_t mesh_run_stop[OP_COUNT_]; // the op each run of vk_exec_mesh_run ended at, since the start + +// For the perf log: the ops the runs ended at most often. +void vk_exec_mesh_run_stops(char *out, size_t n) { + uint32_t c[OP_COUNT_]; + memcpy(c, mesh_run_stop, sizeof c); + size_t at = 0; + *out = 0; + for (int k = 0; k < 4; k++) { + int best = 0; + for (int i = 1; i < OP_COUNT_; i++) + if (c[i] > c[best]) best = i; + if (!c[best]) break; + int w = snprintf(out + at, n - at, "%s%u at %s", k ? ", " : "", c[best], nvn_op_name((uint32_t)best)); + if (w < 0 || (size_t)w >= n - at) break; + at += (size_t)w, c[best] = 0; + } +} + +// The indirect mesh draws that follow the op being replayed (cur: its payload) with only ops of op_linear between +// them, MESH_RUN_OPS at most: their payloads into out, at most max. 0 when the walk is not at cur. +int vk_exec_mesh_run(const void *cur, const void **out, int max) { + const op_hdr *h = AH.at[AH.depth]; + if (!h || nvn_op_payload(h) != cur) return 0; + int n = 0; + h = nvn_op_next(h); + for (int ops = 0; ops < MESH_RUN_OPS && n < max; ops++) { + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; + if (!op_linear(h->op)) { + mesh_run_stop[h->op]++; + break; + } + if (h->op == OP_JUMP) { + h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target; + continue; + } + if (h->op == OP_DRAW_MESH_IND) out[n++] = nvn_op_payload(h); + h = nvn_op_next(h); + } + return n; +} + static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { for (;;) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); const void *a = nvn_op_payload(h); AH.at[depth] = h, AH.depth = depth; // where vk_exec_ahead reads on from + if (!op_linear(h->op)) vk_exec_brk++; switch (h->op) { case OP_END: return NULL; case OP_BEGIN: break; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 4758dda0..4e4e533a 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -263,6 +263,9 @@ uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after t // signal met, and the sum of how many submissions later it came. void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs); extern uint64_t vk_exec_submit_seq; // replay thread: submissions replayed so far +extern uint32_t vk_exec_brk; // replay thread: bumped by every op that is not state or a draw (vk_exec.c op_linear) +int vk_exec_mesh_run(const void *cur, const void **out, int max); +void vk_exec_mesh_run_stops(char *out, size_t n); // NX2NAVI_EVO_TRACE=1 (diagnostics, vk_exec.c "event order trace"): one token per step of the replay's event order. void vk_exec_evt(char what, const void *event, uint32_t value); void vk_exec_evt_frame(uint64_t frame); // the game's present call: the frame's tokens are closed and logged diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 8e5eea68..98a5da9b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5003,7 +5003,44 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { ex->recorded++; } -static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) { +// NX2NAVI_MESH_AHEAD (default 1; 0 = each draw by itself, "ab" switches with the A/B blocks): the copy of a record +// cannot be recorded inside a pass, so every indirect mesh draw ended its pass, and a pass that draws its meshes one +// after another was cut into as many pieces (a village: 45 of them a frame, each with a gap of its own on the GPU). +// The first such draw of a run now fills the entries of the ones that follow it as well, with nothing but state, +// draws and counter reports between them (vk_exec.c op_linear, vk_exec_mesh_run): those draws then find their entry +// and stay in the pass. The culling compute that writes the records cannot run inside that run, and an entry is taken +// only by the draw it was filled for, at the same record address, while no other op was replayed since +// (vk_exec_brk). In the village 42 of the 45 draws a frame are served that way: 3 gaps with these copies a frame +// where there were 41 (0.04 ms of GPU time for 0.22), and about 40 passes fewer to begin and end. +enum { MESH_AHEAD = 64 }; +static struct { const p_draw_mesh_ind *d; uint64_t ind; VkDeviceSize at; } mesh_ahead[MESH_AHEAD]; +static int mesh_ahead_n, mesh_ahead_i; +static uint32_t mesh_ahead_brk; +static uint64_t mesh_ahead_taken, mesh_ahead_own; // since the start: draws served by an entry filled ahead; the others + +static int mesh_ahead_on(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("MESH_AHEAD", "1"); + return RT_KNOB_ON(mode); +} + +static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { + if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) { + char stops[160]; + vk_exec_mesh_run_stops(stops, sizeof stops); + rt_log("perf: indirect mesh draws: %llu so far, %llu of them with their record filled ahead by an earlier draw of " + "their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1), + (unsigned long long)mesh_ahead_taken, stops); + } + if (mesh_ahead_i < mesh_ahead_n) { + if (mesh_ahead[mesh_ahead_i].d == d && mesh_ahead[mesh_ahead_i].ind == d->ind && mesh_ahead_brk == vk_exec_brk) { + *at = mesh_ahead[mesh_ahead_i++].at; + mesh_ahead_taken++; + return 1; + } + mesh_ahead_n = mesh_ahead_i = 0; // the walk went another way: the entries left are never drawn from + } + mesh_ahead_own++; if (!mesh_ring) { VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING, .usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; @@ -5021,6 +5058,27 @@ static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) { VkBufferCopy cp = {src->offset, *at, 4}; vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); mesh_count_note(ex, src); + mesh_ahead_n = mesh_ahead_i = 0; + if (mesh_ahead_on()) { + const void *run[MESH_AHEAD]; + int n = vk_exec_mesh_run(d, run, MESH_AHEAD); + for (int i = 0; i < n; i++) { + const p_draw_mesh_ind *f = run[i]; + VkMemSpan ip; + if (!vk_mem_resolve(f->ind, 12, &ip) || ip.offset % 4) break; // that draw reports it when the walk gets there + VK_MEM_USE(ip); + VkDeviceSize fa = 12ull * (mesh_ring_next++ % MESH_RING); + vk_gpu_gap_note(VK_GAP_MESH_RING, 12); + vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1); + VkBufferCopy fc = {ip.offset, fa, 4}; + vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc); + mesh_count_note(ex, &ip); + ex->recorded += 2; + mesh_ahead[mesh_ahead_n].d = f, mesh_ahead[mesh_ahead_n].ind = f->ind, mesh_ahead[mesh_ahead_n].at = fa; + mesh_ahead_n++; + } + mesh_ahead_brk = vk_exec_brk; + } VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, @@ -5052,7 +5110,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { c0_binst = 0; VK_MEM_USE(ip); VkDeviceSize at; - if (!mesh_ring_entry(ex, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); + if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; vk_exec_frame.draws++; static uint32_t n_logged;