diff --git a/docs/architecture.md b/docs/architecture.md index 7c3830dc..683d3638 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -128,7 +128,9 @@ keeps per-core data without locks and relies on that, so the host models it (`ru their stand-in, and the frame's later refreshes skip them (`runtime/vk_exec_draw.c` `pool_refill`). An area load spreads over several frames instead of holding one for 200-400 ms. - Mesh and task shaders run as `VK_EXT_mesh_shader`; indirect mesh draws read NVN's `{count, first}` record through a - ring of Vulkan `{x, y, z}` records filled on the GPU. + ring of Vulkan `{x, y, z}` records filled on the GPU. The copy cannot be recorded inside a pass, so the first such + draw of a pass fills the records of the mesh draws that follow it too, and the pass is not cut at each of them + (`runtime/vk_exec_draw.c` `mesh_ring_entry`). - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link at a new state, an optimised pipeline replacing it from a background thread. A draw whose pipeline is not ready is skipped when its pass is one the game renders frame after frame: one of its targets was rendered in the last few diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 017f5de3..dfa3c119 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -741,6 +741,11 @@ # "mem: lazy write-back" line says which is in effect and why). 0 elsewhere. With shadow storage (Linux, Wine) it # stays off whatever is set. # wb_lazy = 1 +# With wb_lazy: 1: when the game's CPU code touches such memory, only the 64 KiB around the place it touched are copied +# back and kept up to date from then on; the rest of that buffer stays on the video card. The game reads a few words +# at the start of buffers of several MiB about twice a second. 0: the whole buffer comes back and everything near it +# is copied back every frame for up to 32 seconds, as before (a tenth fewer frames a second in a village). +# wb_lazy_chunk = 1 # Barriers between the game's draw passes and compute dispatches. 1: only where the game itself asks for one, or where # the work changes between drawing and compute; this takes about a tenth off the graphics card's time for a frame. # 0: a full barrier before every pass and every dispatch. Default: 1 on NVIDIA graphics cards, 0 on all others, where it diff --git a/runtime/cfg.c b/runtime/cfg.c index 09b79cef..7e60364e 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -66,7 +66,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_ALIAS", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", @@ -127,7 +127,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; @@ -274,7 +274,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/memory.c b/runtime/memory.c index 425df439..5f57ff9b 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -123,7 +123,8 @@ static void unprobe_locked(uint64_t r) { // the mirror (mem_lazy_skip marks it stale). The first read or write of an armed page faults: lazy_fault has the // back end copy the range's bytes down now (lz_fetch, a GPU copy after everything submitted so far), gives the pages // back and runs the access again, and the range stays out for a while (lz_hot: 1 s, doubling to 32 s while it keeps -// being touched), written back every submission as before. Every thaw (before the kernel writes or reads guest +// being touched), written back every submission as before; by default only the piece around the address goes that +// way and the rest of the range stays armed (NX2NAVI_WB_LAZY_CHUNK, below). Every thaw (before the kernel writes or reads guest // memory, and before mapping changes) releases the armed ranges of its regions the same way. // A region with armed pages is not frozen (freeze_locked wants every page read-write), and a frozen one is not armed. // A fetch brings what the GPU work handed over so far wrote. The part of a submission that is being recorded has left @@ -146,6 +147,42 @@ static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size); // mirror's bytes over the whole range, so a CPU write the mirror does not have yet would be lost to it. static int (*lz_clean)(uint64_t lo, uint64_t hi, void *ctx); static uint64_t lz_unclean; // frz_lock: arms refused since the last report because the mirror lacked a CPU write +// frz_lock: bytes mem_lazy_arm left unarmed since the last report, by reason: the region is frozen, held after a touch +// (lz_hot), held after a thaw, has pages that are not read-write (or lacks a CPU write in the mirror), or the bytes lie +// in the partial pages at a range's ends. They are written back at every submission that writes them. +enum { LZ_NO_FROZEN, LZ_NO_HOT, LZ_NO_THAW, LZ_NO_PAGES, LZ_NO_EDGE, LZ_NO_N }; +static uint64_t lz_no[LZ_NO_N], lz_no_small, lz_arm_calls; // ranges without a whole page; ranges written back +// NX2NAVI_WB_LAZY_CHUNK (default 1; 0 = a touch releases its whole range and holds its regions, as before): the +// first touch of an armed range gives back only the LZ_CHUNK bytes around the address, fetched alone, and only that +// piece is held (lz_hots, 1 s doubling to 32 s like a region's hold); the rest of the range stays armed. More than +// LZ_HEAVY_N touches in one region within LZ_HEAVY_MS are a reader going through the range: its range goes whole +// and its regions are held, as before, and while the region is touched again within LZ_HEAVY_KEEP_MS its touches go +// that way at once, so such a reader pays LZ_HEAVY_N faults more, once. +// Ocarina's CPU reads a few words at the start of buffers its shaders fill (12 bytes into ranges of 6.3 and 15.2 MiB, +// among others), about twice a second in a village and in Hyrule Field. Each read gave back the whole range and held +// every 2 MiB region it reached for up to 32 s, in which nothing was armed: 22 to 26 MiB a frame went back to guest +// memory at the end of every submission although nothing read them, inside the GPU work the game's fence waits for. +// With the pieces 1.7 MiB a frame go back, on 17 to 18 faults in 1,000 frames where there were 14 to 16 and with the +// same time spent fetching (4 to 5 ms in 1,000 frames), and the village ran at 135 to 138 frames a second where it +// had run at 122 to 124. +#define LZ_CHUNK_SHIFT 16 +#define LZ_CHUNK (1ull << LZ_CHUNK_SHIFT) +#define LZ_HOTS 1024 // a power of two +#define LZ_HEAVY_N 4 +#define LZ_HEAVY_MS 1000 +#define LZ_HEAVY_KEEP_MS 60000 +typedef struct { uint64_t key; uint32_t until; uint8_t back; } LzHot; // key: chunk number + 1, 0 free; until as lz_hot +static LzHot lz_hots[LZ_HOTS]; // frz_lock: open addressing, a short linear probe +static int lz_nhots; // frz_lock: entries ever used (0: no lookup needed) +static uint32_t lz_touch_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 of the first piece of the region's run +static uint8_t lz_touch_n[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: pieces given back in that run +static uint32_t lz_heavy_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: ... of its last touch taken as a reader's +static int lz_chunk; // the setting, read in mem_init +static uint64_t lz_pieces, lz_piece_bytes, lz_heavy; // frz_lock, since the last report +// frz_lock: the first faults on armed pages since the last report, for its lines (trace perf) +#define LZ_NOTES 8 +static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale, hold; } lz_note[LZ_NOTES]; +static int lz_nnote; // frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps; static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else @@ -197,6 +234,73 @@ static void lazy_drop_locked(int i, int why) { } } +// frz_lock held. The hold entry of chunk c; make: 1 takes a free slot of the probe for it, or the one whose hold +// ended longest ago. +static LzHot *lazy_hot_locked(uint64_t c, int make) { + uint32_t h = (uint32_t)((c * 0x9e3779b97f4a7c15ull) >> 40); + LzHot *take = NULL; + for (int k = 0; k < 8; k++) { + LzHot *e = &lz_hots[(h + (uint32_t)k) & (LZ_HOTS - 1)]; + if (e->key == c + 1) return e; + if (!take || (take->key && (!e->key || (int32_t)(e->until - take->until) < 0))) take = e; + } + if (!make) return NULL; + *take = (LzHot){c + 1, 0, 0}; + lz_nhots++; + return take; +} + +static int lazy_chunk_held_locked(uint64_t c, uint32_t now) { + if (!lz_nhots) return 0; + const LzHot *e = lazy_hot_locked(c, 0); + return e && e->until && (int32_t)(e->until - now) > 0; +} + +static void lazy_push_locked(LzSeg s) { + if (lz_n == lz_cap) { + lz_cap = lz_cap ? lz_cap * 2 : 256; + lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap); + if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges"); + } + lzs[lz_n++] = s; +} + +// frz_lock held exclusively. A fault at a in segment i: the LZ_CHUNK piece around a goes (its bytes come down first +// when the mirror's are newer, then its pages are read-write again) and is held; what is left of the segment on +// either side stays armed. Returns the piece's size. +static uint64_t lazy_piece_locked(int i, uint64_t a, int write) { + LzSeg s = lzs[i]; + uint64_t c0 = a & ~(LZ_CHUNK - 1), c1 = c0 + LZ_CHUNK; + if (c0 < s.lo) c0 = s.lo; + if (c1 > s.hi) c1 = s.hi; + if (s.stale && lz_fetch) { + uint64_t t0 = plat_now_ns(); + __atomic_store_n(&lz_fetch_now, c1 - c0, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_tid, plat_thread_id(), __ATOMIC_RELEASE); + lz_fetch(s.h0, s.h1, (uint64_t)((int64_t)c0 + s.delta), c1 - c0); + __atomic_store_n(&lz_fetch_ms, (uint32_t)plat_now_ms() | 1, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_total, lz_fetch_total + 1, __ATOMIC_RELAXED); + __atomic_store_n(&lz_fetch_tid, 0, __ATOMIC_RELEASE); + lz_fetches++, lz_fetch_bytes += c1 - c0, lz_fetch_ns += plat_now_ns() - t0; + } + for (uint64_t x = c0, y; x < c1; x = y) + if (plat_mem_state(x, c1, &y) == PLAT_PROT_NONE && !plat_mem_protect(x, y - x, PLAT_PROT_RW)) + rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)x, + (unsigned long long)(y - x)); + lzs[i] = lzs[--lz_n]; + if (c0 > s.lo) lazy_push_locked((LzSeg){s.lo, c0, s.h0, s.h1, s.delta, s.stale}); + if (c1 < s.hi) lazy_push_locked((LzSeg){c1, s.hi, s.h0, s.h1, s.delta, s.stale}); + lazy_flags_locked(s.lo, s.hi); + lz_faults[write != 0]++, lz_pieces++, lz_piece_bytes += c1 - c0; + uint32_t now = (uint32_t)plat_now_ms(); + LzHot *e = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1); + int again = e->until && (int32_t)(now - e->until) < 60000; // touched again within a minute of its last hold + e->back = again ? (uint8_t)(e->back < 32 ? e->back * 2 : 32) : 1; + if (!e->back) e->back = 1; + e->until = (now + (uint32_t)e->back * 1000u) | 1; + return c1 - c0; +} + // frz_lock held exclusively: every segment that reaches into region r goes. static void lazy_drop_region_locked(uint64_t r, int why) { for (int i = lz_n; i-- > 0;) @@ -245,9 +349,15 @@ static int lazy_protect_locked(uint64_t x, uint64_t e, void *ctx) { uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off, void *ctx) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || a >= RT_ADDRESS_LIMIT || !watched(a, n)) return 0; uint64_t lo = (a + PAGE - 1) & ~(PAGE - 1), hi = (a + n) & ~(PAGE - 1), armed = 0; - if (hi <= lo) return 0; int64_t delta = (int64_t)off - (int64_t)a; plat_mutex_lock(&frz_lock); + lz_arm_calls++; + if (hi <= lo) { + lz_no[LZ_NO_EDGE] += n, lz_no_small++; + plat_mutex_unlock(&frz_lock); + return 0; + } + lz_no[LZ_NO_EDGE] += n - (hi - lo); if (lazy_covered_locked(lo, hi)) { plat_mutex_unlock(&frz_lock); return 0; @@ -257,9 +367,28 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t uint64_t r = x >> FRZ_SHIFT; e = (r + 1) << FRZ_SHIFT; if (e > hi) e = hi; - if (frz[r] || prb[r] || (lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0) || - (thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) || !lazy_protect_locked(x, e, ctx)) + if (lz_nhots) { // a held piece stays out: this turn arms what lies before it, the next ones what lies behind + uint64_t c = x >> LZ_CHUNK_SHIFT, y = x; + while (y < e && lazy_chunk_held_locked(c, now)) y = ++c << LZ_CHUNK_SHIFT; + if (y > x) { + if (y > e) y = e; + lz_no[LZ_NO_HOT] += y - x; + e = y; + continue; + } + for (c++; c << LZ_CHUNK_SHIFT < e; c++) + if (lazy_chunk_held_locked(c, now)) { + e = c << LZ_CHUNK_SHIFT; + break; + } + } + int no = frz[r] ? LZ_NO_FROZEN : prb[r] ? LZ_NO_PAGES : lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0 ? LZ_NO_HOT : + thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS ? LZ_NO_THAW : + !lazy_protect_locked(x, e, ctx) ? LZ_NO_PAGES : -1; + if (no >= 0) { + lz_no[no] += e - x; continue; + } uint64_t slo = x, shi = e; uint8_t stale = 0; for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer @@ -270,12 +399,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t lzs[j] = lzs[--lz_n]; j = -1; } - if (lz_n == lz_cap) { - lz_cap = lz_cap ? lz_cap * 2 : 256; - lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap); - if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges"); - } - lzs[lz_n++] = (LzSeg){slo, shi, h0, h1, delta, stale}; + lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale}); __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE); armed += e - x; } @@ -361,7 +485,29 @@ static int lazy_fault(uint64_t a, int write) { continue; } if (w) lz_gaps++; - lazy_drop_locked(i, write != 0); + int k = lz_nnote < LZ_NOTES ? lz_nnote++ : -1; + if (k >= 0) + lz_note[k].addr = a, lz_note[k].lo = lzs[i].lo, lz_note[k].hi = lzs[i].hi, lz_note[k].tid = me, + lz_note[k].write = write != 0, lz_note[k].stale = lzs[i].stale; + uint64_t r = a >> FRZ_SHIFT, out = lzs[i].hi - lzs[i].lo; + uint32_t now = (uint32_t)plat_now_ms(); + int reader = lz_heavy_ms[r] && (int32_t)(now - lz_heavy_ms[r]) < LZ_HEAVY_KEEP_MS; + if (lz_chunk && !reader) { + if (lz_touch_ms[r] && (int32_t)(now - lz_touch_ms[r]) < LZ_HEAVY_MS) { + if (++lz_touch_n[r] > LZ_HEAVY_N) reader = 1, lz_heavy++; + } else { + lz_touch_ms[r] = now | 1, lz_touch_n[r] = 1; + } + } + if (reader) lz_heavy_ms[r] = now | 1; + if (lz_chunk && !reader) { + out = lazy_piece_locked(i, a, write != 0); + if (k >= 0) lz_note[k].hold = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1)->back; + } else { + lazy_drop_locked(i, write != 0); + if (k >= 0) lz_note[k].hold = lz_back[r]; + } + if (k >= 0) lz_note[k].out = out; plat_mutex_unlock(&frz_lock); return 1; } @@ -403,10 +549,18 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; plat_mutex_lock(&frz_lock); uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes, - fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean; + fns = lz_fetch_ns, gaps = lz_gaps, unclean = lz_unclean, no[LZ_NO_N]; int n = lz_n; for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo; lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = lz_gaps = lz_unclean = 0; + uint64_t small = lz_no_small, calls = lz_arm_calls; + int nnote = lz_nnote; + __typeof__(lz_note) note; + memcpy(note, lz_note, sizeof note); + lz_nnote = 0; + memcpy(no, lz_no, sizeof no); + memset(lz_no, 0, sizeof lz_no); + lz_no_small = lz_arm_calls = 0; plat_mutex_unlock(&frz_lock); rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the " "mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of " @@ -414,6 +568,26 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr, (unsigned long long)fw, (unsigned long long)gaps, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6, (unsigned long long)unclean); + uint64_t pieces = lz_pieces, piece_bytes = lz_piece_bytes, heavy = lz_heavy; + lz_pieces = lz_piece_bytes = lz_heavy = 0; + for (int i = 0; i < nnote && (rt_trace & RT_TRACE_PERF); i++) + rt_log("mem: lazy write-back: thread %u %s %#llx, %llu KiB into an armed range of %.2f MiB at %#llx; %llu KiB of " + "it given back%s and held for %u s", note[i].tid, note[i].write ? "wrote" : "read", + (unsigned long long)note[i].addr, (unsigned long long)((note[i].addr - note[i].lo) >> 10), + (note[i].hi - note[i].lo) / 1048576.0, (unsigned long long)note[i].lo, (unsigned long long)(note[i].out >> 10), + note[i].stale ? " (fetched)" : " (its bytes were current)", note[i].hold); + if (pieces || heavy) + rt_log("mem: lazy write-back at frame %llu: %llu touches gave back a piece only (%.2f MiB in all), %llu regions " + "were touched too often in a second and give back whole ranges for now", + (unsigned long long)frame, (unsigned long long)pieces, piece_bytes / 1048576.0, (unsigned long long)heavy); + if (no[LZ_NO_FROZEN] + no[LZ_NO_HOT] + no[LZ_NO_THAW] + no[LZ_NO_PAGES] + no[LZ_NO_EDGE]) + rt_log("mem: lazy write-back at frame %llu: written back and not armed since the last line: %.1f MiB in frozen " + "regions, %.1f MiB in regions the CPU touched lately, %.1f MiB in regions held after a thaw, %.1f MiB in " + "regions with pages that are not read-write or with a CPU write not in the mirror, %.2f MiB in the " + "partial pages at the ends of ranges; %llu ranges written back, %llu of them without a whole page", + (unsigned long long)frame, no[LZ_NO_FROZEN] / 1048576.0, no[LZ_NO_HOT] / 1048576.0, + no[LZ_NO_THAW] / 1048576.0, no[LZ_NO_PAGES] / 1048576.0, no[LZ_NO_EDGE] / 1048576.0, + (unsigned long long)calls, (unsigned long long)small); } // frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. @@ -673,6 +847,7 @@ void mem_init(void) { } else if (cfg_bool("NX2NAVI_WB_LAZY")) { // armed once the back end wants it and can fetch (mem_lazy_enable) plat_mem_access_hook(access_fault); lz_want = 1; + lz_chunk = cfg_bool("NX2NAVI_WB_LAZY_CHUNK"); } const char *pool = cfg_get("NX2NAVI_APP_POOL"); L = (RtGuestLayout){ diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index f4877e70..740e2aba 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1310,6 +1310,7 @@ static void evo_resume(VkExec *ex) { if (!c || !cont_ready(c)) continue; progress = 1; evo_switches++; + vk_exec_brk++; vk_exec_pass_end(ex); vk_exec_barrier(ex); uint8_t *mine = malloc(ST_HI - ST_LO); @@ -1463,12 +1464,78 @@ void vk_exec_ahead(VkExec *ex) { } } +// Ops that only store state or draw: nothing they record runs outside a pass, and the walk goes on with the op after +// them. Counter reports and resets are among them: they close and open an occlusion query inside the pass, and the +// value reaches guest memory from the completion thread once the submission's fence has passed (vk_exec_xfer.c +// oq_cb, counter_cb), never from the GPU inside the run; so are the render conditions, which the replay decides on +// the CPU. Ocarina reports a counter after most of its indirect mesh draws. Any other op the walk replays bumps +// vk_exec_brk, and so does a held submission that goes on in between (evo_resume): what was prepared for the draws +// of a run of such ops (vk_exec_draw.c, the indirect mesh draws' records) holds only while it stays the same. +uint32_t vk_exec_brk; + +static int op_linear(uint32_t op) { + switch (op) { + case OP_BEGIN: case OP_JUMP: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_STENCIL: + case OP_POLY_OFFSET: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP: case OP_RASTER_DISCARD: case OP_PRIM_RESTART: + case OP_COPY_STRIDE: case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: + case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO: + case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC: case OP_DRAW_MESH: + case OP_DRAW_MESH_IND: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: + return 1; + default: return 0; + } +} + +#define MESH_RUN_OPS 2048 +static uint32_t mesh_run_stop[OP_COUNT_]; // the op each run of vk_exec_mesh_run ended at, since the start + +// For the perf log: the ops the runs ended at most often. +void vk_exec_mesh_run_stops(char *out, size_t n) { + uint32_t c[OP_COUNT_]; + memcpy(c, mesh_run_stop, sizeof c); + size_t at = 0; + *out = 0; + for (int k = 0; k < 4; k++) { + int best = 0; + for (int i = 1; i < OP_COUNT_; i++) + if (c[i] > c[best]) best = i; + if (!c[best]) break; + int w = snprintf(out + at, n - at, "%s%u at %s", k ? ", " : "", c[best], nvn_op_name((uint32_t)best)); + if (w < 0 || (size_t)w >= n - at) break; + at += (size_t)w, c[best] = 0; + } +} + +// The indirect mesh draws that follow the op being replayed (cur: its payload) with only ops of op_linear between +// them, MESH_RUN_OPS at most: their payloads into out, at most max. 0 when the walk is not at cur. +int vk_exec_mesh_run(const void *cur, const void **out, int max) { + const op_hdr *h = AH.at[AH.depth]; + if (!h || nvn_op_payload(h) != cur) return 0; + int n = 0; + h = nvn_op_next(h); + for (int ops = 0; ops < MESH_RUN_OPS && n < max; ops++) { + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; + if (!op_linear(h->op)) { + mesh_run_stop[h->op]++; + break; + } + if (h->op == OP_JUMP) { + h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target; + continue; + } + if (h->op == OP_DRAW_MESH_IND) out[n++] = nvn_op_payload(h); + h = nvn_op_next(h); + } + return n; +} + static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { for (;;) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); const void *a = nvn_op_payload(h); AH.at[depth] = h, AH.depth = depth; // where vk_exec_ahead reads on from + if (!op_linear(h->op)) vk_exec_brk++; switch (h->op) { case OP_END: return NULL; case OP_BEGIN: break; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 4758dda0..4e4e533a 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -263,6 +263,9 @@ uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after t // signal met, and the sum of how many submissions later it came. void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs); extern uint64_t vk_exec_submit_seq; // replay thread: submissions replayed so far +extern uint32_t vk_exec_brk; // replay thread: bumped by every op that is not state or a draw (vk_exec.c op_linear) +int vk_exec_mesh_run(const void *cur, const void **out, int max); +void vk_exec_mesh_run_stops(char *out, size_t n); // NX2NAVI_EVO_TRACE=1 (diagnostics, vk_exec.c "event order trace"): one token per step of the replay's event order. void vk_exec_evt(char what, const void *event, uint32_t value); void vk_exec_evt_frame(uint64_t frame); // the game's present call: the frame's tokens are closed and logged diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 8e5eea68..98a5da9b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5003,7 +5003,44 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { ex->recorded++; } -static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) { +// NX2NAVI_MESH_AHEAD (default 1; 0 = each draw by itself, "ab" switches with the A/B blocks): the copy of a record +// cannot be recorded inside a pass, so every indirect mesh draw ended its pass, and a pass that draws its meshes one +// after another was cut into as many pieces (a village: 45 of them a frame, each with a gap of its own on the GPU). +// The first such draw of a run now fills the entries of the ones that follow it as well, with nothing but state, +// draws and counter reports between them (vk_exec.c op_linear, vk_exec_mesh_run): those draws then find their entry +// and stay in the pass. The culling compute that writes the records cannot run inside that run, and an entry is taken +// only by the draw it was filled for, at the same record address, while no other op was replayed since +// (vk_exec_brk). In the village 42 of the 45 draws a frame are served that way: 3 gaps with these copies a frame +// where there were 41 (0.04 ms of GPU time for 0.22), and about 40 passes fewer to begin and end. +enum { MESH_AHEAD = 64 }; +static struct { const p_draw_mesh_ind *d; uint64_t ind; VkDeviceSize at; } mesh_ahead[MESH_AHEAD]; +static int mesh_ahead_n, mesh_ahead_i; +static uint32_t mesh_ahead_brk; +static uint64_t mesh_ahead_taken, mesh_ahead_own; // since the start: draws served by an entry filled ahead; the others + +static int mesh_ahead_on(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("MESH_AHEAD", "1"); + return RT_KNOB_ON(mode); +} + +static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { + if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) { + char stops[160]; + vk_exec_mesh_run_stops(stops, sizeof stops); + rt_log("perf: indirect mesh draws: %llu so far, %llu of them with their record filled ahead by an earlier draw of " + "their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1), + (unsigned long long)mesh_ahead_taken, stops); + } + if (mesh_ahead_i < mesh_ahead_n) { + if (mesh_ahead[mesh_ahead_i].d == d && mesh_ahead[mesh_ahead_i].ind == d->ind && mesh_ahead_brk == vk_exec_brk) { + *at = mesh_ahead[mesh_ahead_i++].at; + mesh_ahead_taken++; + return 1; + } + mesh_ahead_n = mesh_ahead_i = 0; // the walk went another way: the entries left are never drawn from + } + mesh_ahead_own++; if (!mesh_ring) { VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING, .usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; @@ -5021,6 +5058,27 @@ static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) { VkBufferCopy cp = {src->offset, *at, 4}; vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); mesh_count_note(ex, src); + mesh_ahead_n = mesh_ahead_i = 0; + if (mesh_ahead_on()) { + const void *run[MESH_AHEAD]; + int n = vk_exec_mesh_run(d, run, MESH_AHEAD); + for (int i = 0; i < n; i++) { + const p_draw_mesh_ind *f = run[i]; + VkMemSpan ip; + if (!vk_mem_resolve(f->ind, 12, &ip) || ip.offset % 4) break; // that draw reports it when the walk gets there + VK_MEM_USE(ip); + VkDeviceSize fa = 12ull * (mesh_ring_next++ % MESH_RING); + vk_gpu_gap_note(VK_GAP_MESH_RING, 12); + vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1); + VkBufferCopy fc = {ip.offset, fa, 4}; + vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc); + mesh_count_note(ex, &ip); + ex->recorded += 2; + mesh_ahead[mesh_ahead_n].d = f, mesh_ahead[mesh_ahead_n].ind = f->ind, mesh_ahead[mesh_ahead_n].at = fa; + mesh_ahead_n++; + } + mesh_ahead_brk = vk_exec_brk; + } VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, @@ -5052,7 +5110,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { c0_binst = 0; VK_MEM_USE(ip); VkDeviceSize at; - if (!mesh_ring_entry(ex, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); + if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; vk_exec_frame.draws++; static uint32_t n_logged;