diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 9b51914f..22e32eff 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -640,6 +640,10 @@ # first submission ends when that submission's passes are through, without waiting for the copy back (3 to 4 ms of # every frame). 0: the copy back is part of the submission, and the wait covers it. # wb_split = 1 +# 1: what the game's shaders write into its memory is copied back from the video card once, and after that only when +# the game's CPU code touches it (most of it never is: 66 MiB a frame in Hyrule Field). 0: copied back at the end of +# every GPU submission, as before. +# wb_lazy = 1 # 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every # page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in # Kokiri Forest. diff --git a/runtime/cfg.c b/runtime/cfg.c index b8d051c8..48271760 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -64,7 +64,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", @@ -123,7 +123,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"WB_LAZY", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; @@ -244,7 +244,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", "1", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/host.h b/runtime/host.h index f86f7a6e..24365288 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -185,6 +185,14 @@ void mem_thaw(uint64_t a, uint64_t n); // makes the frozen r // before a kernel write (file read) into guest memory void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws); // NX2NAVI_WB_PROBE=1 (memory.c, "Probed regions"): who reads what the GPU wrote back into guest memory. +// NX2NAVI_WB_LAZY (memory.c, "Lazy write-back"): pages the GPU wrote that come down only when the CPU touches them. +int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size)); +uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off); +int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max); +int mem_lazy_armed(uint64_t a, uint64_t n); +void mem_lazy_release(uint64_t a, uint64_t n); +void mem_lazy_report(uint64_t frame, uint64_t skipped); +int mem_lazy_pick(uint64_t *addr, int max); // test: addresses of armed pages int mem_probe(uint64_t a, uint64_t n); // the 2 MiB regions inside [a, a+n) fault on their next access int mem_probing(void); // the probe is on void mem_probe_report(uint64_t frame); // logs the accesses since the last report diff --git a/runtime/memory.c b/runtime/memory.c index c6a7c02e..7016e1c7 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -108,9 +108,231 @@ static void unprobe_locked(uint64_t r) { plat_dec32(&nprb); } +// Lazy write-back (NX2NAVI_WB_LAZY, vk_memory.c "lazy write-back"). What a shader wrote into a mirrored pool used to be +// copied back into guest memory at the end of every submission, in case the CPU reads it. Ocarina's CPU does not read +// most of it: standing in Hyrule Field, 66 MiB a frame went back over PCIe, 7 to 9 ms of each frame's 13 to 15 ms of +// GPU time, and the write-back probe found no access to it in 40 s. So a range that went back once is armed here: its +// whole pages are made inaccessible (mem_lazy_arm), and while it stays armed the back end leaves its later writes in +// the mirror (mem_lazy_skip marks it stale). The first read or write of an armed page faults: lazy_fault has the +// back end copy the range's bytes down now (lz_fetch, a GPU copy after everything submitted so far), gives the pages +// back and runs the access again, and the range stays out for a while (lz_hot: 1 s, doubling to 32 s while it keeps +// being touched), written back every submission as before. Every thaw (before the kernel writes or reads guest +// memory, and before mapping changes) releases the armed ranges of its regions the same way. +// A region with armed pages is not frozen (freeze_locked wants every page read-write), and a frozen one is not armed. +typedef struct { uint64_t lo, hi, h0, h1; int64_t delta; uint8_t stale; } LzSeg; // pages [lo, hi); buffer offset = address + delta +static LzSeg *lzs; // frz_lock +static int lz_n, lz_cap; // frz_lock +static volatile uint8_t lzr[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // a segment reaches into the region +static uint32_t lz_hot[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 until which nothing is armed here +static uint8_t lz_back[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: seconds of the region's last hold +static volatile int32_t lz_on; +static int lz_want; // the setting, read in mem_init +static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size); +// frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time +static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns; + +static void lazy_flags_locked(uint64_t lo, uint64_t hi) { + for (uint64_t r = lo >> FRZ_SHIFT; r <= (hi - 1) >> FRZ_SHIFT; r++) { + int any = 0; + for (int j = 0; j < lz_n && !any; j++) + any = lzs[j].lo < ((r + 1) << FRZ_SHIFT) && lzs[j].hi > (r << FRZ_SHIFT); + __atomic_store_n(&lzr[r], (uint8_t)any, __ATOMIC_RELEASE); + } +} + +// frz_lock held exclusively. Segment i goes: its bytes come down first when the mirror's are newer, then its pages +// are read-write again. why: 0 a read fault, 1 a write fault (both hold the regions back), else a thaw or the mirror +// going away. +static void lazy_drop_locked(int i, int why) { + LzSeg s = lzs[i]; + if (s.stale && lz_fetch) { + uint64_t t0 = plat_now_ns(); + lz_fetch(s.h0, s.h1, (uint64_t)((int64_t)s.lo + s.delta), s.hi - s.lo); + lz_fetches++, lz_fetch_bytes += s.hi - s.lo, lz_fetch_ns += plat_now_ns() - t0; + } + for (uint64_t a = s.lo, b; a < s.hi; a = b) + if (plat_mem_state(a, s.hi, &b) == PLAT_PROT_NONE && !plat_mem_protect(a, b - a, PLAT_PROT_RW)) + rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)a, + (unsigned long long)(b - a)); + lzs[i] = lzs[--lz_n]; + lazy_flags_locked(s.lo, s.hi); + if (why > 1) { lz_other++; return; } + lz_faults[why]++; + uint32_t now = (uint32_t)plat_now_ms(); + for (uint64_t r = s.lo >> FRZ_SHIFT; r <= (s.hi - 1) >> FRZ_SHIFT; r++) { + int again = lz_hot[r] && (int32_t)(now - lz_hot[r]) < 60000; // touched again within a minute of its last hold + uint8_t b = again ? (uint8_t)(lz_back[r] < 32 ? lz_back[r] * 2 : 32) : 1; + lz_back[r] = b ? b : 1; + lz_hot[r] = (now + (uint32_t)lz_back[r] * 1000u) | 1; + } +} + +// frz_lock held exclusively: every segment that reaches into region r goes. +static void lazy_drop_region_locked(uint64_t r, int why) { + for (int i = lz_n; i-- > 0;) + if (lzs[i].lo < ((r + 1) << FRZ_SHIFT) && lzs[i].hi > (r << FRZ_SHIFT)) lazy_drop_locked(i, why); +} + +static int lazy_covered_locked(uint64_t lo, uint64_t hi) { + for (int i = 0; i < lz_n; i++) + if (lzs[i].lo <= lo && lzs[i].hi >= hi) return 1; + return 0; +} + +// frz_lock held exclusively: [x, e), inside one region, becomes inaccessible when every committed page of it is +// read-write, or inaccessible as part of a segment already; anything else (a read-only page, a hole, a page the game +// made inaccessible itself) leaves it as it is. +static int lazy_protect_locked(uint64_t x, uint64_t e) { + uint64_t run[16][2]; + int k = 0; + for (uint64_t a = x, b; a < e; a = b) { + int st = plat_mem_state(a, e, &b); + if (st == PLAT_PROT_RW) { + if (k == 16) return 0; + run[k][0] = a, run[k][1] = b, k++; + } else if (st != PLAT_PROT_NONE || !lazy_covered_locked(a, b)) return 0; + } + for (int i = 0; i < k; i++) + if (!plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_NONE)) { + while (i--) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW); + return 0; + } + return 1; +} + +// Arms the whole pages of [a, a + n), which the back end has just written back from the buffer h0 into h1 at offset +// off: they fault on their next access. Returns the bytes armed now. Not in a frozen, probed, held or thaw-held region. +uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off) { + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || a >= RT_ADDRESS_LIMIT || !watched(a, n)) return 0; + uint64_t lo = (a + PAGE - 1) & ~(PAGE - 1), hi = (a + n) & ~(PAGE - 1), armed = 0; + if (hi <= lo) return 0; + int64_t delta = (int64_t)off - (int64_t)a; + plat_mutex_lock(&frz_lock); + if (lazy_covered_locked(lo, hi)) { + plat_mutex_unlock(&frz_lock); + return 0; + } + uint32_t now = (uint32_t)plat_now_ms(); + for (uint64_t x = lo, e; x < hi; x = e) { + uint64_t r = x >> FRZ_SHIFT; + e = (r + 1) << FRZ_SHIFT; + if (e > hi) e = hi; + if (frz[r] || prb[r] || (lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0) || + (thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) || !lazy_protect_locked(x, e)) + continue; + uint64_t slo = x, shi = e; + uint8_t stale = 0; + for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer + if (lzs[j].h0 == h0 && lzs[j].delta == delta && lzs[j].lo <= shi && lzs[j].hi >= slo) { + if (lzs[j].lo < slo) slo = lzs[j].lo; + if (lzs[j].hi > shi) shi = lzs[j].hi; + stale |= lzs[j].stale; + lzs[j] = lzs[--lz_n]; + j = -1; + } + if (lz_n == lz_cap) { + lz_cap = lz_cap ? lz_cap * 2 : 256; + lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap); + if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges"); + } + lzs[lz_n++] = (LzSeg){slo, shi, h0, h1, delta, stale}; + __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE); + armed += e - x; + } + plat_mutex_unlock(&frz_lock); + return armed; +} + +// The armed parts of [a, a + n), sorted, into out (at most max); they count as stale from here on: the caller leaves +// the bytes it is about to write there in the mirror. Returns how many. +int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max) { + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || !n || a >= RT_ADDRESS_LIMIT || a + n > RT_ADDRESS_LIMIT) return 0; + int any = 0, k = 0; + for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = lzr[r]; + if (!any) return 0; + plat_mutex_lock(&frz_lock); + for (int i = 0; i < lz_n && k < max; i++) { + uint64_t lo = lzs[i].lo > a ? lzs[i].lo : a, hi = lzs[i].hi < a + n ? lzs[i].hi : a + n; + if (lo >= hi) continue; + lzs[i].stale = 1; + int at = k++; + for (; at > 0 && out[at - 1][0] > lo; at--) out[at][0] = out[at - 1][0], out[at][1] = out[at - 1][1]; + out[at][0] = lo, out[at][1] = hi; + } + plat_mutex_unlock(&frz_lock); + return k; +} + +// [a, a + n) is armed as a whole still. +int mem_lazy_armed(uint64_t a, uint64_t n) { + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; + plat_mutex_lock(&frz_lock); + int ok = lazy_covered_locked(a, a + n); + plat_mutex_unlock(&frz_lock); + return ok; +} + +// Releases what is armed in [a, a + n): before the buffers a fetch would read go away. +void mem_lazy_release(uint64_t a, uint64_t n) { + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || !n) return; + plat_mutex_lock(&frz_lock); + for (int i = lz_n; i-- > 0;) + if (lzs[i].lo < a + n && lzs[i].hi > a) lazy_drop_locked(i, 4); + plat_mutex_unlock(&frz_lock); +} + +// The back end is ready to fetch: arming starts. 0 when the setting is off (or the probe is on, which watches the +// same pages). +int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size)) { + if (!lz_want) return 0; + lz_fetch = fetch; + __atomic_store_n(&lz_on, 1, __ATOMIC_RELEASE); + return 1; +} + +static int lazy_fault(uint64_t a, int write) { + if (a >= RT_ADDRESS_LIMIT || !__atomic_load_n(&lzr[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE)) return 0; + int hit = 0; + plat_mutex_lock(&frz_lock); + for (int i = 0; i < lz_n && !hit; i++) + if (a >= lzs[i].lo && a < lzs[i].hi) lazy_drop_locked(i, write != 0), hit = 1; + plat_mutex_unlock(&frz_lock); + return hit; +} + +// Test (NX2NAVI_WB_LAZY_TOUCH): the first addresses of up to max armed segments, another set each call, for a caller +// that reads them to make the faults a game's access would. +int mem_lazy_pick(uint64_t *addr, int max) { + static int next; + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; + int k = 0; + plat_mutex_lock(&frz_lock); + for (; k < max && k < lz_n; k++) addr[k] = lzs[(next + k) % lz_n].lo; + next += k; + plat_mutex_unlock(&frz_lock); + return k; +} + +// One log line since the last one; skipped: the bytes the back end left in the mirror since then. +void mem_lazy_report(uint64_t frame, uint64_t skipped) { + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; + plat_mutex_lock(&frz_lock); + uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes, + fns = lz_fetch_ns; + int n = lz_n; + for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo; + lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = 0; + plat_mutex_unlock(&frz_lock); + rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the " + "mirror, %llu read and %llu write faults, %llu other releases, %llu fetches of %.1f MiB in %.2f ms", + (unsigned long long)frame, n, bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr, + (unsigned long long)fw, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6); +} + // frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here. static void thaw_locked(uint64_t r, int why) { if (why == 2) thaw_ms[r] = (uint32_t)plat_now_ms() | 1; + if (lzr[r]) lazy_drop_region_locked(r, why + 1); // not a fault on an armed page: no hold if (prb[r]) unprobe_locked(r); if (!frz[r]) return; // The read-only runs only: freeze_locked left reserved holes (heap pages given back) as they were. Every committed @@ -212,7 +434,8 @@ int mem_probe(uint64_t a, uint64_t n) { return armed; } -int mem_probing(void) { return __atomic_load_n(&prb_on, __ATOMIC_RELAXED); } +// Pages of guest memory may be inaccessible (the probe, or the lazy write-back): a kernel read of them has to thaw first. +int mem_probing(void) { return __atomic_load_n(&prb_on, __ATOMIC_RELAXED) || __atomic_load_n(&lz_on, __ATOMIC_RELAXED); } // The platform's access-fault hook while the probe is on: the first access of a probed region ends its probe and is // noted; 1 runs the access again. A read that faulted in a region another thread gave back meanwhile retries too; a @@ -234,6 +457,16 @@ static int probe_fault(uint64_t a, int write, uint64_t ip) { return plat_mem_state(a & ~(PAGE - 1), (a & ~(PAGE - 1)) + PAGE, &end) >= PLAT_PROT_READ; } +// The platform's access-fault hook: an armed page of the lazy write-back first, then the probe. A read that faulted on +// a page another thread gave back meanwhile runs again; a write goes on to thaw_fault. +static int access_fault(uint64_t a, int write, uint64_t ip) { + if (__atomic_load_n(&lz_on, __ATOMIC_RELAXED) && lazy_fault(a, write)) return 1; + if (__atomic_load_n(&prb_on, __ATOMIC_RELAXED)) return probe_fault(a, write, ip); + if (write || a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return 0; + uint64_t end; + return plat_mem_state(a & ~(PAGE - 1), (a & ~(PAGE - 1)) + PAGE, &end) >= PLAT_PROT_READ; +} + // One log block for the probe since the last one: how many regions were armed, how many probes a read or a write // ended, and the first accesses by instruction (an address inside the executable as nx2navi+offset, for the linker // map or nm). Nothing while the probe is off. @@ -348,8 +581,11 @@ void mem_init(void) { // depends on NX2NAVI_APP_POOL and on the resolution settings (resolution.c decides them once, on this first call). cfg_init(); if (cfg_bool("NX2NAVI_WB_PROBE")) { // a diagnosis: off, no access fault reaches probe_fault and nothing is probed - plat_mem_access_hook(probe_fault); + plat_mem_access_hook(access_fault); __atomic_store_n(&prb_on, 1, __ATOMIC_RELEASE); + } else if (cfg_bool("NX2NAVI_WB_LAZY")) { // armed only once the back end can fetch (mem_lazy_enable) + plat_mem_access_hook(access_fault); + lz_want = 1; } const char *pool = cfg_get("NX2NAVI_APP_POOL"); L = (RtGuestLayout){ diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 0135a0df..d2436cfa 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1149,9 +1149,17 @@ static void gpu_written(uint64_t addr, uint64_t size, int back) { } void vk_gpu_written(uint64_t addr, uint64_t size) { gpu_written(addr, size, 0); } +static int wb_lazy(void); + void vk_gpu_written_back(uint64_t addr, uint64_t size) { gpu_written(addr, size, 1); mem_probe(addr, size); // NX2NAVI_WB_PROBE: the bytes are in guest memory now; note who reads them next + VkMemSpan sp; + if (size && wb_lazy() && vk_mem_resolve(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on + const PoolRec *r = sp.pool; + if (r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo && addr + size <= r->import_hi) + mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo); + } } int vk_mem_mirrored(uint64_t addr, uint64_t size) { @@ -1566,6 +1574,7 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why) for (; n < nv && freed < need; n++) { PoolRec *r = v[n]; idle_for[n] = msub - r->acc_sub; + mem_lazy_release(r->import_lo, r->import_hi - r->import_lo); // what only the mirror holds comes down first mirror_forget(r); // off the list, mirror_gen bumped: no buffer use gets the mirror from here on r->mstate = MS_NONE, r->acc_sub = 0, r->retry_sub = 0; dead[n] = r->mbuf, dead_a[n] = r->malloc_; @@ -2102,7 +2111,109 @@ static void snap_free(void *arg) { // completion thread: the submission is don free(f); } +// ---- lazy write-back (NX2NAVI_WB_LAZY, default 1; memory.c "Lazy write-back") +// A range a shader wrote goes back to guest memory once and is then armed (vk_gpu_written_back, mem_lazy_arm). While it +// stays armed, gpu_wrote leaves the later writes of its pages in the mirror: no snapshot, no merge, no copy. When the +// CPU touches a page, memory.c calls lazy_fetch with the mirror and the imported buffer of the range: a copy of its +// own, sent after every submission handed over so far (the held ones first), and waited for. A submission that is +// still being recorded when that happens has already left its write out, so vk_mem_mirror_end copies the ranges it +// skipped that are no longer armed (lz_skip). +static VkCommandPool lz_pool; +static VkCommandBuffer lz_cb; +static VkFence lz_fence; +static volatile int32_t lz_state; // 0 not asked, 1 on, 2 off +static PlatMutex lz_init_lock = PLAT_MUTEX_INIT; +typedef struct { PoolRec *r; uint64_t off, size; } LzSkip; +static LzSkip *lz_skip; // replay: what this submission left in the mirror +static int lz_nskip, lz_cskip; +static uint64_t lz_skipped; // replay: bytes left in the mirror since the last log line + +static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, + VkAccessFlags2 da); + +// memory.c's fetch (frz_lock held there, so one at a time): [off, off + size) of the mirror h0 into the imported +// buffer h1, after all the GPU work handed over so far. +static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) { + VkCtx *c = vk_ctx(); + VkBufferCopy reg = {off, off, size}; + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkResetCommandPool(c->device, lz_pool, 0); + vkBeginCommandBuffer(lz_cb, &bi); + mbarrier(lz_cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + vkCmdCopyBuffer(lz_cb, (VkBuffer)(uintptr_t)h0, (VkBuffer)(uintptr_t)h1, 1, ®); + mbarrier(lz_cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + vkEndCommandBuffer(lz_cb); + VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &lz_cb}; + vk_queue_lock(); + vk_sync_flush_held(); // the held submissions wrote first + VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, lz_fence); + vk_queue_unlock(); + if (r == VK_SUCCESS) r = vkWaitForFences(c->device, 1, &lz_fence, VK_TRUE, 10000000000ull); + if (r != VK_SUCCESS) { + vk_device_fault_report(r); + rt_fatal("vk: bringing %llu bytes the GPU wrote down to guest memory failed (%d)", (unsigned long long)size, (int)r); + } + vkResetFences(c->device, 1, &lz_fence); +} + +static int wb_lazy(void) { + int s = __atomic_load_n(&lz_state, __ATOMIC_ACQUIRE); + if (s) return s == 1; + plat_mutex_lock(&lz_init_lock); + if (!lz_state) { + VkCtx *c = vk_ctx(); + int on = vk_mirror_on && c && cfg_bool("NX2NAVI_WB_LAZY"); + if (on) { + VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, + .queueFamilyIndex = c->gfx_family}; + VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + on = vkCreateCommandPool(c->device, &pi, NULL, &lz_pool) == VK_SUCCESS; + VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = lz_pool, + .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; + on = on && vkAllocateCommandBuffers(c->device, &ai, &lz_cb) == VK_SUCCESS && + vkCreateFence(c->device, &fi, NULL, &lz_fence) == VK_SUCCESS; + on = on && mem_lazy_enable(lazy_fetch); + } + rt_log("mem: what shaders write into mirrored pools %s (NX2NAVI_WB_LAZY)", + on ? "goes back to guest memory once and then only when the CPU touches it" + : "goes back to guest memory at the end of every submission"); + __atomic_store_n(&lz_state, on ? 1 : 2, __ATOMIC_RELEASE); + } + plat_mutex_unlock(&lz_init_lock); + return lz_state == 1; +} + +static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader); + static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { + PoolRec *r = sp->pool; + if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size || !wb_lazy()) { + gpu_wrote_range(sp, size, shader); + return; + } + uint64_t g0 = r->import_lo + sp->offset, g1 = g0 + size, run = g0, out[16][2]; + int k = mem_lazy_skip(g0, size, out, 16); + for (int i = 0; i <= k; i++) { + uint64_t lo = i < k ? out[i][0] : g1, hi = i < k ? out[i][1] : g1; + if (lo > run) { // not armed: written back as ever + VkMemSpan part = *sp; + part.offset = sp->offset + (run - g0); + gpu_wrote_range(&part, lo - run, shader); + } + if (i < k) { + if (lz_nskip == lz_cskip) grow((void **)&lz_skip, &lz_cskip, sizeof *lz_skip); + lz_skip[lz_nskip++] = (LzSkip){r, sp->offset + (lo - g0), hi - lo}; + lz_skipped += hi - lo; + } + run = hi; + } +} + +static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; r->vd_wrote_sub = msub, r->vd_wrote_n++; @@ -2371,6 +2482,12 @@ int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); } int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { if (!vk_mirror_on) return 0; int rec = 0; + // What this submission left in the mirror for a range that was armed then and is not now (the CPU touched it while + // the submission was recorded): its bytes go down after all, as a copy. + for (int i = 0; i < lz_nskip; i++) + if (!mem_lazy_armed(lz_skip[i].r->import_lo + lz_skip[i].off, lz_skip[i].size)) + range_add(&wbl, &nwbl, &cwbl, lz_skip[i].r, lz_skip[i].off, lz_skip[i].size); + lz_nskip = 0; if (nsnl) { // the merge reads the mirror the submission's shaders wrote and writes the imported buffers mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, @@ -2467,6 +2584,20 @@ static void mirror_log(uint64_t frame) { uint64_t fz, ff, ft, fo; mem_freeze_stats(&fz, &ff, &ft, &fo); mem_probe_report(frame); + mem_lazy_report(frame, lz_skipped); + lz_skipped = 0; + // NX2NAVI_WB_LAZY_TOUCH= (test, 0 off): at each of these lines this thread reads a byte of n armed ranges, as + // the game's CPU might, so the fault, the fetch and the hold run without waiting for the game to do it. + static int touch = -1; + if (touch < 0) touch = cfg_int("NX2NAVI_WB_LAZY_TOUCH"); + if (touch > 0) { + uint64_t at[16]; + int k = mem_lazy_pick(at, touch > 16 ? 16 : touch); + for (int i = 0; i < k; i++) { + volatile uint8_t v = *(volatile const uint8_t *)(uintptr_t)at[i]; + (void)v; + } + } if (rt_trace & RT_TRACE_PERF) rt_log("mem: write-watch frame %llu: per frame, every consumer: %.1f queries over %.1f MiB, %.3f ms (%.1f found " "%.0f written pages, %.3f ms); frozen %.0f MiB; since the last line: freezes %llu, thaws by a write %llu, by a file read "