diff --git a/runtime/host.h b/runtime/host.h index 3b5928db..7861672e 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -245,6 +245,7 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o int mem_lazy_armed(uint64_t a, uint64_t n); // Copies the ranges src[i] = {address, bytes} of guest memory whose pages read without a fault to dst[i], ok[i] set. int mem_copy_readable(uint8_t *const *dst, const uint64_t (*src)[2], int n, uint8_t *ok); +int mem_lazy_touches(uint64_t a, uint64_t n); // 0: the CPU can read [a, a + n) without a fault now void mem_lazy_window(int open); void mem_lazy_release(uint64_t a, uint64_t n); void mem_lazy_report(uint64_t frame, uint64_t skipped); diff --git a/runtime/memory.c b/runtime/memory.c index 9802157d..a1c9f7b4 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -518,8 +518,22 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o return k; } +// 0 when no page of [a, a + n) can be inaccessible right now (armed by the lazy write-back or probed), so the CPU can +// read it without a fault; 1 when one may be. +int mem_lazy_touches(uint64_t a, uint64_t n) { + if (!n) return 0; + if (a >= RT_ADDRESS_LIMIT || a + n > RT_ADDRESS_LIMIT || __atomic_load_n(&prb_on, __ATOMIC_RELAXED)) return 1; + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; + int any = 0; + for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = __atomic_load_n(&lzr[r], __ATOMIC_ACQUIRE); + if (!any) return 0; + int hit = 0; plat_mutex_lock(&lz_lock); + for (int i = 0; i < lz_n && !hit; i++) hit = lzs[i].lo < a + n && lzs[i].hi > a; plat_mutex_unlock(&lz_lock); + return hit; +} + // [a, a + n) is armed as a whole still. int mem_lazy_armed(uint64_t a, uint64_t n) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 320d7160..c2a71790 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -741,6 +741,9 @@ NVN_API(nvnQueueSubmitCommands) { // point). Copies of the data at submit made it worse, a mix of two frames' versions. // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits // after every submission, 0 never. + // Since the staged mirror uploads (vk_memory.c) the copies of guest memory a submission reads are taken while the + // replay records it, and the wait ends once the replay has submitted it (vk_sync_recorded_wait); the GPU's wait + // below remains for a submission whose GPU reads guest memory itself. // The wait (exec_wait, vk_sync.c vk_sync_exec_wait) ends when the GPU has run the submission's passes and the // results the game reads once it goes on are in its memory, the last frame's occlusion counts among them; the // write-backs and the completion thread's other side effects are not waited for. NX2NAVI_WB_SPLIT=0 makes it diff --git a/runtime/vk.h b/runtime/vk.h index 781f5355..522df9b2 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -347,6 +347,10 @@ void vk_mem_mirror_begin(void); int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); int vk_mem_mirror_pending(void); void vk_mem_mirror_prologue(VkCommandBuffer cb); +// Staged mirror uploads (vk_memory.c): the GPU of the submission being recorded reads guest memory itself (noted by +// vk_mem_note_live); vk_mem_take_live returns that and starts the next submission's note. +void vk_mem_note_live(void); +int vk_mem_take_live(void); void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over void vk_mem_lazy_handed(void); void vk_mem_lazy_decide(void); // once the driver is known (after vk_caps_driver): on or off, with one log line @@ -606,7 +610,8 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n); // vk_texture.c: s // set and brings it up to date. vk_pool_takes counts the takes whose sync left no slot behind (vk_exec_draw.c pool_sync); // once it has moved on by more than VK_POOL_SETS since a pool entry changed, every set holds the change (vk_texture.c // retire_sweep). It may count fewer takes than there were, which only delays that. -#define VK_POOL_SETS 4 +#define VK_POOL_SETS 8 // 4 before: with the game no longer held for the GPU after a frame's first submission, the + // replay waited for a set's GPU work at 15% of the switches uint64_t vk_pool_takes(void); // Texture lock held: 1 while the texture pool refresh looks its slots up (vk_exec_draw.c pool_sync_entries), 0 after. // Those lookups leave out a texture evicted by disuse, and are not counted as the texture being used. @@ -715,6 +720,7 @@ uint64_t vk_sync_reserve(void); // asynchronous replay: the value the int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted // The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet. int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns); +int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)); int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own // NX2NAVI_TRACE=perf (vk_sync.c "the executed timeline"): what a wait on that timeline left behind since the last call, // whether the completion thread's callback running now belongs to a value whose waiter went on already, and the diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 1893f6fc..1bc66186 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -302,9 +302,14 @@ int vk_replay_async(void) { } // With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's. -static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { +static int vk_be_gpu_wait(uint64_t value, uint64_t timeout_ns) { return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns); } +// The submit wait (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): the GPU's only where the submission reads guest memory on the +// GPU (vk_sync_recorded_wait). +static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { + return vk_sync_recorded_wait(value, timeout_ns, vk_be_gpu_wait); +} // The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end. int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index de6280f0..34b62d60 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2622,7 +2622,10 @@ void vk_mem_use(VkMemSpan *sp) { return; } // a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors) - if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) return; + if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) { + vk_mem_note_live(); // the GPU reads this pool's guest memory as it runs (vk_mem_take_live) + return; + } int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); mem_use_locked(sp); @@ -2638,7 +2641,10 @@ static void mem_use_locked(VkMemSpan *sp) { r->mstate = MS_NONE, r->retry_sub = 0; // refuse sets it again } if (r->mstate == MS_NONE) mirror_create(r); - if (r->mstate != MS_ON) return; + if (r->mstate != MS_ON) { // the GPU reads this pool's guest memory as it runs (staged mirror uploads) + vk_mem_note_live(); + return; + } if (r->acc_sub != msub) mirror_sync(r); r->vd_use_sub = msub; sp->buffer = r->mbuf; @@ -2646,7 +2652,11 @@ static void mem_use_locked(VkMemSpan *sp) { void vk_mem_pick(VkMemSpan *sp) { PoolRec *r = sp->pool; - if (!r || r->mstate != MS_ON) return; + if (!r) return; + if (r->mstate != MS_ON) { // guest memory itself, read by the GPU when it runs (vk_mem_take_live) + vk_mem_note_live(); + return; + } if (r->acc_sub != msub) { int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); @@ -3424,11 +3434,13 @@ static void hot_check(void) { if (mirror_hot < 0) { mirror_hot = p < 0 ? 0 : p > 1000 ? 1000 : p; if (mirror_hot) rt_log("mem: pools copied %d%% of their size a frame stay in system memory (NX2NAVI_MIRROR_HOT)", mirror_hot); + } if (!mirror_hot) return; uint64_t fr = mirror_frame; if (!hot_frame0 || fr < hot_frame0) { hot_frame0 = fr ? fr : 1; return; + } if (fr - hot_frame0 < HOT_FRAMES || npend || nwbl) return; // nsnl: the previous submission's, recorded already double frames = (double)(fr - hot_frame0); hot_frame0 = fr; @@ -3442,6 +3454,7 @@ static void hot_check(void) { r->hot_bytes = (uint64_t)per; // kept for the log line below r->hot_bytes = 0, i++; // !twin with a table record: the table's writes are in the mirror alone (mirror_evict) continue; + } mem_lazy_release(r->import_lo, r->import_hi - r->import_lo); // what only the mirror holds comes down first // as mirror_evict, under the exclusive lock (vk_pool_fini takes pools off the list under it), but for good: // MS_REFUSED with no retry (retry_sub 0), so neither vk_mem_use nor the budget retries mirror it again @@ -3456,6 +3469,7 @@ static void hot_check(void) { dead[nout] = r->mbuf, dead_a[nout] = r->malloc_; r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL; out[nout++] = r; + } plat_mutex_unlock(&pool_lock); for (int k = 0; k < nout; k++) { PoolRec *r = out[k]; @@ -3467,6 +3481,9 @@ static void hot_check(void) { if (++n_demoted <= 32) rt_log("mem: pool %#llx (%.1f MiB, %s) copied %.1f MiB a frame through its mirror: system memory from now on", (unsigned long long)r->lo, n / 1048576.0, f, per / 1048576.0); + } +} + void vk_mem_mirror_begin(void) { if (vk_mirror_on) hot_check(); if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions @@ -3516,28 +3533,51 @@ void vk_mem_mirror_begin(void) { int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; } -// NX2NAVI_MIRROR_CAPTURE_KB (default 0, off): the pages the CPU wrote since a VRAM mirror's last sync are copied out of -// guest memory by the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a -// host buffer the submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them -// from there into the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or -// more after the game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile, -// values meant for a later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest -// memory is still on the GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over -// the mirror's newer ones), and when its device address table writes reach guest memory as well (a twin: what they -// write is not recorded); so does a range with a page that cannot be read without a fault (mem_copy_readable: a -// lazily armed range). The whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU. +// NX2NAVI_MIRROR_CAPTURE_KB: the pages the CPU wrote since a VRAM mirror's last sync are copied out of guest memory by +// the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a host buffer the +// submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them from there into +// the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or more after the +// game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile, values meant for a +// later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest memory is still on the +// GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over the mirror's newer ones), +// and when its device address table writes reach guest memory as well (a twin: what they write is not recorded); so +// does a range with a page that cannot be read without a fault (mem_copy_readable: a lazily armed range). The +// whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU. +// Unset or 0 with NX2NAVI_MIRROR_STAGE=1 (the default here): CAPTURE_STAGE_KB. A submission whose prologue copied +// everything this way reads nothing of guest memory late, so its submit wait ends once the replay has submitted it +// (vk_sync_recorded_wait); one with a range left to the GPU-time copy, or a draw that reads a pool without a mirror, +// is noted (vk_mem_note_live) and its submit wait is the GPU's. The first form of this (0.8.0) copied while the replay +// recorded and left out neither the write-backs in flight nor the twins. +#define CAPTURE_STAGE_KB (64 << 10) +static volatile int32_t mirror_live; // the submission being recorded had the GPU read guest memory + +static int stage_on(void) { + static int on = -1; + if (on < 0) on = rt_knob("MIRROR_STAGE", "1") != 0; + return on; +} + static uint64_t capture_cap(void) { static int64_t cap = -1; if (cap < 0) { int kb = cfg_int("NX2NAVI_MIRROR_CAPTURE_KB"); + if (kb <= 0 && stage_on()) kb = CAPTURE_STAGE_KB; cap = kb > 0 ? (int64_t)kb << 10 : 0; if (cap) rt_log("mem: mirror pages the CPU wrote are copied out of guest memory when their part is handed over, up to " - "%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB)", kb); + "%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB, NX2NAVI_MIRROR_STAGE)", kb); } return (uint64_t)cap; } +void vk_mem_note_live(void) { __atomic_store_n(&mirror_live, 1, __ATOMIC_RELAXED); } + +int vk_mem_take_live(void) { + int live = __atomic_exchange_n(&mirror_live, 0, __ATOMIC_RELAXED); + if (!vk_mirror_on || shadow_on || !capture_cap()) live = 1; + return live; +} + static MRange *capm; // replay: mirror_capture's merged ranges static uint64_t *caps; // replay: the chunk offset of each, UINT64_MAX for one left to the GPU-time copy static uint8_t **capd, *capok; // replay: mem_copy_readable's destinations and results @@ -3619,6 +3659,7 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb) { mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); mirror_capture(cb); + if (npend) vk_mem_note_live(); // left to the GPU-time copy: the GPU reads guest memory late copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies); if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index dce3d334..abe93c1f 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -48,6 +48,10 @@ static PlatCond cv_work = PLAT_COND_INIT; // a submission, a callback or stop static PlatCond cv_done = PLAT_COND_INIT; // done advanced static uint64_t issued, done; // newest submitted value; newest value whose side effects ran static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued) +// Staged mirror uploads (vk_memory.c "staged mirror uploads"): per submission value, whether its GPU still reads guest +// memory itself (value << 1 | live), under st_lock. A value not found counts as live. +#define LIVE_RING 4096 +static uint64_t live_of[LIVE_RING]; static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; @@ -509,6 +513,7 @@ static void gw_wait(uint64_t value, uint64_t t0, uint64_t t1, uint64_t newest, u static void gw_log(uint64_t frame, int frames) { plat_mutex_lock(&gw_lock); uint64_t n = gw_n, ns = gw_ns, rep = gw_rep_ns, gpu = gw_gpu_ns, lead = gw_lead_ns, age = gw_age, before = gw_before, + lr = gw_lat_rep, lg = gw_lat_gpu, ln = gw_lat_n, mx = gw_max_ns, unk = gw_unknown, rn = gw_rec_n, gw_n = gw_ns = gw_rep_ns = gw_gpu_ns = gw_lead_ns = gw_age = gw_before = gw_lat_rep = gw_lat_gpu = gw_lat_n = 0; plat_mutex_unlock(&gw_lock); if (!n) { @@ -963,6 +968,7 @@ static VkCommandPool pre_pool; static CmdSlot *pre_slots; static int npre, pre_pair = -1; static VkCommandBuffer pre_cb; + VkCommandBuffer vk_sync_pre(void) { if (pre_cb) return pre_cb; VkCtx *c = vk_ctx(); @@ -1260,9 +1266,15 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { s->fence = sent ? fence : 0; vk_mem_note_submit(fence); gw_mark(fence, 1); + // after the last part's prologue: every staged copy of the submission is taken; a probe's submit leaves the note to + // the replay's submission it may have come in the middle of + int live = own ? 1 : vk_mem_take_live(); + int res = own ? 0x1f : vk_exec_take_results(); plat_mutex_lock(&st_lock); + live_of[fence & (LIVE_RING - 1)] = fence << 1 | (uint64_t)(live != 0); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); + plat_cond_broadcast(&cv_done); // vk_sync_recorded_wait plat_mutex_unlock(&st_lock); static unsigned subs; if (++subs % 4096 == 0) { @@ -1337,6 +1349,46 @@ static void results_run(uint64_t upto) { static int fence_wait_impl(uint64_t value, uint64_t timeout_ns); + +// The game's wait after a frame's first submission (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): until the replay has +// submitted value, which took every copy of guest memory the submission reads; then also until the GPU has run it when +// its GPU reads guest memory itself (vk_mem_take_live) or NX2NAVI_SUBMIT_FINISH_GPU=1. 1 when reached. +int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)) { + vk_sync_start(); + if (gpu < 0) { + gpu = rt_knob("SUBMIT_FINISH_GPU", "0") != 0; + rt_log("vk: the wait after a frame's first submission ends %s (NX2NAVI_SUBMIT_FINISH_GPU=%d)", + gpu ? "when the GPU has run it" : "when the replay has submitted it, or the GPU has run it where it reads guest " + "memory itself", gpu); + } + if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; + if (gpu) return gpu_wait(value, timeout_ns); + uint64_t w0 = plat_now_ns(), newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE); + uint64_t sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE); + int live = 1, ok = 1; + plat_mutex_lock(&st_lock); + if (value > issued && value > reserved) ok = 0; // never handed over + while (ok && issued < value && done < value) { + if (timeout_ns < (1ull << 62)) { + uint64_t el = plat_now_ns() - w0; + if (el >= timeout_ns) { ok = 0; break; } + plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el); + } else { + plat_cond_wait(&cv_done, &st_lock); + } + } + uint64_t w1 = plat_now_ns(); + if (w1 > w0) { + if (!live) gw_wait(value, w0, w1, newest > sent ? newest : sent, sent); // a GPU wait counts there + plat_mutex_lock(&gw_lock); + gw_rec_n++; + if (!live) gw_rec_done++; + plat_mutex_unlock(&gw_lock); + } + if (!ok) return 0; + return live ? gpu_wait(value, timeout_ns) : 1; +} + int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) { uint64_t p0 = perf_begin(); uint64_t newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE), sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE);