diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 579f560b..076febae 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -976,3 +976,8 @@ # those second writes cost about 3 ms of GPU time a frame. 0 (default): every mirror sends them. A pool that needs them # and is left without shows as foliage stretched into columns or flickering. # gpa_twin_max_kb = 0 +# 1 (default): shaders that follow the game's pointers read a memory pool's new copy in video memory only once that +# copy has been filled; until then they read the game's own memory. 0: they read the new copy at once, as up to 0.9.2, +# where a card over its video memory budget could lose the device at an area load (a GPU read fault at the first byte +# of the new copy). +# gpa_late = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index 8075fbf4..0d843815 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -47,7 +47,7 @@ static const char *const known[] = { "DESC_TEMPLATE", "DEVICE_PROFILE", "DMABUF", "DMABUF_MB", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DRAW_CRUMBS", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "GPU_TIMELINE", "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", - "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", + "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_LATE", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_LIVE", "HEAP_LAZY", "HDR_PEAK_NITS", "HDR_UI_NITS", // retired: the HDR look's settings of earlier versions, now fixed (vk_hdr.c, shaders/hdr_tonemap.frag); read by // nothing, listed so that a file which still sets them logs no unknown key diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index 32cec945..9daf0a14 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -319,20 +319,20 @@ void vk_gpa_clear(uint64_t lo, uint64_t hi) { blocks_release(p0, p1); } -static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { - if (!vk_gpa_on() || !buf) return old_rec; - uint64_t dev = vk_gpa_device_address(buf); - uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; - plat_mutex_lock(&gpa_lock); - // the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too. - // NX2NAVI_GPA_TWIN=0 (diagnosis): no twin, the writes go to the mirror alone, as before patch 0034. +// NX2NAVI_GPA_TWIN=0 (diagnosis): no twin, the writes go to the mirror alone, as before patch 0034. +static int twins_on(void) { static int twins = -1; if (twins < 0) { const char *e = cfg_get("NX2NAVI_GPA_TWIN"); twins = !(e && !strcmp(e, "0")); } - uint64_t twin = old_rec && twins && keep_twin ? (uint64_t)G.cpu[REC_WORD + old_rec * REC_WORDS] | - (uint64_t)G.cpu[REC_WORD + old_rec * REC_WORDS + 1] << 32 - : dev - buf_lo; - uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0; - if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } + return twins; +} + +// gpa_lock held: the first address of record rec (the buffer its pages read). +static uint64_t rec_first(uint32_t rec) { + return (uint64_t)G.cpu[REC_WORD + rec * REC_WORDS] | (uint64_t)G.cpu[REC_WORD + rec * REC_WORDS + 1] << 32; +} + +// gpa_lock held: the pages of [p0, p1) that name old_rec (or none) name r from here on. +static void pages_move(uint64_t p0, uint64_t p1, uint32_t old_rec, uint32_t r) { for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 1); if (!w) { @@ -342,11 +342,35 @@ static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, } if (!G.cpu[w] || G.cpu[w] == old_rec) page_put(w, r); } +} + +static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { + if (!vk_gpa_on() || !buf) return old_rec; + uint64_t dev = vk_gpa_device_address(buf); + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + // the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too. + uint64_t twin = old_rec && twins_on() && keep_twin ? rec_first(old_rec) : dev - buf_lo; + uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0; + if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } + pages_move(p0, p1, old_rec, r); plat_mutex_unlock(&gpa_lock); if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); return r; } +uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { + if (!old_rec || !buf || !vk_gpa_on() || !twins_on()) return 0; + uint64_t dev = vk_gpa_device_address(buf); + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + uint32_t r = dev ? rec_alloc(rec_first(old_rec), dev - buf_lo) : 0; + if (r) pages_move(p0, p1, old_rec, r); + plat_mutex_unlock(&gpa_lock); + if (r) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); + return r; +} + uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { return remap(lo, hi, old_rec, buf, buf_lo, 1); } diff --git a/runtime/vk_gpa.h b/runtime/vk_gpa.h index f5e1d4e1..29df805e 100644 --- a/runtime/vk_gpa.h +++ b/runtime/vk_gpa.h @@ -24,5 +24,9 @@ uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, // The same without the twin: the pages name buf alone (a pool whose VRAM mirror is evicted goes back to its imported // buffer). uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); +// The pages of [lo, hi) that name old_rec keep reading old_rec's buffer and get buf as their twin: shaders read the +// pool where they did and write buf too (a VRAM mirror until its first copy has run, vk_memory.c mirror_create). +// Returns the new record, or 0 when there is none to give or twins are off (old_rec then stays as it is). +uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); void vk_gpa_report(void); void vk_gpa_shutdown(void); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 33308bc8..c1f1e447 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -61,6 +61,8 @@ typedef struct PoolRec { // VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only. uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED uint32_t gpa_rec; // POOL_IMPORT: its record in the device address table (vk_gpa.c), 0 none + uint64_t gpa_at; // pool_lock: the submission after which the table reads the mirror (gpa_switch); + // 0 when it does already, or there is no mirror struct GpaMap { uint64_t lo, hi; uint32_t rec; } *vmaps; // POOL_VIRT: its mapped ranges in the table int nvmaps, cvmaps; VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf @@ -2756,6 +2758,52 @@ static int twin_wanted(const PoolRec *r) { return !twin_max || (int64_t)(r->import_hi - r->import_lo) <= twin_max; } +// ---- when the device address table starts reading a new mirror (NX2NAVI_GPA_LATE, default 1; 0 = at once, as before) +// A mirror is made while the replay records a submission, and its first copy of the pool is in that submission's +// prologue. The table is memory the shaders read as they run, so with its pages moved to the mirror at once, every +// part already handed to the queue (the submissions before, and the earlier parts of this one) read the mirror through +// the table before that copy had run: memory nothing had written yet, and on a card over its budget a block the +// driver had not made resident for those parts. A GTX 1650 4 GB lost its device that way at area loads in every +// session: six reports of a GPU read fault at byte 0 of a mirror made in the faulting submission, each the first byte +// of a new 2 MiB-aligned block, with mirrors being made over the budget; with the table kept empty the same routes ran +// without a loss. +// Now the pages keep reading the imported buffer and name the mirror as their twin (vk_gpa_remap_writes): what shaders +// write through the table goes to both, so the mirror misses nothing, and what they read is guest memory, as before +// the mirror existed. Once the GPU has finished the submission whose prologue holds the copy, the pages move to the +// mirror with the imported buffer as the twin (gpa_switch, at the start of a later submission). In between, a shader +// that reads through the table what another wrote through a buffer binding sees guest memory, which gets such a write +// at the write-back; data written through the table itself, as the game's foliage, instance and light records are, +// is read where it was written. +static int n_gpa_wait; // pool_lock: mirrors whose pages still read the imported buffer (PoolRec.gpa_at) +static uint64_t n_gpa_late; // replay: pages moved to their mirror after its first copy, so far + +static int gpa_late(void) { + static int on = -1; + if (on < 0) { + on = rt_knob("GPA_LATE", "1") == 1; + rt_log("mem: the device address table reads a new VRAM mirror %s (NX2NAVI_GPA_LATE)", + on ? "once the mirror's first copy has run" : "at once, as before"); + } + return on; +} + +// Replay thread, at the start of a submission: the mirrors whose first copy the GPU has finished take the table's reads. +static void gpa_switch(void) { + if (!__atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) return; + uint64_t done = vk_sync_fence_done(); + plat_mutex_lock(&pool_lock); + for (int i = 0; i < nm && n_gpa_wait; i++) { + PoolRec *r = mlist[i]; + if (!r->gpa_at || done < r->gpa_at) continue; + r->gpa_at = 0, n_gpa_wait--; + if (!r->gpa_rec || !r->mbuf) continue; + r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) + : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + n_gpa_late++; + } + plat_mutex_unlock(&pool_lock); +} + // Replay thread: the pool's mirror on its first buffer use. static void mirror_create(PoolRec *r) { VkCtx *c = vk_ctx(); @@ -2848,16 +2896,28 @@ static void mirror_create(PoolRec *r) { r->acc_sub = msub; r->mstate = MS_ON; r->asked_room = 0; - // The device address table names the mirror from now on: the GPU's writes go there, and the sync copies the CPU's. - // With a twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. + // The device address table names the mirror: the GPU's writes go there, and the sync copies the CPU's. With a + // twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. Reads + // move to the mirror once its first copy has run (gpa_late, below). r->twin = (uint8_t)twin_wanted(r); if (!r->twin) { plat_inc64(&m_untwinned_n); plat_add64(&m_untwinned_bytes, (int64_t)n); } - if (r->gpa_rec) - r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) - : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + if (r->gpa_rec) { + plat_mutex_lock(&pool_lock); + uint32_t late = gpa_late() ? vk_gpa_remap_writes(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) : 0; + if (late) { + uint64_t f = vk_exec_recording_fence(); + r->gpa_rec = late; + r->gpa_at = f ? f : vk_sync_issued() + 1; // the submission being recorded: its prologue holds the copy + n_gpa_wait++; + } else { + r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) + : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); + } + plat_mutex_unlock(&pool_lock); + } range_add(&pend, &npend, &cpend, r, 0, n); mrep.first_n++, mrep.first_bytes += n; plat_inc64(&m_live_n); @@ -2866,6 +2926,7 @@ static void mirror_create(PoolRec *r) { static void mirror_forget(PoolRec *r) { __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); + if (r->gpa_at) r->gpa_at = 0, n_gpa_wait--; // the caller takes the table's pages off the mirror for (int i = 0; i < nm; i++) if (mlist[i] == r) { memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist); @@ -3932,7 +3993,7 @@ static void hot_check(void) { } void vk_mem_mirror_begin(void) { - if (vk_mirror_on) hot_check(); + if (vk_mirror_on) gpa_switch(), hot_check(); if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions if (!freeze_idle) return; plat_mutex_lock(&gen_lock); @@ -4387,6 +4448,10 @@ static void mirror_log(uint64_t frame) { "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; flushed and not copied (write-watched) %.3f MiB%s", (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M, a.flush_skipped / f / M, capl); + if (n_gpa_late || __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) + rt_log("mem: mirror frame %llu: the device address table took up %llu mirrors after their first copy so far, %d " + "wait for theirs (NX2NAVI_GPA_LATE)", (unsigned long long)frame, (unsigned long long)n_gpa_late, + __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)); } // After vram_log: every heap's usage as the driver reports it (VK_EXT_memory_budget) beside VMA's share of it, VMA by diff --git a/tests/gpa_check.c b/tests/gpa_check.c index 2d194db0..7bda3d39 100644 --- a/tests/gpa_check.c +++ b/tests/gpa_check.c @@ -125,6 +125,29 @@ int main(void) { CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "a plain page is its own twin"); gpu_done(); + // a new mirror takes the writes first and the reads once its first copy has run (vk_memory.c mirror_create, + // gpa_switch): until then the pages read the imported buffer and the mirror is their twin + VkBuffer m1 = new_buf(); + uint32_t w0 = vk_gpa_remap_writes(a0, a1, r0, m1, a0); + CHECK(w0 && w0 != r0, "a new mirror gets a record for the writes"); + expect(a0, a1, b0, a0, "a new mirror before its first copy: reads stay on the imported buffer"); + CHECK(read_through(a0 + 4096, 1) == dev_of(m1) + 4096 && read_through(a1 - 4096, 1) == dev_of(m1) + (a1 - 4096 - a0), + "a new mirror before its first copy is the twin, which takes the writes"); + gpu_done(); + r0 = vk_gpa_remap(a0, a1, w0, m1, a0); + expect(a0, a1, m1, a0, "a new mirror after its first copy: reads move to it"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "and the imported buffer is the twin again"); + // evicted while it still waits for its first copy: the pages go back to the imported buffer alone + r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0); + gpu_done(); + VkBuffer m2 = new_buf(); + w0 = vk_gpa_remap_writes(a0, a1, r0, m2, a0); + r0 = vk_gpa_remap_plain(a0, a1, w0, b0, a0); + expect(a0, a1, b0, a0, "a mirror gone before its first copy"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "leaves the imported buffer as its own twin"); + CHECK(!vk_gpa_remap_writes(a0, a1, 0, m2, a0), "a pool without a record gets none for a mirror's writes"); + gpu_done(); + // an unmapped range reads the zero buffer at once, and its blocks serve no other range while the GPU may still read // the old L1 entries uint32_t old_blk[3] = {block_of(a0), block_of(a0 + 16 * MiB), block_of(a0 + 32 * MiB)};