diff --git a/docs/README-BuildKit.md b/docs/README-BuildKit.md index 6328d01e..84dbb72b 100644 --- a/docs/README-BuildKit.md +++ b/docs/README-BuildKit.md @@ -95,8 +95,7 @@ With 16 GB of RAM, add `--jobs 4`: the compiles run beside the 26 GB RomFS image processor can fill memory and swap until the kernel stops processes of the desktop session (about 10 minutes on a 12-thread Ryzen with 4 jobs). Do not play or prepare shaders while it builds. -On NVIDIA, start the game with `NX2NAVI_SHADOW=0` if the 3D world stays black while the HUD draws (issue #72). The -profile, logs and settings are in the `--out` folder as on Windows. +The profile, logs and settings are in the `--out` folder as on Windows. On NVIDIA the driver keeps the shaders it compiled in `nvidia-cache` beside `ocarina`, not in `~/.cache/nvidia`, where one run adds at most 1 GiB while the preparation at the end of the build makes about 1.5 GB. The game makes and sets diff --git a/docs/building.md b/docs/building.md index 47243cf7..bf90059d 100644 --- a/docs/building.md +++ b/docs/building.md @@ -245,8 +245,11 @@ Memory: on Windows the GPU imports guest memory and `GetWriteWatch` tracks the C re-protecting imported memory invalidates the driver's userptr buffer on every submit (about 1.3 ms per 16 MiB on amdgpu), and importing pins the pages, which drops their write protection. Each CPU-visible pool therefore gets a host-visible copy (shadow storage, `runtime/vk_memory.c`): the pages the CPU wrote go in before each submit, and the GPU's -writes come back at its fence. In play that costs about 0.2 ms a frame and doubles the memory of those pools (about -1.1 GB). `NX2NAVI_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as +writes come back at its fence. Pages both sides write keep a base copy, so each direction moves only its own bytes; +a VRAM mirror's write-back merge marks the pages it changed, and only those are compared at the fence; a lookup that +reads guest memory itself (the uniform snapshot, `vk_mem_lookup`) does not queue its pool for the copy-in. In Kokiri +Forest (1.1.1, RTX 4090) that costs about 1 ms a frame (copy-in 0.5 ms, write-back 0.6 ms), and the copies hold about +5 GB. `NX2NAVI_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as well: the shadow copies are chosen for the measured cost above, not for a missing extension. The Deck's GPU (Van Gogh) has no native ASTC, so its textures go through the compute decoder (`runtime/vk_astc.c`, diff --git a/runtime/shaders/wb_merge.comp b/runtime/shaders/wb_merge.comp index 493a0325..16ec6516 100644 --- a/runtime/shaders/wb_merge.comp +++ b/runtime/shaders/wb_merge.comp @@ -4,12 +4,16 @@ // submission's GPU work, so words the GPU did not change keep what the CPU wrote meanwhile. // One invocation per word of the listed ranges; ranges are sorted by their first invocation (x: word offset in the // mirror and imported bindings, y: word offset in the snapshot binding, z: words, w: first invocation). +// With track set (shadow storage) a word that changed also marks its 4 KiB page of guest memory in Pages: page0 is the +// bindings' start within its page, so the CPU's write-back at the fence reads only the marked pages of the shadow +// (vk_memory.c, shadow_wb_cb). layout(local_size_x = 64) in; layout(std430, set = 0, binding = 0) readonly buffer Post { uint post[]; }; layout(std430, set = 0, binding = 1) readonly buffer Pre { uint pre[]; }; layout(std430, set = 0, binding = 2) buffer Dst { uint dst[]; }; layout(std430, set = 0, binding = 3) readonly buffer Ranges { uvec4 rng[]; }; -layout(push_constant) uniform Push { uint nranges, total; } pc; +layout(std430, set = 0, binding = 4) writeonly buffer Pages { uint pages[]; }; +layout(push_constant) uniform Push { uint nranges, total, page0, track; } pc; void main() { uint i = gl_GlobalInvocationID.x; @@ -23,5 +27,8 @@ void main() { uvec4 r = rng[lo]; uint k = i - r.w; uint v = post[r.x + k]; - if (v != pre[r.y + k]) dst[r.x + k] = v; + if (v != pre[r.y + k]) { + dst[r.x + k] = v; + if (pc.track != 0u) pages[(pc.page0 + (r.x + k) * 4u) >> 12] = 1u; + } } diff --git a/runtime/vk.h b/runtime/vk.h index daab4870..554bfb00 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -358,6 +358,9 @@ void vk_headroom_stop(void); // before vmaDestroyAllocator uint64_t vk_headroom_held(void); // bytes the spare blocks' anchors hold VkDeviceSize vk_vma_block_size(void); // VMA's block size on heaps over 1 GiB (NX2NAVI_VMA_BLOCK_MB) int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed +// vk_mem_resolve for a caller that reads guest memory itself or only asks what backs it: with shadow storage the pool +// is not copied in for it (vk_memory.c, mem_resolve). +int vk_mem_lookup(uint64_t addr, uint64_t size, VkMemSpan *out); // Changes whenever vk_mem_resolve plus VK_MEM_USE may give another span for an address: a pool added, finalized or // mapped, a mirror created, refused or forgotten, a submission started. uint64_t vk_mem_gen(void); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a0342f1c..93a68771 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1248,7 +1248,7 @@ static uint64_t ubo_ptr(const VkExec *ex, int st, const GfxShaderRes *r) { uint64_t off = (uint64_t)r->handle * 4, a = 0; VkMemSpan sp; if (!u->addr || u->size < off + 8 || upd_read(ex, u->addr + off, &a, 8)) return a; - if (vk_mem_resolve(u->addr + off, 8, &sp)) memcpy(&a, (const void *)(uintptr_t)(u->addr + off), 8); + if (vk_mem_lookup(u->addr + off, 8, &sp)) memcpy(&a, (const void *)(uintptr_t)(u->addr + off), 8); return a; } @@ -1332,7 +1332,7 @@ static uint64_t ptr_addr(const VkExec *ex, int st, const GfxShaderRes *r) { uint64_t off = r->ptr - 1u, a = 0; VkMemSpan sp; if (!b || !b->addr || (b->size && b->size < off + 8) || upd_read(ex, b->addr + off, &a, 8)) return a; - if (vk_mem_resolve(b->addr + off, 8, &sp)) memcpy(&a, (const void *)(uintptr_t)(b->addr + off), 8); + if (vk_mem_lookup(b->addr + off, 8, &sp)) memcpy(&a, (const void *)(uintptr_t)(b->addr + off), 8); return a; } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 091cae66..fc2f8e6d 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -82,6 +82,10 @@ typedef struct PoolRec { uint8_t *smap; // its mapping; offsets as in buf (from import_lo) uint32_t sgen; // gen_lock: every page write stamped up to this generation is in the shadow volatile uint8_t stouched; // on the touched list: the GPU may use it in the submission being recorded + // shadow_lock: base copies of the pages a noted GPU write covers (merge_page), open addressing by page address; + // csb a power of two, page 0 an empty slot + struct SBase { uint64_t page; uint8_t *base; } *sb; + uint32_t nsb, csb; // NX2NAVI_TRACE=perf per-pool mirror traffic since the last mirror_pool_log, replay thread (the log reads and clears // them without a lock: a count lost to the race only blurs a diagnostic) uint64_t st_pages, st_flush, st_wb, st_merge; // bytes: written pages and flush ranges copied up, GPU writes back @@ -260,6 +264,8 @@ static void destroy_rec(VkCtx *c, PoolRec *r) { if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL); if (r->mem) vkFreeMemory(c->device, r->mem, NULL); if (r->alloc) vmaFreeMemory(c->vma, r->alloc); + for (uint32_t i = 0; i < r->csb; i++) free(r->sb[i].base); // shadow base copies shadow_forget left (shutdown) + free(r->sb); free(r->pt); free(r); } @@ -461,9 +467,12 @@ static int resolve_check_on(void) { return on; } -static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out); +static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touch); -int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { +// touch: the GPU may read the pool's buffer in the submission being recorded, so its shadow goes on the touched list +// (shadow storage). A lookup (vk_mem_lookup) leaves it off: the caller reads guest memory itself or only asks what +// backs the address. A pool in the device address table is touched either way, as shaders reach it through pointers. +static int mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) { memset(out, 0, sizeof *out); uint64_t g = __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE), end = addr + size; const ResolveHit *h = rhit_slot(addr); @@ -475,19 +484,23 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { out->cls = h->cls; out->pool = h->pool; if (h->alloc) out->alloc_offset = out->offset; - if (shadow_on && h->pool && !((PoolRec *)h->pool)->stouched) shadow_touch(h->pool); + if (shadow_on && h->pool && !((PoolRec *)h->pool)->stouched && (touch || ((PoolRec *)h->pool)->gpa_rec)) + shadow_touch(h->pool); if (resolve_check_on()) { VkMemSpan s; - if (!resolve_locked(addr, size, &s) || memcmp(&s, out, sizeof s)) + if (!resolve_locked(addr, size, &s, touch) || memcmp(&s, out, sizeof s)) rt_fatal("vk: resolve cache of 0x%llx differs from the pool index", (unsigned long long)addr); } return 1; } - return resolve_locked(addr, size, out); + return resolve_locked(addr, size, out, touch); } +int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { return mem_resolve(addr, size, out, 1); } +int vk_mem_lookup(uint64_t addr, uint64_t size, VkMemSpan *out) { return mem_resolve(addr, size, out, 0); } + // The pool index lookup under the lock; a non-overlapped import or physical pool goes into the address's cache slot. -static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) { +static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) { memset(out, 0, sizeof *out); plat_mutex_lock_shared(&pool_lock); const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)addr, size); @@ -509,7 +522,7 @@ static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) { out->buffer = r->buf; out->offset = addr - r->import_lo; out->avail = r->import_hi - addr; - if (r->smap && !r->stouched) shadow_touch(r); + if (r->smap && !r->stouched && (touch || r->gpa_rec)) shadow_touch(r); ok = 1; } else if (r && r->kind == POOL_PHYS) { out->buffer = r->buf; @@ -545,7 +558,7 @@ uint64_t vk_mem_gen(void) { int vk_mem_class_of(uint64_t addr) { VkMemSpan m; - return vk_mem_resolve(addr, 1, &m) ? m.cls : VK_MCLS_NONE; + return vk_mem_lookup(addr, 1, &m) ? m.cls : VK_MCLS_NONE; } // ---------------------------------------------------------------- census @@ -968,17 +981,26 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { // - CPU -> shadow: right before each submit (vk_mem_shadow_sync), the pools the replay resolved since the last one are // queried for written pages; pages stamped after a pool's sgen are copied in by the CPU. // - GPU -> guest: the replay notes each GPU write into CPU-visible memory at record time (vk_mem_gpu_pending) and -// reports it at the fence (vk_gpu_written), which copies those bytes back into guest memory. Until then the CPU -// copies skip them: a page the CPU and the GPU both write keeps the GPU's bytes. +// reports it at the fence (vk_gpu_written), which copies the GPU's bytes back into guest memory. +// - Pages both write: a GPU write is noted with the whole range its binding names, and the CPU writes other fields of the +// same records meanwhile. Each page such a range covers keeps a base copy from its first note on, the bytes guest +// memory and the shadow last agreed on. A copy-in takes only the bytes where guest memory differs from the base (the +// CPU's writes) and the write-back at the fence only those where the shadow differs from it (the GPU's), and either +// moves the base along (merge_page). A byte both changed takes the CPU's value when a copy-in comes first and the +// GPU's when the fence does. Whole-range copies both ways lost the CPU's bytes: Ocarina writes a lighting parameter +// into a buffer whose binding its compute passes write every frame, the GPU only ever read zero there, the lighting +// coefficients and then the volumetric fog came out NaN, and the fog pass drew the 3D scene black. // NX2NAVI_SHADOW=1/0 forces the mode on or off; the default is on for Linux and Wine. -typedef struct { uint64_t lo, hi; } SRange; +#define SB_PAGE 4096ull static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock static PoolRec **stouch; // shadow_lock: pools resolved since the last sync static int nstouch, cstouch; -static SRange *spend; // shadow_lock: GPU writes recorded and not copied back yet -static int nspend, cspend; +static uint64_t s_sb_pages; // shadow_lock: base copies kept (sbase_make) static int64_t s_live_n, s_live_bytes; // shadow_lock static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks; // shadow_lock: since the last log +static uint64_t s_wb_seen, s_wb_ticks; // shadow_lock: bytes the fence write-backs compared, their time +static uint64_t s_q_ticks; // shadow_lock: the copy-ins' write-watch queries +static int wb_no_reup(void); static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) { r->import_lo = addr & ~0xfffull; @@ -1020,47 +1042,175 @@ static void shadow_touch(PoolRec *r) { plat_mutex_unlock(&shadow_lock); } +// ---- base copies of the pages both write ("Pages both write" above). shadow_lock held throughout. +static uint32_t sbase_hash(uint64_t pg) { return (uint32_t)((pg >> 12) * 0x9E3779B97F4A7C15ull >> 32); } + +// The base copy of guest page pg of r, NULL when it has none. +static uint8_t *sbase_find(const PoolRec *r, uint64_t pg) { + if (!r->nsb) return NULL; + for (uint32_t i = sbase_hash(pg) & (r->csb - 1);; i = (i + 1) & (r->csb - 1)) { + if (r->sb[i].page == pg) return r->sb[i].base; + if (!r->sb[i].page) return NULL; + } +} + +static void sbase_insert(struct SBase *t, uint32_t cap, uint64_t pg, uint8_t *base) { + uint32_t i = sbase_hash(pg) & (cap - 1); + while (t[i].page) i = (i + 1) & (cap - 1); + t[i].page = pg, t[i].base = base; +} + +// Page pg of r gets a base copy unless it has one. Taken from the shadow: no GPU write was noted on the page before, so +// the shadow holds what guest memory held at its last copy-in, and guest memory differs from it by the CPU's writes since. +static void sbase_make(PoolRec *r, uint64_t pg) { + if (sbase_find(r, pg)) return; + if ((r->nsb + 1) * 4 > r->csb * 3) { // at most three quarters full + uint32_t cap = r->csb ? r->csb * 2 : 16; + struct SBase *t = calloc(cap, sizeof *t); + if (!t) rt_fatal("vk: out of memory for the shadow's base copies"); + for (uint32_t i = 0; i < r->csb; i++) + if (r->sb[i].page) sbase_insert(t, cap, r->sb[i].page, r->sb[i].base); + free(r->sb); + r->sb = t, r->csb = cap; + } + uint8_t *b = malloc(SB_PAGE); + if (!b) rt_fatal("vk: out of memory for the shadow's base copies"); + memcpy(b, r->smap + (pg - r->import_lo), SB_PAGE); + sbase_insert(r->sb, r->csb, pg, b); + r->nsb++; + s_sb_pages++; +} + +static void sbase_free(PoolRec *r) { + for (uint32_t i = 0; i < r->csb; i++) free(r->sb[i].base); + free(r->sb); + s_sb_pages -= r->nsb; + r->sb = NULL, r->nsb = r->csb = 0; +} + +// 0xff in each byte of x that is not zero, 0 in the others. +static inline uint64_t byte_mask(uint64_t x) { + x |= x >> 4, x |= x >> 2, x |= x >> 1; + return (x & 0x0101010101010101ull) * 0xff; +} + +// The bytes of v that m selects into dst (little-endian), the other bytes of dst left as they are: the other side may +// be writing them at the same time. +static inline void store_bytes(uint8_t *dst, uint64_t v, uint64_t m) { + if (m == ~0ull) { + memcpy(dst, &v, 8); + return; + } + for (int k = 0; k < 8; k++) + if (m >> (8 * k) & 0xff) dst[k] = (uint8_t)(v >> (8 * k)); +} + +// n bytes of one page with base copy b: the bytes where src differs from the base go into dst and into the base. src +// and dst are guest memory and the shadow, one way or the other; each src byte that moves is read once, as its side +// may be writing it. Returns the bytes moved. +static uint64_t merge_page(uint8_t *dst, const uint8_t *src, uint8_t *b, uint64_t n) { + uint64_t moved = 0, i = 0; + for (; i < n && ((uintptr_t)(src + i) & 7); i++) { + uint8_t v = ((const volatile uint8_t *)src)[i]; + if (v != b[i]) dst[i] = b[i] = v, moved++; + } + for (; i + 8 <= n; i += 8) { + // Most of a page is unchanged: a 64-byte block equal to its base is passed over with one vector compare. A + // byte its side writes after the compare is a later write, which the next copy-in or write-back takes. + if (!((uintptr_t)(src + i) & 63) && i + 64 <= n) { + uint64_t x = 0; + for (int k = 0; k < 64; k += 8) { + uint64_t sv, ov; + memcpy(&sv, src + i + k, 8), memcpy(&ov, b + i + k, 8); + x |= sv ^ ov; + } + if (!x) { + i += 56; + continue; + } + } + uint64_t s = *(const volatile uint64_t *)(src + i), o; + memcpy(&o, b + i, 8); + uint64_t m = byte_mask(s ^ o); + if (!m) continue; + store_bytes(dst + i, s, m); + memcpy(b + i, &s, 8); // the bytes m leaves out are equal in both + moved += (uint64_t)__builtin_popcountll(m) / 8; + } + for (; i < n; i++) { + uint8_t v = ((const volatile uint8_t *)src)[i]; + if (v != b[i]) dst[i] = b[i] = v, moved++; + } + return moved; +} + // vk_pool_fini, after the pool left the index: off the touched list before its record goes. static void shadow_forget(PoolRec *r) { plat_mutex_lock(&shadow_lock); for (int i = 0; i < nstouch; i++) if (stouch[i] == r) { stouch[i] = stouch[--nstouch]; break; } s_live_n--, s_live_bytes -= (int64_t)(r->import_hi - r->import_lo); + sbase_free(r); plat_mutex_unlock(&shadow_lock); } -// shadow_lock held: copies guest [a, b) into the shadow, leaving out the bytes of GPU writes not copied back yet -// (spend from index i on). -static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b, int i) { - for (; i < nspend; i++) - if (spend[i].lo < b && spend[i].hi > a) { - if (spend[i].lo > a) shadow_copy(r, a, spend[i].lo, i + 1); - if (spend[i].hi < b) shadow_copy(r, spend[i].hi, b, i + 1); - return; +// shadow_lock held: copies guest [a, b) into the shadow; of a page with a base copy only the CPU's bytes (merge_page). +static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b) { + while (a < b) { + uint64_t e = b; + uint8_t *base = NULL; + if (r->nsb) { // the plain run up to the next page with a base copy + for (e = a; e < b; e = (e & ~(SB_PAGE - 1)) + SB_PAGE) + if ((base = sbase_find(r, e & ~(SB_PAGE - 1)))) break; + if (e > b) e = b; + if (e == a) { // a starts on such a page: its part of [a, b) + uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < b ? pg + SB_PAGE : b; + s_copied += merge_page(r->smap + (a - r->import_lo), (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a); + a = pe; + continue; + } } - memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, b - a); - s_copied += b - a; + memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, e - a); + s_copied += e - a; + a = e; + } } -// shadow_lock and gen_lock held: stamps the pool's written pages and copies those newer than its shadow. -static void shadow_sync_pool(PoolRec *r) { +// shadow_lock and gen_lock held: stamps the pool's written pages and copies those newer than its shadow. The pool's +// regions are queried through the write-watch snapshots of the guest submission (snap_regions, epoch), which the +// texture checks and the mirror syncs share: each 2 MiB region at most once per guest submission. A query of every +// touched pool at every part's submit was about 380 write-watch calls and 10 ms a frame of the submitting thread in +// Kokiri Forest, while the GPU waited for the next part (RTX 4090, 2026-11-09). A page the CPU writes after its +// region's snapshot goes in at the next guest submission's, as it does into a VRAM mirror. +// cpu (wb_no_reup): the pages are read from the tables without the stamps of a mirror's write-back (stamp_page_as), +// since that write-back went through the shadow, which holds those bytes already; with the other tables every storage +// buffer the game's compute shaders wrote was compared again at the next copy-in. +static void shadow_sync_pool(PoolRec *r, uint32_t epoch, int cpu) { uint64_t lo = r->import_lo, hi = r->import_hi; - uint32_t g = ++gen_now; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; int tracked = 1; - for (uint64_t a = lo; a < hi;) { // one query per run of regions that are not frozen - uint64_t b = (a | ((1ull << REG_SHIFT) - 1)) + 1; - if (b > hi) b = hi; - if (mem_frozen(a)) { a = b; continue; } - while (b < hi && !mem_frozen(b)) b = b + (1ull << REG_SHIFT) < hi ? b + (1ull << REG_SHIFT) : hi; - s_queries++; - if (!query_stamp(a, b, g)) tracked = 0; - a = b; + uint64_t t0 = qpc(); + if (epoch && r1 < SNAP_REGIONS) { + s_queries += snap_regions(r0, r1, epoch); + for (uint64_t k = r0; k <= r1 && tracked; k++) tracked = !snap_untracked[k]; + } else { // outside the snapshot regions, or before the first guest submission: one query per unfrozen run + uint32_t g = ++gen_now; + for (uint64_t a = lo; a < hi;) { + uint64_t b = (a | ((1ull << REG_SHIFT) - 1)) + 1; + if (b > hi) b = hi; + if (mem_frozen(a)) { a = b; continue; } + while (b < hi && !mem_frozen(b)) b = b + (1ull << REG_SHIFT) < hi ? b + (1ull << REG_SHIFT) : hi; + s_queries++; + if (!query_stamp(a, b, g)) tracked = 0; + a = b; + } } + s_q_ticks += qpc() - t0; uint32_t since = r->sgen; r->sgen = gen_now; if (!tracked) { // nothing tells which pages changed: all of them s_untracked++; - shadow_copy(r, lo, hi, 0); + shadow_copy(r, lo, hi); return; } uint64_t run = 0; @@ -1068,27 +1218,29 @@ static void shadow_sync_pool(PoolRec *r) { uint64_t a = k << REG_SHIFT, b = a + (1ull << REG_SHIFT); if (a < lo) a = lo; if (b > hi) b = hi; - const uint32_t *ch = gen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk - if (reg_max[k] <= since || !ch) { - if (run) shadow_copy(r, run, a, 0), run = 0; + const uint32_t *ch = (cpu ? cgen_chunk : gen_chunk)[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk + if ((cpu ? creg_max : reg_max)[k] <= since || !ch) { + if (run) shadow_copy(r, run, a), run = 0; continue; } for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) { int dirty = ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since; if (dirty && !run) run = x; - else if (!dirty && run) shadow_copy(r, run, x, 0), run = 0; + else if (!dirty && run) shadow_copy(r, run, x), run = 0; } } - if (run) shadow_copy(r, run, hi, 0); + if (run) shadow_copy(r, run, hi); } void vk_mem_shadow_sync(void) { if (!shadow_on) return; uint64_t t0 = qpc(); + uint32_t epoch = vk_texture_epoch(); + int cpu = vk_mirror_on && wb_no_reup(); // before the locks: its first call logs plat_mutex_lock(&shadow_lock); if (nstouch) { plat_mutex_lock(&gen_lock); - for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i]), stouch[i]->stouched = 0; + for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i], epoch, cpu), stouch[i]->stouched = 0; plat_mutex_unlock(&gen_lock); s_syncs += (uint64_t)nstouch; nstouch = 0; @@ -1097,12 +1249,18 @@ void vk_mem_shadow_sync(void) { plat_mutex_unlock(&shadow_lock); } +// Record time: the GPU may write [addr, addr+size) of CPU-visible memory. With shadow storage the pages it covers get +// their base copies before the submission's copy-in (vk_mem_shadow_sync) can reach them. void vk_mem_gpu_pending(uint64_t addr, uint64_t size) { if (!shadow_on || !size) return; + plat_mutex_lock_shared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, addr, size); + PoolRec *r = e ? e->buf : NULL; plat_mutex_lock(&shadow_lock); - if (nspend == cspend) grow((void **)&spend, &cspend, sizeof *spend); - spend[nspend++] = (SRange){addr, addr + size}; + if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi) + for (uint64_t pg = addr & ~(SB_PAGE - 1); pg < addr + size; pg += SB_PAGE) sbase_make(r, pg); plat_mutex_unlock(&shadow_lock); + plat_mutex_unlock_shared(&pool_lock); } // NX2NAVI_WB_NO_REUP (default 1; 0 = off): the pages a VRAM mirror wrote back into guest memory are not copied up into @@ -1123,22 +1281,35 @@ static int wb_no_reup(void) { return wb_reup_off; } -static void gpu_written(uint64_t addr, uint64_t size, int back) { - if (shadow_on && size) { - plat_mutex_lock_shared(&pool_lock); - const pidx_ent *e = pidx_find(&pool_index, addr, size); - PoolRec *r = e ? e->buf : NULL; - plat_mutex_lock(&shadow_lock); - if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi) { - mem_thaw(addr, size); - memcpy((void *)(uintptr_t)addr, r->smap + (addr - r->import_lo), size); - s_wb_bytes += size; +// Completion thread: the GPU's bytes of [addr, addr+size) from the shadow into guest memory (merge_page against the +// base copies; a page without one whole). Returns the bytes moved. +static uint64_t shadow_write_back(uint64_t addr, uint64_t size) { + uint64_t moved = 0; + plat_mutex_lock_shared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, addr, size); + PoolRec *r = e ? e->buf : NULL; + plat_mutex_lock(&shadow_lock); + if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi) { + uint64_t t0 = qpc(); + mem_thaw(addr, size); + for (uint64_t a = addr, end = addr + size; a < end;) { + uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < end ? pg + SB_PAGE : end; + uint8_t *base = sbase_find(r, pg), *s = r->smap + (a - r->import_lo); + if (base) moved += merge_page((uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a); + else memcpy((void *)(uintptr_t)a, s, pe - a), moved += pe - a; // a write never noted as pending + a = pe; } - for (int i = 0; i < nspend; i++) - if (spend[i].lo == addr && spend[i].hi == addr + size) { spend[i] = spend[--nspend]; break; } - plat_mutex_unlock(&shadow_lock); - plat_mutex_unlock_shared(&pool_lock); + s_wb_bytes += moved, s_wb_seen += size, s_wb_ticks += qpc() - t0; } + plat_mutex_unlock(&shadow_lock); + plat_mutex_unlock_shared(&pool_lock); + return moved; +} + +static void gpu_written(uint64_t addr, uint64_t size, int back) { + // back: the bytes are in a VRAM mirror, and its write-back brings them into the shadow; that submission's + // shadow_wb_cb, after the replay's own callbacks, then copies them into guest memory. + if (shadow_on && size && !back) shadow_write_back(addr, size); if (!size || nvn_gpu_only(addr)) return; uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; back = back && wb_no_reup(); // read before gen_lock: its first call logs @@ -1154,9 +1325,11 @@ static int lazy_clean(uint64_t lo, uint64_t hi, void *ctx); void vk_gpu_written_back(uint64_t addr, uint64_t size) { gpu_written(addr, size, 1); - mem_probe(addr, size); // NX2NAVI_WB_PROBE: the bytes are in guest memory now; note who reads them next + // NX2NAVI_WB_PROBE: the bytes are in guest memory now; note who reads them next. With shadow storage they get there + // later, in the submission's shadow_wb_cb, whose thaw would end the probe at once: that job arms it instead. + if (!shadow_on) mem_probe(addr, size); VkMemSpan sp; - if (size && wb_lazy() && vk_mem_resolve(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on + if (size && wb_lazy() && vk_mem_lookup(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on PoolRec *r = sp.pool; if (r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo && addr + size <= r->import_hi) mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo, r); @@ -1165,7 +1338,7 @@ void vk_gpu_written_back(uint64_t addr, uint64_t size) { int vk_mem_mirrored(uint64_t addr, uint64_t size) { VkMemSpan sp; - if (!vk_mirror_on || !size || !vk_mem_resolve(addr, size, &sp) || !sp.pool) return 0; + if (!vk_mirror_on || !size || !vk_mem_lookup(addr, size, &sp) || !sp.pool) return 0; const PoolRec *r = sp.pool; return r->mstate == MS_ON && addr >= r->import_lo && addr + size <= r->import_hi; } @@ -1174,11 +1347,12 @@ static void shadow_log(uint64_t frame) { if (!shadow_on) return; plat_mutex_lock(&shadow_lock); rt_log("mem: shadow at frame %llu: %lld pools %.1f MiB; since the last: %llu pool syncs, %llu queries, %.1f MiB " - "copied in (%llu untracked), %.1f MiB written back, %.2f ms, %d GPU writes pending", + "copied in (%llu untracked), %.2f ms (queries %.2f); %.1f MiB written back of %.1f MiB the GPU may have written, %.2f ms; %llu " + "pages the GPU writes keep a base copy", (unsigned long long)frame, (long long)s_live_n, s_live_bytes / 1048576.0, (unsigned long long)s_syncs, - (unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, s_wb_bytes / 1048576.0, - qpc_ms(s_ticks), nspend); - s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = 0; + (unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, qpc_ms(s_ticks), + qpc_ms(s_q_ticks), s_wb_bytes / 1048576.0, s_wb_seen / 1048576.0, qpc_ms(s_wb_ticks), (unsigned long long)s_sb_pages); + s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = s_wb_seen = s_wb_ticks = s_q_ticks = 0; plat_mutex_unlock(&shadow_lock); } @@ -1928,14 +2102,14 @@ static struct { static int merge_init(VkCtx *c) { WM.tried = 1; WM.push = (PFN_vkCmdPushDescriptorSetKHR)vk_dpa(c->device, "vkCmdPushDescriptorSetKHR"); - VkDescriptorSetLayoutBinding b[4]; - for (int i = 0; i < 4; i++) + VkDescriptorSetLayoutBinding b[5]; + for (int i = 0; i < 5; i++) b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}; VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, - .bindingCount = 4, .pBindings = b}; - VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 8}; + .bindingCount = 5, .pBindings = b}; + VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 16}; VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof wb_merge_spv, .pCode = wb_merge_spv}; VkShaderModule m = VK_NULL_HANDLE; @@ -1969,7 +2143,9 @@ static int merging(void) { } // A chunk of at least size bytes in chunks[]: the smallest fitting one of the pool, else a new buffer; -1 when the -// allocation fails. +// allocation fails. host: 0 device memory (snapshots), CHUNK_LIST host memory the CPU writes (range lists), +// CHUNK_MARKS host memory the CPU reads back (the merge's page marks, shadow_wb_cb). +enum { CHUNK_LIST = 1, CHUNK_MARKS = 2 }; static int chunk_new(uint64_t size, int host) { uint64_t want = host ? SNAP_LIST_MIN : SNAP_CHUNK; while (want < size) want <<= 1; @@ -1991,7 +2167,9 @@ static int chunk_new(uint64_t size, int host) { .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, - .flags = host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | + .flags = host == CHUNK_MARKS ? VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | + VMA_ALLOCATION_CREATE_MAPPED_BIT + : host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT : 0}; if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ch->buf, &ch->alloc, NULL) != VK_SUCCESS) return -1; ch->size = want, ch->host = host; @@ -2032,67 +2210,182 @@ static int snap_cmp(const void *x, const void *y) { return a->off < b->off ? -1 : a->off > b->off; } -// One dispatch: ranges s[0..n) share a pool and a chunk and lie in [base, base + span) of the pool. +// One dispatch: ranges s[0..n) share a pool and a chunk and lie in [base, base + span) of the pool. marks: the chunk +// of the page marks, its first at marks_off, or -1 when the dispatch marks none (the range list stands in for the +// binding, which the shader then does not write). static void merge_dispatch(VkCommandBuffer cb, const SnapRange *s, int n, uint64_t base, uint64_t span, int list, - uint64_t list_off, uint32_t total) { - VkDescriptorBufferInfo bi[4] = {{s->r->mbuf, base, span}, {chunks[s->chunk].buf, 0, chunks[s->chunk].size}, - {s->r->buf, base, span}, {chunks[list].buf, list_off, (uint64_t)n * 16}}; - VkWriteDescriptorSet w[4]; - for (int i = 0; i < 4; i++) + uint64_t list_off, uint32_t total, int marks, uint64_t marks_off) { + uint64_t npg = ((base & (SB_PAGE - 1)) + span + SB_PAGE - 1) / SB_PAGE; + VkDescriptorBufferInfo bi[5] = {{s->r->mbuf, base, span}, {chunks[s->chunk].buf, 0, chunks[s->chunk].size}, + {s->r->buf, base, span}, {chunks[list].buf, list_off, (uint64_t)n * 16}, + marks >= 0 ? (VkDescriptorBufferInfo){chunks[marks].buf, marks_off, npg * 4} + : (VkDescriptorBufferInfo){chunks[list].buf, list_off, (uint64_t)n * 16}}; + VkWriteDescriptorSet w[5]; + for (int i = 0; i < 5; i++) w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i, .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &bi[i]}; - rec_push(WM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.layout, 0, 4, w); // in call order (vk_rec.h) - uint32_t pc[2] = {(uint32_t)n, total}; + rec_push(WM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.layout, 0, 5, w); // in call order (vk_rec.h) + uint32_t pc[4] = {(uint32_t)n, total, (uint32_t)(base & (SB_PAGE - 1)), marks >= 0}; vkCmdPushConstants(cb, WM.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); vkCmdDispatch(cb, (total + 63) / 64, 1, 1); } +// ---- shadow storage: the GPU's writes into VRAM mirrors back into guest memory +// The mirror's write-back (vk_mem_mirror_end) merges or copies them into the shadow, and the submission's job, added to +// its fence after the replay's own callbacks, copies them on into guest memory: of a merge dispatch only the pages its +// shader marked (wb_merge.comp), of a copy every page, compared with its base copy (merge_page). The fence callback of +// each noted range (gpu_written) compared all of it: 65 MiB of storage buffers a frame in Kokiri Forest, 9 ms a frame +// of the completion thread, behind which the game's fence waits sat (RTX 4090, 2026-11-09). +typedef struct { uint64_t addr, size; uint32_t mark0; } SwbRange; // guest range; its first page's mark, or SWB_ALL +#define SWB_ALL UINT32_MAX +typedef struct { VmaAllocation marks_alloc; const uint32_t *marks; int n; SwbRange g[]; } SwbJob; +static SwbRange *swl; // replay: this submission's merge dispatches +static int nswl, cswl; +static int swl_marks = -1; // replay: their page marks' chunk, -1 none + +static void swl_add(uint64_t addr, uint64_t size, uint32_t mark0) { + if (nswl == cswl) grow((void **)&swl, &cswl, sizeof *swl); + swl[nswl++] = (SwbRange){addr, size, mark0}; +} + +// Completion thread: [a, b) into guest memory, and the pages that changed stamped as the mirror's write-back. +static void swb_run(uint64_t a, uint64_t b) { + if (b <= a || !shadow_write_back(a, b - a) || nvn_gpu_only(a)) return; + int back = wb_no_reup(); + plat_mutex_lock(&gen_lock); + uint32_t g = ++gen_now; + for (uint64_t x = a & ~0xfffull; x < b; x += 1ull << GEN_PAGE_SHIFT) stamp_page_as(x, g, back); + plat_mutex_unlock(&gen_lock); +} + +static void shadow_wb_cb(void *arg) { // completion thread + SwbJob *j = arg; + if (j->marks_alloc) vmaInvalidateAllocation(vk_ctx()->vma, j->marks_alloc, 0, VK_WHOLE_SIZE); + for (int i = 0; i < j->n; i++) { + const SwbRange *g = &j->g[i]; + uint64_t end = g->addr + g->size; + if (g->mark0 == SWB_ALL) { + swb_run(g->addr, end); + continue; + } + uint64_t run = 0; // start of the run of marked pages; guest address 0 is never a pool's + uint32_t k = g->mark0; + for (uint64_t pg = g->addr & ~(SB_PAGE - 1); pg < end; pg += SB_PAGE, k++) { + uint64_t a = pg > g->addr ? pg : g->addr; + if (j->marks[k]) { + if (!run) run = a; + } else if (run) { + swb_run(run, a); + run = 0; + } + } + if (run) swb_run(run, end); + } + // NX2NAVI_WB_PROBE (vk_gpu_written_back): every range is in guest memory now; a later range's thaw would end the + // probe of a region an earlier one armed, so the probes come last + for (int i = 0; i < j->n; i++) mem_probe(j->g[i].addr, j->g[i].size); + free(j); +} + +// vk_mem_mirror_end, before the snapshot chunks go back at the fence (snap_free reuses the marks' chunk): the job of +// this submission's merge dispatches and copies. +static void swb_job(uint64_t fence) { + SwbJob *j = malloc(sizeof *j + sizeof(SwbRange) * (size_t)(nswl + nwbl)); + if (!j) rt_fatal("vk: out of memory"); + j->marks_alloc = NULL, j->marks = NULL; + if (swl_marks >= 0) { + VmaAllocationInfo info; + vmaGetAllocationInfo(vk_ctx()->vma, chunks[swl_marks].alloc, &info); + j->marks_alloc = chunks[swl_marks].alloc, j->marks = info.pMappedData; + } + memcpy(j->g, swl, sizeof *swl * (size_t)nswl); + j->n = nswl; + for (int i = 0; i < nwbl; i++) j->g[j->n++] = (SwbRange){wbl[i].r->import_lo + wbl[i].off, wbl[i].size, SWB_ALL}; + nswl = 0, swl_marks = -1; + vk_completion_add(fence, shadow_wb_cb, j); +} + // vk_mem_mirror_end: the merge dispatches of this submission's ranges. Returns the commands recorded; 0 with nsnl -// left as it was when the range list cannot be allocated (the caller copies them plainly). +// left as it was when the range list cannot be allocated (the caller copies them plainly). With shadow storage each +// dispatch also marks the pages it changed, for swb_job. static int wb_merge_record(VkCommandBuffer cb) { VkCtx *c = vk_ctx(); uint64_t align = c->props.limits.minStorageBufferOffsetAlignment; if (align < 16) align = 16; SnapRange *s = malloc(sizeof *s * (size_t)nsnl); - if (!s) rt_fatal("vk: out of memory"); + struct MGroup { int i, j; uint64_t base, end; uint32_t total; } *gr = malloc(sizeof *gr * (size_t)nsnl); + if (!s || !gr) rt_fatal("vk: out of memory"); memcpy(s, snl, sizeof *s * (size_t)nsnl); qsort(s, (size_t)nsnl, sizeof *s, snap_cmp); - // the range lists: at most one 16-byte entry per range, each dispatch's list at an aligned offset - int list = chunk_new((uint64_t)nsnl * 16 + (uint64_t)nsnl * align, 1); - if (list < 0) { free(s); return 0; } - chunks[list].used = UINT64_MAX; // not a snapshot chunk - VmaAllocationInfo li; - vmaGetAllocationInfo(c->vma, chunks[list].alloc, &li); - uint32_t *lm = li.pMappedData; - uint64_t lo = 0; - int rec = 0; - vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.pipe); + // the dispatches: ranges of one pool and snapshot chunk, within MERGE_SPAN bytes and MERGE_WORDS words + int ng = 0; + uint64_t nmarks = 0; // bytes of page marks, each dispatch's at an aligned offset for (int i = 0; i < nsnl;) { uint64_t base = s[i].off & ~(align - 1), end = s[i].off + s[i].size; uint32_t total = 0; int j = i; - lo = (lo + align - 1) & ~(align - 1); - uint64_t l0 = lo; for (; j < nsnl && s[j].r == s[i].r && s[j].chunk == s[i].chunk; j++) { uint64_t e = s[j].off + s[j].size > end ? s[j].off + s[j].size : end; if (j > i && (e - base > MERGE_SPAN || total + s[j].size / 4 > MERGE_WORDS)) break; end = e; - uint32_t *q = lm + lo / 4; - q[0] = (uint32_t)((s[j].off - base) / 4), q[1] = (uint32_t)(s[j].soff / 4), q[2] = (uint32_t)(s[j].size / 4); - q[3] = total; total += (uint32_t)(s[j].size / 4); + } + gr[ng++] = (struct MGroup){i, j, base, end, total}; + nmarks += (((base & (SB_PAGE - 1)) + end - base + SB_PAGE - 1) / SB_PAGE * 4 + align - 1) & ~(align - 1); + i = j; + } + // the range lists: at most one 16-byte entry per range, each dispatch's list at an aligned offset + int list = chunk_new((uint64_t)nsnl * 16 + (uint64_t)nsnl * align, CHUNK_LIST); + if (list < 0) { + free(s), free(gr); + return 0; + } + chunks[list].used = UINT64_MAX; // not a snapshot chunk + int marks = shadow_on ? chunk_new(nmarks, CHUNK_MARKS) : -1; + uint32_t *mm = NULL; + if (marks >= 0) { + chunks[marks].used = UINT64_MAX; + VmaAllocationInfo mi; + vmaGetAllocationInfo(c->vma, chunks[marks].alloc, &mi); + mm = mi.pMappedData; + memset(mm, 0, nmarks); + vmaFlushAllocation(c->vma, chunks[marks].alloc, 0, VK_WHOLE_SIZE); + swl_marks = marks; + } + VmaAllocationInfo li; + vmaGetAllocationInfo(c->vma, chunks[list].alloc, &li); + uint32_t *lm = li.pMappedData; + uint64_t lo = 0, mo = 0; + int rec = 0; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.pipe); + for (int k = 0; k < ng; k++) { + const struct MGroup *d = &gr[k]; + lo = (lo + align - 1) & ~(align - 1); + uint64_t l0 = lo; + uint32_t first = 0; // the range's first invocation + for (int j = d->i; j < d->j; j++) { + uint32_t *q = lm + lo / 4; + q[0] = (uint32_t)((s[j].off - d->base) / 4), q[1] = (uint32_t)(s[j].soff / 4), q[2] = (uint32_t)(s[j].size / 4); + q[3] = first; + first += (uint32_t)(s[j].size / 4); lo += 16; mrep.merge_bytes += s[j].size; } - merge_dispatch(cb, s + i, j - i, base, end - base, list, l0, total); + merge_dispatch(cb, s + d->i, d->j - d->i, d->base, d->end - d->base, list, l0, d->total, marks, mo); + if (shadow_on) { + uint64_t ga = s[d->i].r->import_lo + d->base; + if (marks >= 0) swl_add(ga, d->end - d->base, (uint32_t)(mo / 4)); + else + for (int j = d->i; j < d->j; j++) swl_add(s[j].r->import_lo + s[j].off, s[j].size, SWB_ALL); + } + mo += (((d->base & (SB_PAGE - 1)) + d->end - d->base + SB_PAGE - 1) / SB_PAGE * 4 + align - 1) & ~(align - 1); mrep.merge_dispatches++; rec += 3; - i = j; } vmaFlushAllocation(c->vma, chunks[list].alloc, 0, VK_WHOLE_SIZE); for (int i = 0; i < nsnl; i++) snl[i].r->wb_sub = msub; - free(s); + free(s), free(gr); return rec; } @@ -2187,7 +2480,9 @@ static int wb_lazy(void) { plat_mutex_lock(&lz_init_lock); if (!lz_state) { VkCtx *c = vk_ctx(); - int on = vk_mirror_on && c && cfg_bool("NX2NAVI_WB_LAZY"); + // Not with shadow storage: a fetch copies the mirror into the pool's buffer, which is guest memory only when + // the pool is imported; a shadow's bytes reach guest memory in the submission's shadow_wb_cb. + int on = vk_mirror_on && !shadow_on && c && cfg_bool("NX2NAVI_WB_LAZY"); if (on) { VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .queueFamilyIndex = c->gfx_family}; @@ -2199,9 +2494,10 @@ static int wb_lazy(void) { vkCreateFence(c->device, &fi, NULL, &lz_fence) == VK_SUCCESS; on = on && mem_lazy_enable(lazy_fetch, lazy_clean); } - rt_log("mem: what shaders write into mirrored pools %s (NX2NAVI_WB_LAZY)", + rt_log("mem: what shaders write into mirrored pools %s (NX2NAVI_WB_LAZY%s)", on ? "goes back to guest memory once and then only when the CPU touches it" - : "goes back to guest memory at the end of every submission"); + : "goes back to guest memory at the end of every submission", + shadow_on ? "; shadow storage keeps it off" : ""); __atomic_store_n(&lz_state, on ? 1 : 2, __ATOMIC_RELEASE); } plat_mutex_unlock(&lz_init_lock); @@ -2543,6 +2839,7 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); } + if (shadow_on && (nswl || nwbl)) swb_job(fence); // before snap_free takes back the marks' chunk if (nchunks) { // the last part's prologue still records snapshot copies into them; back to the pool at the fence SnapFree *f = malloc(sizeof *f + sizeof(SnapChunk) * (size_t)nchunks); if (!f) rt_fatal("vk: out of memory"); @@ -3422,13 +3719,14 @@ void vk_mem_shutdown(void) { plat_mutex_lock(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); pool_index.n = 0; - nm = npend = nwbl = nsnl = snl_copied = 0; // mirrors went with their pools + nm = npend = nwbl = nsnl = snl_copied = nswl = 0; // mirrors went with their pools + swl_marks = -1; plat_mutex_lock(&snap_pool_lock); for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc); nsnap_pool = 0, snap_pool_bytes = 0; plat_mutex_unlock(&snap_pool_lock); plat_mutex_lock(&shadow_lock); - nstouch = nspend = 0; // shadows too + nstouch = 0, s_sb_pages = 0; // shadows too (their base copies went with their records) s_live_n = s_live_bytes = 0; plat_mutex_unlock(&shadow_lock); for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 8faa319a..1ea5661b 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -520,7 +520,7 @@ static void destroy_tex(Tex *e); static int pool_fits(VkCtx *c, const Tex *e, const VkMemoryRequirements *req, uint64_t guest_size, VkMemSpan *sp) { if ((e->f.depth || e->f.stencil) && req->size > guest_size) return 0; if (!req->size) return 0; // an image the device could not lay out (image_size) - if (!vk_mem_resolve(e->pub.addr, req->size, sp) || !sp->alloc || sp->alloc_offset % req->alignment) return 0; + if (!vk_mem_lookup(e->pub.addr, req->size, sp) || !sp->alloc || sp->alloc_offset % req->alignment) return 0; VmaAllocationInfo info; vmaGetAllocationInfo(c->vma, sp->alloc, &info); return req->memoryTypeBits >> info.memoryType & 1; diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index b83544c8..f66ce50d 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -143,7 +143,7 @@ static void collect(const op_hdr *h, int depth) { if (k && known) { if (!(nc[ci].ok >> (k + 1) & 1)) continue; } else if (k) { - if (!vk_mem_resolve(lo, at + n - lo, &sp) || sp.alloc || !sp.pool || sp.avail < at + n - lo || + if (!vk_mem_lookup(lo, at + n - lo, &sp) || sp.alloc || !sp.pool || sp.avail < at + n - lo || (sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) continue; ok |= 1 << (k + 1); @@ -267,7 +267,7 @@ void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) { for (i++; i < U.ncol && U.col[i].lo <= hi; i++) if (U.col[i].hi > hi) hi = U.col[i].hi; VkMemSpan sp; - if (!vk_mem_resolve(lo, hi - lo, &sp) || sp.alloc || !sp.pool || sp.avail < hi - lo || + if (!vk_mem_lookup(lo, hi - lo, &sp) || sp.alloc || !sp.pool || sp.avail < hi - lo || (sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) { U.refused++; continue; @@ -350,7 +350,7 @@ static int wr_overlaps(uint64_t lo, uint64_t hi) { static int cpu_written(uint64_t addr, uint64_t n) { // the same test vk_usnap_begin keeps regions by VkMemSpan sp; - return vk_mem_resolve(addr, n, &sp) && !sp.alloc && sp.pool && sp.avail >= n && + return vk_mem_lookup(addr, n, &sp) && !sp.alloc && sp.pool && sp.avail >= n && (sp.cls == VK_MCLS_HOST_CACHED || sp.cls == VK_MCLS_HOST_UNC); }