Fix the black 3D world under shadow storage on Linux and Proton, and copy only what each side wrote.

Shadow storage noted a GPU write with the whole range its binding names: the copy-in skipped that range and
the fence copied all of it back, so CPU writes to other fields of the same records never reached the GPU.
Ocarina's lighting parameter is such a field: the lighting came out NaN and the fog pass drew the scene
black. Each page a noted range covers now keeps a base copy, the copy-in moves only the bytes the CPU
changed and the write-back only those the GPU changed (vk_memory.c merge_page). The mirror's merge shader
marks the 4 KiB pages it changed, and one job per submission copies only those back (shadow_wb_cb); the
copy-in reads the CPU-side generation tables through the shared write-watch snapshots; and a caller that
reads guest memory itself resolves with vk_mem_lookup, which does not queue the pool for the copy-in.

Pull request #394 by jacc. Its changes are taken as they are; one hunk met the lazy write-back that came
later (vk_gpu_written_back) and is merged by hand. Added here: the lazy write-back stays off with shadow
storage, because its fetch copies the mirror into the pool's buffer, which is guest memory only for an
imported pool. Files other sessions are changing: runtime/vk_memory.c (the shadow section, the merge
record, vk_gpu_written_back, wb_lazy) and three one-word call changes in runtime/vk_usnap.c.

Without shadow storage (Windows outside Wine) the merge pipeline gains an unused fifth binding and two
push constants and nothing else changes. The Windows files compile; nothing was run, on Windows, Wine or
Linux.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 06:24:18 -05:00
parent f97358ddc4
commit f740e95a62
8 changed files with 425 additions and 115 deletions

View File

@@ -82,6 +82,10 @@ typedef struct PoolRec {
uint8_t *smap; // its mapping; offsets as in buf (from import_lo)
uint32_t sgen; // gen_lock: every page write stamped up to this generation is in the shadow
volatile uint8_t stouched; // on the touched list: the GPU may use it in the submission being recorded
// shadow_lock: base copies of the pages a noted GPU write covers (merge_page), open addressing by page address;
// csb a power of two, page 0 an empty slot
struct SBase { uint64_t page; uint8_t *base; } *sb;
uint32_t nsb, csb;
// NX2NAVI_TRACE=perf per-pool mirror traffic since the last mirror_pool_log, replay thread (the log reads and clears
// them without a lock: a count lost to the race only blurs a diagnostic)
uint64_t st_pages, st_flush, st_wb, st_merge; // bytes: written pages and flush ranges copied up, GPU writes back
@@ -260,6 +264,8 @@ static void destroy_rec(VkCtx *c, PoolRec *r) {
if (r->buf) vkDestroyBuffer(c->device, r->buf, NULL);
if (r->mem) vkFreeMemory(c->device, r->mem, NULL);
if (r->alloc) vmaFreeMemory(c->vma, r->alloc);
for (uint32_t i = 0; i < r->csb; i++) free(r->sb[i].base); // shadow base copies shadow_forget left (shutdown)
free(r->sb);
free(r->pt);
free(r);
}
@@ -461,9 +467,12 @@ static int resolve_check_on(void) {
return on;
}
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out);
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touch);
int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) {
// touch: the GPU may read the pool's buffer in the submission being recorded, so its shadow goes on the touched list
// (shadow storage). A lookup (vk_mem_lookup) leaves it off: the caller reads guest memory itself or only asks what
// backs the address. A pool in the device address table is touched either way, as shaders reach it through pointers.
static int mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) {
memset(out, 0, sizeof *out);
uint64_t g = __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE), end = addr + size;
const ResolveHit *h = rhit_slot(addr);
@@ -475,19 +484,23 @@ int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) {
out->cls = h->cls;
out->pool = h->pool;
if (h->alloc) out->alloc_offset = out->offset;
if (shadow_on && h->pool && !((PoolRec *)h->pool)->stouched) shadow_touch(h->pool);
if (shadow_on && h->pool && !((PoolRec *)h->pool)->stouched && (touch || ((PoolRec *)h->pool)->gpa_rec))
shadow_touch(h->pool);
if (resolve_check_on()) {
VkMemSpan s;
if (!resolve_locked(addr, size, &s) || memcmp(&s, out, sizeof s))
if (!resolve_locked(addr, size, &s, touch) || memcmp(&s, out, sizeof s))
rt_fatal("vk: resolve cache of 0x%llx differs from the pool index", (unsigned long long)addr);
}
return 1;
}
return resolve_locked(addr, size, out);
return resolve_locked(addr, size, out, touch);
}
int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out) { return mem_resolve(addr, size, out, 1); }
int vk_mem_lookup(uint64_t addr, uint64_t size, VkMemSpan *out) { return mem_resolve(addr, size, out, 0); }
// The pool index lookup under the lock; a non-overlapped import or physical pool goes into the address's cache slot.
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) {
static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) {
memset(out, 0, sizeof *out);
plat_mutex_lock_shared(&pool_lock);
const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)addr, size);
@@ -509,7 +522,7 @@ static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out) {
out->buffer = r->buf;
out->offset = addr - r->import_lo;
out->avail = r->import_hi - addr;
if (r->smap && !r->stouched) shadow_touch(r);
if (r->smap && !r->stouched && (touch || r->gpa_rec)) shadow_touch(r);
ok = 1;
} else if (r && r->kind == POOL_PHYS) {
out->buffer = r->buf;
@@ -545,7 +558,7 @@ uint64_t vk_mem_gen(void) {
int vk_mem_class_of(uint64_t addr) {
VkMemSpan m;
return vk_mem_resolve(addr, 1, &m) ? m.cls : VK_MCLS_NONE;
return vk_mem_lookup(addr, 1, &m) ? m.cls : VK_MCLS_NONE;
}
// ---------------------------------------------------------------- census
@@ -968,17 +981,26 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
// - CPU -> shadow: right before each submit (vk_mem_shadow_sync), the pools the replay resolved since the last one are
// queried for written pages; pages stamped after a pool's sgen are copied in by the CPU.
// - GPU -> guest: the replay notes each GPU write into CPU-visible memory at record time (vk_mem_gpu_pending) and
// reports it at the fence (vk_gpu_written), which copies those bytes back into guest memory. Until then the CPU
// copies skip them: a page the CPU and the GPU both write keeps the GPU's bytes.
// reports it at the fence (vk_gpu_written), which copies the GPU's bytes back into guest memory.
// - Pages both write: a GPU write is noted with the whole range its binding names, and the CPU writes other fields of the
// same records meanwhile. Each page such a range covers keeps a base copy from its first note on, the bytes guest
// memory and the shadow last agreed on. A copy-in takes only the bytes where guest memory differs from the base (the
// CPU's writes) and the write-back at the fence only those where the shadow differs from it (the GPU's), and either
// moves the base along (merge_page). A byte both changed takes the CPU's value when a copy-in comes first and the
// GPU's when the fence does. Whole-range copies both ways lost the CPU's bytes: Ocarina writes a lighting parameter
// into a buffer whose binding its compute passes write every frame, the GPU only ever read zero there, the lighting
// coefficients and then the volumetric fog came out NaN, and the fog pass drew the 3D scene black.
// NX2NAVI_SHADOW=1/0 forces the mode on or off; the default is on for Linux and Wine.
typedef struct { uint64_t lo, hi; } SRange;
#define SB_PAGE 4096ull
static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock
static PoolRec **stouch; // shadow_lock: pools resolved since the last sync
static int nstouch, cstouch;
static SRange *spend; // shadow_lock: GPU writes recorded and not copied back yet
static int nspend, cspend;
static uint64_t s_sb_pages; // shadow_lock: base copies kept (sbase_make)
static int64_t s_live_n, s_live_bytes; // shadow_lock
static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks; // shadow_lock: since the last log
static uint64_t s_wb_seen, s_wb_ticks; // shadow_lock: bytes the fence write-backs compared, their time
static uint64_t s_q_ticks; // shadow_lock: the copy-ins' write-watch queries
static int wb_no_reup(void);
static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
r->import_lo = addr & ~0xfffull;
@@ -1020,47 +1042,175 @@ static void shadow_touch(PoolRec *r) {
plat_mutex_unlock(&shadow_lock);
}
// ---- base copies of the pages both write ("Pages both write" above). shadow_lock held throughout.
static uint32_t sbase_hash(uint64_t pg) { return (uint32_t)((pg >> 12) * 0x9E3779B97F4A7C15ull >> 32); }
// The base copy of guest page pg of r, NULL when it has none.
static uint8_t *sbase_find(const PoolRec *r, uint64_t pg) {
if (!r->nsb) return NULL;
for (uint32_t i = sbase_hash(pg) & (r->csb - 1);; i = (i + 1) & (r->csb - 1)) {
if (r->sb[i].page == pg) return r->sb[i].base;
if (!r->sb[i].page) return NULL;
}
}
static void sbase_insert(struct SBase *t, uint32_t cap, uint64_t pg, uint8_t *base) {
uint32_t i = sbase_hash(pg) & (cap - 1);
while (t[i].page) i = (i + 1) & (cap - 1);
t[i].page = pg, t[i].base = base;
}
// Page pg of r gets a base copy unless it has one. Taken from the shadow: no GPU write was noted on the page before, so
// the shadow holds what guest memory held at its last copy-in, and guest memory differs from it by the CPU's writes since.
static void sbase_make(PoolRec *r, uint64_t pg) {
if (sbase_find(r, pg)) return;
if ((r->nsb + 1) * 4 > r->csb * 3) { // at most three quarters full
uint32_t cap = r->csb ? r->csb * 2 : 16;
struct SBase *t = calloc(cap, sizeof *t);
if (!t) rt_fatal("vk: out of memory for the shadow's base copies");
for (uint32_t i = 0; i < r->csb; i++)
if (r->sb[i].page) sbase_insert(t, cap, r->sb[i].page, r->sb[i].base);
free(r->sb);
r->sb = t, r->csb = cap;
}
uint8_t *b = malloc(SB_PAGE);
if (!b) rt_fatal("vk: out of memory for the shadow's base copies");
memcpy(b, r->smap + (pg - r->import_lo), SB_PAGE);
sbase_insert(r->sb, r->csb, pg, b);
r->nsb++;
s_sb_pages++;
}
static void sbase_free(PoolRec *r) {
for (uint32_t i = 0; i < r->csb; i++) free(r->sb[i].base);
free(r->sb);
s_sb_pages -= r->nsb;
r->sb = NULL, r->nsb = r->csb = 0;
}
// 0xff in each byte of x that is not zero, 0 in the others.
static inline uint64_t byte_mask(uint64_t x) {
x |= x >> 4, x |= x >> 2, x |= x >> 1;
return (x & 0x0101010101010101ull) * 0xff;
}
// The bytes of v that m selects into dst (little-endian), the other bytes of dst left as they are: the other side may
// be writing them at the same time.
static inline void store_bytes(uint8_t *dst, uint64_t v, uint64_t m) {
if (m == ~0ull) {
memcpy(dst, &v, 8);
return;
}
for (int k = 0; k < 8; k++)
if (m >> (8 * k) & 0xff) dst[k] = (uint8_t)(v >> (8 * k));
}
// n bytes of one page with base copy b: the bytes where src differs from the base go into dst and into the base. src
// and dst are guest memory and the shadow, one way or the other; each src byte that moves is read once, as its side
// may be writing it. Returns the bytes moved.
static uint64_t merge_page(uint8_t *dst, const uint8_t *src, uint8_t *b, uint64_t n) {
uint64_t moved = 0, i = 0;
for (; i < n && ((uintptr_t)(src + i) & 7); i++) {
uint8_t v = ((const volatile uint8_t *)src)[i];
if (v != b[i]) dst[i] = b[i] = v, moved++;
}
for (; i + 8 <= n; i += 8) {
// Most of a page is unchanged: a 64-byte block equal to its base is passed over with one vector compare. A
// byte its side writes after the compare is a later write, which the next copy-in or write-back takes.
if (!((uintptr_t)(src + i) & 63) && i + 64 <= n) {
uint64_t x = 0;
for (int k = 0; k < 64; k += 8) {
uint64_t sv, ov;
memcpy(&sv, src + i + k, 8), memcpy(&ov, b + i + k, 8);
x |= sv ^ ov;
}
if (!x) {
i += 56;
continue;
}
}
uint64_t s = *(const volatile uint64_t *)(src + i), o;
memcpy(&o, b + i, 8);
uint64_t m = byte_mask(s ^ o);
if (!m) continue;
store_bytes(dst + i, s, m);
memcpy(b + i, &s, 8); // the bytes m leaves out are equal in both
moved += (uint64_t)__builtin_popcountll(m) / 8;
}
for (; i < n; i++) {
uint8_t v = ((const volatile uint8_t *)src)[i];
if (v != b[i]) dst[i] = b[i] = v, moved++;
}
return moved;
}
// vk_pool_fini, after the pool left the index: off the touched list before its record goes.
static void shadow_forget(PoolRec *r) {
plat_mutex_lock(&shadow_lock);
for (int i = 0; i < nstouch; i++)
if (stouch[i] == r) { stouch[i] = stouch[--nstouch]; break; }
s_live_n--, s_live_bytes -= (int64_t)(r->import_hi - r->import_lo);
sbase_free(r);
plat_mutex_unlock(&shadow_lock);
}
// shadow_lock held: copies guest [a, b) into the shadow, leaving out the bytes of GPU writes not copied back yet
// (spend from index i on).
static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b, int i) {
for (; i < nspend; i++)
if (spend[i].lo < b && spend[i].hi > a) {
if (spend[i].lo > a) shadow_copy(r, a, spend[i].lo, i + 1);
if (spend[i].hi < b) shadow_copy(r, spend[i].hi, b, i + 1);
return;
// shadow_lock held: copies guest [a, b) into the shadow; of a page with a base copy only the CPU's bytes (merge_page).
static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b) {
while (a < b) {
uint64_t e = b;
uint8_t *base = NULL;
if (r->nsb) { // the plain run up to the next page with a base copy
for (e = a; e < b; e = (e & ~(SB_PAGE - 1)) + SB_PAGE)
if ((base = sbase_find(r, e & ~(SB_PAGE - 1)))) break;
if (e > b) e = b;
if (e == a) { // a starts on such a page: its part of [a, b)
uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < b ? pg + SB_PAGE : b;
s_copied += merge_page(r->smap + (a - r->import_lo), (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a);
a = pe;
continue;
}
}
memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, b - a);
s_copied += b - a;
memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, e - a);
s_copied += e - a;
a = e;
}
}
// shadow_lock and gen_lock held: stamps the pool's written pages and copies those newer than its shadow.
static void shadow_sync_pool(PoolRec *r) {
// shadow_lock and gen_lock held: stamps the pool's written pages and copies those newer than its shadow. The pool's
// regions are queried through the write-watch snapshots of the guest submission (snap_regions, epoch), which the
// texture checks and the mirror syncs share: each 2 MiB region at most once per guest submission. A query of every
// touched pool at every part's submit was about 380 write-watch calls and 10 ms a frame of the submitting thread in
// Kokiri Forest, while the GPU waited for the next part (RTX 4090, 2026-11-09). A page the CPU writes after its
// region's snapshot goes in at the next guest submission's, as it does into a VRAM mirror.
// cpu (wb_no_reup): the pages are read from the tables without the stamps of a mirror's write-back (stamp_page_as),
// since that write-back went through the shadow, which holds those bytes already; with the other tables every storage
// buffer the game's compute shaders wrote was compared again at the next copy-in.
static void shadow_sync_pool(PoolRec *r, uint32_t epoch, int cpu) {
uint64_t lo = r->import_lo, hi = r->import_hi;
uint32_t g = ++gen_now;
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
int tracked = 1;
for (uint64_t a = lo; a < hi;) { // one query per run of regions that are not frozen
uint64_t b = (a | ((1ull << REG_SHIFT) - 1)) + 1;
if (b > hi) b = hi;
if (mem_frozen(a)) { a = b; continue; }
while (b < hi && !mem_frozen(b)) b = b + (1ull << REG_SHIFT) < hi ? b + (1ull << REG_SHIFT) : hi;
s_queries++;
if (!query_stamp(a, b, g)) tracked = 0;
a = b;
uint64_t t0 = qpc();
if (epoch && r1 < SNAP_REGIONS) {
s_queries += snap_regions(r0, r1, epoch);
for (uint64_t k = r0; k <= r1 && tracked; k++) tracked = !snap_untracked[k];
} else { // outside the snapshot regions, or before the first guest submission: one query per unfrozen run
uint32_t g = ++gen_now;
for (uint64_t a = lo; a < hi;) {
uint64_t b = (a | ((1ull << REG_SHIFT) - 1)) + 1;
if (b > hi) b = hi;
if (mem_frozen(a)) { a = b; continue; }
while (b < hi && !mem_frozen(b)) b = b + (1ull << REG_SHIFT) < hi ? b + (1ull << REG_SHIFT) : hi;
s_queries++;
if (!query_stamp(a, b, g)) tracked = 0;
a = b;
}
}
s_q_ticks += qpc() - t0;
uint32_t since = r->sgen;
r->sgen = gen_now;
if (!tracked) { // nothing tells which pages changed: all of them
s_untracked++;
shadow_copy(r, lo, hi, 0);
shadow_copy(r, lo, hi);
return;
}
uint64_t run = 0;
@@ -1068,27 +1218,29 @@ static void shadow_sync_pool(PoolRec *r) {
uint64_t a = k << REG_SHIFT, b = a + (1ull << REG_SHIFT);
if (a < lo) a = lo;
if (b > hi) b = hi;
const uint32_t *ch = gen_chunk[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk
if (reg_max[k] <= since || !ch) {
if (run) shadow_copy(r, run, a, 0), run = 0;
const uint32_t *ch = (cpu ? cgen_chunk : gen_chunk)[a >> GEN_CHUNK_SHIFT]; // a 2 MiB region lies in one chunk
if ((cpu ? creg_max : reg_max)[k] <= since || !ch) {
if (run) shadow_copy(r, run, a), run = 0;
continue;
}
for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) {
int dirty = ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since;
if (dirty && !run) run = x;
else if (!dirty && run) shadow_copy(r, run, x, 0), run = 0;
else if (!dirty && run) shadow_copy(r, run, x), run = 0;
}
}
if (run) shadow_copy(r, run, hi, 0);
if (run) shadow_copy(r, run, hi);
}
void vk_mem_shadow_sync(void) {
if (!shadow_on) return;
uint64_t t0 = qpc();
uint32_t epoch = vk_texture_epoch();
int cpu = vk_mirror_on && wb_no_reup(); // before the locks: its first call logs
plat_mutex_lock(&shadow_lock);
if (nstouch) {
plat_mutex_lock(&gen_lock);
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i]), stouch[i]->stouched = 0;
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i], epoch, cpu), stouch[i]->stouched = 0;
plat_mutex_unlock(&gen_lock);
s_syncs += (uint64_t)nstouch;
nstouch = 0;
@@ -1097,12 +1249,18 @@ void vk_mem_shadow_sync(void) {
plat_mutex_unlock(&shadow_lock);
}
// Record time: the GPU may write [addr, addr+size) of CPU-visible memory. With shadow storage the pages it covers get
// their base copies before the submission's copy-in (vk_mem_shadow_sync) can reach them.
void vk_mem_gpu_pending(uint64_t addr, uint64_t size) {
if (!shadow_on || !size) return;
plat_mutex_lock_shared(&pool_lock);
const pidx_ent *e = pidx_find(&pool_index, addr, size);
PoolRec *r = e ? e->buf : NULL;
plat_mutex_lock(&shadow_lock);
if (nspend == cspend) grow((void **)&spend, &cspend, sizeof *spend);
spend[nspend++] = (SRange){addr, addr + size};
if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi)
for (uint64_t pg = addr & ~(SB_PAGE - 1); pg < addr + size; pg += SB_PAGE) sbase_make(r, pg);
plat_mutex_unlock(&shadow_lock);
plat_mutex_unlock_shared(&pool_lock);
}
// NX2NAVI_WB_NO_REUP (default 1; 0 = off): the pages a VRAM mirror wrote back into guest memory are not copied up into
@@ -1123,22 +1281,35 @@ static int wb_no_reup(void) {
return wb_reup_off;
}
static void gpu_written(uint64_t addr, uint64_t size, int back) {
if (shadow_on && size) {
plat_mutex_lock_shared(&pool_lock);
const pidx_ent *e = pidx_find(&pool_index, addr, size);
PoolRec *r = e ? e->buf : NULL;
plat_mutex_lock(&shadow_lock);
if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi) {
mem_thaw(addr, size);
memcpy((void *)(uintptr_t)addr, r->smap + (addr - r->import_lo), size);
s_wb_bytes += size;
// Completion thread: the GPU's bytes of [addr, addr+size) from the shadow into guest memory (merge_page against the
// base copies; a page without one whole). Returns the bytes moved.
static uint64_t shadow_write_back(uint64_t addr, uint64_t size) {
uint64_t moved = 0;
plat_mutex_lock_shared(&pool_lock);
const pidx_ent *e = pidx_find(&pool_index, addr, size);
PoolRec *r = e ? e->buf : NULL;
plat_mutex_lock(&shadow_lock);
if (r && r->smap && addr >= r->import_lo && addr + size <= r->import_hi) {
uint64_t t0 = qpc();
mem_thaw(addr, size);
for (uint64_t a = addr, end = addr + size; a < end;) {
uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < end ? pg + SB_PAGE : end;
uint8_t *base = sbase_find(r, pg), *s = r->smap + (a - r->import_lo);
if (base) moved += merge_page((uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a);
else memcpy((void *)(uintptr_t)a, s, pe - a), moved += pe - a; // a write never noted as pending
a = pe;
}
for (int i = 0; i < nspend; i++)
if (spend[i].lo == addr && spend[i].hi == addr + size) { spend[i] = spend[--nspend]; break; }
plat_mutex_unlock(&shadow_lock);
plat_mutex_unlock_shared(&pool_lock);
s_wb_bytes += moved, s_wb_seen += size, s_wb_ticks += qpc() - t0;
}
plat_mutex_unlock(&shadow_lock);
plat_mutex_unlock_shared(&pool_lock);
return moved;
}
static void gpu_written(uint64_t addr, uint64_t size, int back) {
// back: the bytes are in a VRAM mirror, and its write-back brings them into the shadow; that submission's
// shadow_wb_cb, after the replay's own callbacks, then copies them into guest memory.
if (shadow_on && size && !back) shadow_write_back(addr, size);
if (!size || nvn_gpu_only(addr)) return;
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
back = back && wb_no_reup(); // read before gen_lock: its first call logs
@@ -1154,9 +1325,11 @@ static int lazy_clean(uint64_t lo, uint64_t hi, void *ctx);
void vk_gpu_written_back(uint64_t addr, uint64_t size) {
gpu_written(addr, size, 1);
mem_probe(addr, size); // NX2NAVI_WB_PROBE: the bytes are in guest memory now; note who reads them next
// NX2NAVI_WB_PROBE: the bytes are in guest memory now; note who reads them next. With shadow storage they get there
// later, in the submission's shadow_wb_cb, whose thaw would end the probe at once: that job arms it instead.
if (!shadow_on) mem_probe(addr, size);
VkMemSpan sp;
if (size && wb_lazy() && vk_mem_resolve(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on
if (size && wb_lazy() && vk_mem_lookup(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on
PoolRec *r = sp.pool;
if (r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo && addr + size <= r->import_hi)
mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo, r);
@@ -1165,7 +1338,7 @@ void vk_gpu_written_back(uint64_t addr, uint64_t size) {
int vk_mem_mirrored(uint64_t addr, uint64_t size) {
VkMemSpan sp;
if (!vk_mirror_on || !size || !vk_mem_resolve(addr, size, &sp) || !sp.pool) return 0;
if (!vk_mirror_on || !size || !vk_mem_lookup(addr, size, &sp) || !sp.pool) return 0;
const PoolRec *r = sp.pool;
return r->mstate == MS_ON && addr >= r->import_lo && addr + size <= r->import_hi;
}
@@ -1174,11 +1347,12 @@ static void shadow_log(uint64_t frame) {
if (!shadow_on) return;
plat_mutex_lock(&shadow_lock);
rt_log("mem: shadow at frame %llu: %lld pools %.1f MiB; since the last: %llu pool syncs, %llu queries, %.1f MiB "
"copied in (%llu untracked), %.1f MiB written back, %.2f ms, %d GPU writes pending",
"copied in (%llu untracked), %.2f ms (queries %.2f); %.1f MiB written back of %.1f MiB the GPU may have written, %.2f ms; %llu "
"pages the GPU writes keep a base copy",
(unsigned long long)frame, (long long)s_live_n, s_live_bytes / 1048576.0, (unsigned long long)s_syncs,
(unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, s_wb_bytes / 1048576.0,
qpc_ms(s_ticks), nspend);
s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = 0;
(unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, qpc_ms(s_ticks),
qpc_ms(s_q_ticks), s_wb_bytes / 1048576.0, s_wb_seen / 1048576.0, qpc_ms(s_wb_ticks), (unsigned long long)s_sb_pages);
s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = s_wb_seen = s_wb_ticks = s_q_ticks = 0;
plat_mutex_unlock(&shadow_lock);
}
@@ -1928,14 +2102,14 @@ static struct {
static int merge_init(VkCtx *c) {
WM.tried = 1;
WM.push = (PFN_vkCmdPushDescriptorSetKHR)vk_dpa(c->device, "vkCmdPushDescriptorSetKHR");
VkDescriptorSetLayoutBinding b[4];
for (int i = 0; i < 4; i++)
VkDescriptorSetLayoutBinding b[5];
for (int i = 0; i < 5; i++)
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
.bindingCount = 4, .pBindings = b};
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 8};
.bindingCount = 5, .pBindings = b};
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 16};
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof wb_merge_spv,
.pCode = wb_merge_spv};
VkShaderModule m = VK_NULL_HANDLE;
@@ -1969,7 +2143,9 @@ static int merging(void) {
}
// A chunk of at least size bytes in chunks[]: the smallest fitting one of the pool, else a new buffer; -1 when the
// allocation fails.
// allocation fails. host: 0 device memory (snapshots), CHUNK_LIST host memory the CPU writes (range lists),
// CHUNK_MARKS host memory the CPU reads back (the merge's page marks, shadow_wb_cb).
enum { CHUNK_LIST = 1, CHUNK_MARKS = 2 };
static int chunk_new(uint64_t size, int host) {
uint64_t want = host ? SNAP_LIST_MIN : SNAP_CHUNK;
while (want < size) want <<= 1;
@@ -1991,7 +2167,9 @@ static int chunk_new(uint64_t size, int host) {
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.flags = host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
.flags = host == CHUNK_MARKS ? VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT |
VMA_ALLOCATION_CREATE_MAPPED_BIT
: host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
VMA_ALLOCATION_CREATE_MAPPED_BIT : 0};
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ch->buf, &ch->alloc, NULL) != VK_SUCCESS) return -1;
ch->size = want, ch->host = host;
@@ -2032,67 +2210,182 @@ static int snap_cmp(const void *x, const void *y) {
return a->off < b->off ? -1 : a->off > b->off;
}
// One dispatch: ranges s[0..n) share a pool and a chunk and lie in [base, base + span) of the pool.
// One dispatch: ranges s[0..n) share a pool and a chunk and lie in [base, base + span) of the pool. marks: the chunk
// of the page marks, its first at marks_off, or -1 when the dispatch marks none (the range list stands in for the
// binding, which the shader then does not write).
static void merge_dispatch(VkCommandBuffer cb, const SnapRange *s, int n, uint64_t base, uint64_t span, int list,
uint64_t list_off, uint32_t total) {
VkDescriptorBufferInfo bi[4] = {{s->r->mbuf, base, span}, {chunks[s->chunk].buf, 0, chunks[s->chunk].size},
{s->r->buf, base, span}, {chunks[list].buf, list_off, (uint64_t)n * 16}};
VkWriteDescriptorSet w[4];
for (int i = 0; i < 4; i++)
uint64_t list_off, uint32_t total, int marks, uint64_t marks_off) {
uint64_t npg = ((base & (SB_PAGE - 1)) + span + SB_PAGE - 1) / SB_PAGE;
VkDescriptorBufferInfo bi[5] = {{s->r->mbuf, base, span}, {chunks[s->chunk].buf, 0, chunks[s->chunk].size},
{s->r->buf, base, span}, {chunks[list].buf, list_off, (uint64_t)n * 16},
marks >= 0 ? (VkDescriptorBufferInfo){chunks[marks].buf, marks_off, npg * 4}
: (VkDescriptorBufferInfo){chunks[list].buf, list_off, (uint64_t)n * 16}};
VkWriteDescriptorSet w[5];
for (int i = 0; i < 5; i++)
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)i,
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &bi[i]};
rec_push(WM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.layout, 0, 4, w); // in call order (vk_rec.h)
uint32_t pc[2] = {(uint32_t)n, total};
rec_push(WM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.layout, 0, 5, w); // in call order (vk_rec.h)
uint32_t pc[4] = {(uint32_t)n, total, (uint32_t)(base & (SB_PAGE - 1)), marks >= 0};
vkCmdPushConstants(cb, WM.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc);
vkCmdDispatch(cb, (total + 63) / 64, 1, 1);
}
// ---- shadow storage: the GPU's writes into VRAM mirrors back into guest memory
// The mirror's write-back (vk_mem_mirror_end) merges or copies them into the shadow, and the submission's job, added to
// its fence after the replay's own callbacks, copies them on into guest memory: of a merge dispatch only the pages its
// shader marked (wb_merge.comp), of a copy every page, compared with its base copy (merge_page). The fence callback of
// each noted range (gpu_written) compared all of it: 65 MiB of storage buffers a frame in Kokiri Forest, 9 ms a frame
// of the completion thread, behind which the game's fence waits sat (RTX 4090, 2026-11-09).
typedef struct { uint64_t addr, size; uint32_t mark0; } SwbRange; // guest range; its first page's mark, or SWB_ALL
#define SWB_ALL UINT32_MAX
typedef struct { VmaAllocation marks_alloc; const uint32_t *marks; int n; SwbRange g[]; } SwbJob;
static SwbRange *swl; // replay: this submission's merge dispatches
static int nswl, cswl;
static int swl_marks = -1; // replay: their page marks' chunk, -1 none
static void swl_add(uint64_t addr, uint64_t size, uint32_t mark0) {
if (nswl == cswl) grow((void **)&swl, &cswl, sizeof *swl);
swl[nswl++] = (SwbRange){addr, size, mark0};
}
// Completion thread: [a, b) into guest memory, and the pages that changed stamped as the mirror's write-back.
static void swb_run(uint64_t a, uint64_t b) {
if (b <= a || !shadow_write_back(a, b - a) || nvn_gpu_only(a)) return;
int back = wb_no_reup();
plat_mutex_lock(&gen_lock);
uint32_t g = ++gen_now;
for (uint64_t x = a & ~0xfffull; x < b; x += 1ull << GEN_PAGE_SHIFT) stamp_page_as(x, g, back);
plat_mutex_unlock(&gen_lock);
}
static void shadow_wb_cb(void *arg) { // completion thread
SwbJob *j = arg;
if (j->marks_alloc) vmaInvalidateAllocation(vk_ctx()->vma, j->marks_alloc, 0, VK_WHOLE_SIZE);
for (int i = 0; i < j->n; i++) {
const SwbRange *g = &j->g[i];
uint64_t end = g->addr + g->size;
if (g->mark0 == SWB_ALL) {
swb_run(g->addr, end);
continue;
}
uint64_t run = 0; // start of the run of marked pages; guest address 0 is never a pool's
uint32_t k = g->mark0;
for (uint64_t pg = g->addr & ~(SB_PAGE - 1); pg < end; pg += SB_PAGE, k++) {
uint64_t a = pg > g->addr ? pg : g->addr;
if (j->marks[k]) {
if (!run) run = a;
} else if (run) {
swb_run(run, a);
run = 0;
}
}
if (run) swb_run(run, end);
}
// NX2NAVI_WB_PROBE (vk_gpu_written_back): every range is in guest memory now; a later range's thaw would end the
// probe of a region an earlier one armed, so the probes come last
for (int i = 0; i < j->n; i++) mem_probe(j->g[i].addr, j->g[i].size);
free(j);
}
// vk_mem_mirror_end, before the snapshot chunks go back at the fence (snap_free reuses the marks' chunk): the job of
// this submission's merge dispatches and copies.
static void swb_job(uint64_t fence) {
SwbJob *j = malloc(sizeof *j + sizeof(SwbRange) * (size_t)(nswl + nwbl));
if (!j) rt_fatal("vk: out of memory");
j->marks_alloc = NULL, j->marks = NULL;
if (swl_marks >= 0) {
VmaAllocationInfo info;
vmaGetAllocationInfo(vk_ctx()->vma, chunks[swl_marks].alloc, &info);
j->marks_alloc = chunks[swl_marks].alloc, j->marks = info.pMappedData;
}
memcpy(j->g, swl, sizeof *swl * (size_t)nswl);
j->n = nswl;
for (int i = 0; i < nwbl; i++) j->g[j->n++] = (SwbRange){wbl[i].r->import_lo + wbl[i].off, wbl[i].size, SWB_ALL};
nswl = 0, swl_marks = -1;
vk_completion_add(fence, shadow_wb_cb, j);
}
// vk_mem_mirror_end: the merge dispatches of this submission's ranges. Returns the commands recorded; 0 with nsnl
// left as it was when the range list cannot be allocated (the caller copies them plainly).
// left as it was when the range list cannot be allocated (the caller copies them plainly). With shadow storage each
// dispatch also marks the pages it changed, for swb_job.
static int wb_merge_record(VkCommandBuffer cb) {
VkCtx *c = vk_ctx();
uint64_t align = c->props.limits.minStorageBufferOffsetAlignment;
if (align < 16) align = 16;
SnapRange *s = malloc(sizeof *s * (size_t)nsnl);
if (!s) rt_fatal("vk: out of memory");
struct MGroup { int i, j; uint64_t base, end; uint32_t total; } *gr = malloc(sizeof *gr * (size_t)nsnl);
if (!s || !gr) rt_fatal("vk: out of memory");
memcpy(s, snl, sizeof *s * (size_t)nsnl);
qsort(s, (size_t)nsnl, sizeof *s, snap_cmp);
// the range lists: at most one 16-byte entry per range, each dispatch's list at an aligned offset
int list = chunk_new((uint64_t)nsnl * 16 + (uint64_t)nsnl * align, 1);
if (list < 0) { free(s); return 0; }
chunks[list].used = UINT64_MAX; // not a snapshot chunk
VmaAllocationInfo li;
vmaGetAllocationInfo(c->vma, chunks[list].alloc, &li);
uint32_t *lm = li.pMappedData;
uint64_t lo = 0;
int rec = 0;
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.pipe);
// the dispatches: ranges of one pool and snapshot chunk, within MERGE_SPAN bytes and MERGE_WORDS words
int ng = 0;
uint64_t nmarks = 0; // bytes of page marks, each dispatch's at an aligned offset
for (int i = 0; i < nsnl;) {
uint64_t base = s[i].off & ~(align - 1), end = s[i].off + s[i].size;
uint32_t total = 0;
int j = i;
lo = (lo + align - 1) & ~(align - 1);
uint64_t l0 = lo;
for (; j < nsnl && s[j].r == s[i].r && s[j].chunk == s[i].chunk; j++) {
uint64_t e = s[j].off + s[j].size > end ? s[j].off + s[j].size : end;
if (j > i && (e - base > MERGE_SPAN || total + s[j].size / 4 > MERGE_WORDS)) break;
end = e;
uint32_t *q = lm + lo / 4;
q[0] = (uint32_t)((s[j].off - base) / 4), q[1] = (uint32_t)(s[j].soff / 4), q[2] = (uint32_t)(s[j].size / 4);
q[3] = total;
total += (uint32_t)(s[j].size / 4);
}
gr[ng++] = (struct MGroup){i, j, base, end, total};
nmarks += (((base & (SB_PAGE - 1)) + end - base + SB_PAGE - 1) / SB_PAGE * 4 + align - 1) & ~(align - 1);
i = j;
}
// the range lists: at most one 16-byte entry per range, each dispatch's list at an aligned offset
int list = chunk_new((uint64_t)nsnl * 16 + (uint64_t)nsnl * align, CHUNK_LIST);
if (list < 0) {
free(s), free(gr);
return 0;
}
chunks[list].used = UINT64_MAX; // not a snapshot chunk
int marks = shadow_on ? chunk_new(nmarks, CHUNK_MARKS) : -1;
uint32_t *mm = NULL;
if (marks >= 0) {
chunks[marks].used = UINT64_MAX;
VmaAllocationInfo mi;
vmaGetAllocationInfo(c->vma, chunks[marks].alloc, &mi);
mm = mi.pMappedData;
memset(mm, 0, nmarks);
vmaFlushAllocation(c->vma, chunks[marks].alloc, 0, VK_WHOLE_SIZE);
swl_marks = marks;
}
VmaAllocationInfo li;
vmaGetAllocationInfo(c->vma, chunks[list].alloc, &li);
uint32_t *lm = li.pMappedData;
uint64_t lo = 0, mo = 0;
int rec = 0;
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, WM.pipe);
for (int k = 0; k < ng; k++) {
const struct MGroup *d = &gr[k];
lo = (lo + align - 1) & ~(align - 1);
uint64_t l0 = lo;
uint32_t first = 0; // the range's first invocation
for (int j = d->i; j < d->j; j++) {
uint32_t *q = lm + lo / 4;
q[0] = (uint32_t)((s[j].off - d->base) / 4), q[1] = (uint32_t)(s[j].soff / 4), q[2] = (uint32_t)(s[j].size / 4);
q[3] = first;
first += (uint32_t)(s[j].size / 4);
lo += 16;
mrep.merge_bytes += s[j].size;
}
merge_dispatch(cb, s + i, j - i, base, end - base, list, l0, total);
merge_dispatch(cb, s + d->i, d->j - d->i, d->base, d->end - d->base, list, l0, d->total, marks, mo);
if (shadow_on) {
uint64_t ga = s[d->i].r->import_lo + d->base;
if (marks >= 0) swl_add(ga, d->end - d->base, (uint32_t)(mo / 4));
else
for (int j = d->i; j < d->j; j++) swl_add(s[j].r->import_lo + s[j].off, s[j].size, SWB_ALL);
}
mo += (((d->base & (SB_PAGE - 1)) + d->end - d->base + SB_PAGE - 1) / SB_PAGE * 4 + align - 1) & ~(align - 1);
mrep.merge_dispatches++;
rec += 3;
i = j;
}
vmaFlushAllocation(c->vma, chunks[list].alloc, 0, VK_WHOLE_SIZE);
for (int i = 0; i < nsnl; i++) snl[i].r->wb_sub = msub;
free(s);
free(s), free(gr);
return rec;
}
@@ -2187,7 +2480,9 @@ static int wb_lazy(void) {
plat_mutex_lock(&lz_init_lock);
if (!lz_state) {
VkCtx *c = vk_ctx();
int on = vk_mirror_on && c && cfg_bool("NX2NAVI_WB_LAZY");
// Not with shadow storage: a fetch copies the mirror into the pool's buffer, which is guest memory only when
// the pool is imported; a shadow's bytes reach guest memory in the submission's shadow_wb_cb.
int on = vk_mirror_on && !shadow_on && c && cfg_bool("NX2NAVI_WB_LAZY");
if (on) {
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.queueFamilyIndex = c->gfx_family};
@@ -2199,9 +2494,10 @@ static int wb_lazy(void) {
vkCreateFence(c->device, &fi, NULL, &lz_fence) == VK_SUCCESS;
on = on && mem_lazy_enable(lazy_fetch, lazy_clean);
}
rt_log("mem: what shaders write into mirrored pools %s (NX2NAVI_WB_LAZY)",
rt_log("mem: what shaders write into mirrored pools %s (NX2NAVI_WB_LAZY%s)",
on ? "goes back to guest memory once and then only when the CPU touches it"
: "goes back to guest memory at the end of every submission");
: "goes back to guest memory at the end of every submission",
shadow_on ? "; shadow storage keeps it off" : "");
__atomic_store_n(&lz_state, on ? 1 : 2, __ATOMIC_RELEASE);
}
plat_mutex_unlock(&lz_init_lock);
@@ -2543,6 +2839,7 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
}
if (shadow_on && (nswl || nwbl)) swb_job(fence); // before snap_free takes back the marks' chunk
if (nchunks) { // the last part's prologue still records snapshot copies into them; back to the pool at the fence
SnapFree *f = malloc(sizeof *f + sizeof(SnapChunk) * (size_t)nchunks);
if (!f) rt_fatal("vk: out of memory");
@@ -3422,13 +3719,14 @@ void vk_mem_shutdown(void) {
plat_mutex_lock(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
pool_index.n = 0;
nm = npend = nwbl = nsnl = snl_copied = 0; // mirrors went with their pools
nm = npend = nwbl = nsnl = snl_copied = nswl = 0; // mirrors went with their pools
swl_marks = -1;
plat_mutex_lock(&snap_pool_lock);
for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc);
nsnap_pool = 0, snap_pool_bytes = 0;
plat_mutex_unlock(&snap_pool_lock);
plat_mutex_lock(&shadow_lock);
nstouch = nspend = 0; // shadows too
nstouch = 0, s_sb_pages = 0; // shadows too (their base copies went with their records)
s_live_n = s_live_bytes = 0;
plat_mutex_unlock(&shadow_lock);
for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran