Staged mirror uploads: the replay copies guest bytes for the VRAM mirrors while it records, and the first-submission wait ends once the replay has submitted; 8 texture pool sets.

On 0.9.1 this is the release's own mirror capture (NX2NAVI_MIRROR_CAPTURE_KB) turned on: with
NX2NAVI_MIRROR_STAGE=1 (the default) the pages the CPU wrote are copied out of guest memory, up to 64 MiB a
part, when the part is handed to the queue. A submission whose prologue copied everything that way reads no
guest memory late, so the game's wait after a frame's first submission ends once the replay has submitted it
(vk_sync_recorded_wait). One with a range left to the GPU-time copy (a pool with a write-back in flight or a
twin) or a draw that reads a pool without a mirror is noted (vk_mem_note_live), and its wait is the GPU's.
Also 8 texture pool sets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tobim6
2026-11-12 16:22:47 +01:00
parent 7d54ee73c7
commit ad288e3bc1
7 changed files with 138 additions and 16 deletions

View File

@@ -245,6 +245,7 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o
int mem_lazy_armed(uint64_t a, uint64_t n);
// Copies the ranges src[i] = {address, bytes} of guest memory whose pages read without a fault to dst[i], ok[i] set.
int mem_copy_readable(uint8_t *const *dst, const uint64_t (*src)[2], int n, uint8_t *ok);
int mem_lazy_touches(uint64_t a, uint64_t n); // 0: the CPU can read [a, a + n) without a fault now
void mem_lazy_window(int open);
void mem_lazy_release(uint64_t a, uint64_t n);
void mem_lazy_report(uint64_t frame, uint64_t skipped);

View File

@@ -518,8 +518,22 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o
return k;
}
// 0 when no page of [a, a + n) can be inaccessible right now (armed by the lazy write-back or probed), so the CPU can
// read it without a fault; 1 when one may be.
int mem_lazy_touches(uint64_t a, uint64_t n) {
if (!n) return 0;
if (a >= RT_ADDRESS_LIMIT || a + n > RT_ADDRESS_LIMIT || __atomic_load_n(&prb_on, __ATOMIC_RELAXED)) return 1;
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0;
int any = 0;
for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = __atomic_load_n(&lzr[r], __ATOMIC_ACQUIRE);
if (!any) return 0;
int hit = 0;
plat_mutex_lock(&lz_lock);
for (int i = 0; i < lz_n && !hit; i++) hit = lzs[i].lo < a + n && lzs[i].hi > a;
plat_mutex_unlock(&lz_lock);
return hit;
}
// [a, a + n) is armed as a whole still.
int mem_lazy_armed(uint64_t a, uint64_t n) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0;

View File

@@ -741,6 +741,9 @@ NVN_API(nvnQueueSubmitCommands) {
// point). Copies of the data at submit made it worse, a mix of two frames' versions.
// NX2NAVI_SUBMIT_FINISH=<mask>: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits
// after every submission, 0 never.
// Since the staged mirror uploads (vk_memory.c) the copies of guest memory a submission reads are taken while the
// replay records it, and the wait ends once the replay has submitted it (vk_sync_recorded_wait); the GPU's wait
// below remains for a submission whose GPU reads guest memory itself.
// The wait (exec_wait, vk_sync.c vk_sync_exec_wait) ends when the GPU has run the submission's passes and the
// results the game reads once it goes on are in its memory, the last frame's occlusion counts among them; the
// write-backs and the completion thread's other side effects are not waited for. NX2NAVI_WB_SPLIT=0 makes it

View File

@@ -347,6 +347,10 @@ void vk_mem_mirror_begin(void);
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence);
int vk_mem_mirror_pending(void);
void vk_mem_mirror_prologue(VkCommandBuffer cb);
// Staged mirror uploads (vk_memory.c): the GPU of the submission being recorded reads guest memory itself (noted by
// vk_mem_note_live); vk_mem_take_live returns that and starts the next submission's note.
void vk_mem_note_live(void);
int vk_mem_take_live(void);
void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over
void vk_mem_lazy_handed(void);
void vk_mem_lazy_decide(void); // once the driver is known (after vk_caps_driver): on or off, with one log line
@@ -606,7 +610,8 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n); // vk_texture.c: s
// set and brings it up to date. vk_pool_takes counts the takes whose sync left no slot behind (vk_exec_draw.c pool_sync);
// once it has moved on by more than VK_POOL_SETS since a pool entry changed, every set holds the change (vk_texture.c
// retire_sweep). It may count fewer takes than there were, which only delays that.
#define VK_POOL_SETS 4
#define VK_POOL_SETS 8 // 4 before: with the game no longer held for the GPU after a frame's first submission, the
// replay waited for a set's GPU work at 15% of the switches
uint64_t vk_pool_takes(void);
// Texture lock held: 1 while the texture pool refresh looks its slots up (vk_exec_draw.c pool_sync_entries), 0 after.
// Those lookups leave out a texture evicted by disuse, and are not counted as the texture being used.
@@ -715,6 +720,7 @@ uint64_t vk_sync_reserve(void); // asynchronous replay: the value the
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet.
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns);
int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t));
int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own
// NX2NAVI_TRACE=perf (vk_sync.c "the executed timeline"): what a wait on that timeline left behind since the last call,
// whether the completion thread's callback running now belongs to a value whose waiter went on already, and the

View File

@@ -302,9 +302,14 @@ int vk_replay_async(void) {
}
// With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's.
static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) {
static int vk_be_gpu_wait(uint64_t value, uint64_t timeout_ns) {
return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns);
}
// The submit wait (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): the GPU's only where the submission reads guest memory on the
// GPU (vk_sync_recorded_wait).
static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) {
return vk_sync_recorded_wait(value, timeout_ns, vk_be_gpu_wait);
}
// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end.
int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); }

View File

@@ -2622,7 +2622,10 @@ void vk_mem_use(VkMemSpan *sp) {
return;
}
// a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors)
if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) return;
if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) {
vk_mem_note_live(); // the GPU reads this pool's guest memory as it runs (vk_mem_take_live)
return;
}
int lk = vk_exec_shared();
if (lk) plat_mutex_lock(&use_lock);
mem_use_locked(sp);
@@ -2638,7 +2641,10 @@ static void mem_use_locked(VkMemSpan *sp) {
r->mstate = MS_NONE, r->retry_sub = 0; // refuse sets it again
}
if (r->mstate == MS_NONE) mirror_create(r);
if (r->mstate != MS_ON) return;
if (r->mstate != MS_ON) { // the GPU reads this pool's guest memory as it runs (staged mirror uploads)
vk_mem_note_live();
return;
}
if (r->acc_sub != msub) mirror_sync(r);
r->vd_use_sub = msub;
sp->buffer = r->mbuf;
@@ -2646,7 +2652,11 @@ static void mem_use_locked(VkMemSpan *sp) {
void vk_mem_pick(VkMemSpan *sp) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON) return;
if (!r) return;
if (r->mstate != MS_ON) { // guest memory itself, read by the GPU when it runs (vk_mem_take_live)
vk_mem_note_live();
return;
}
if (r->acc_sub != msub) {
int lk = vk_exec_shared();
if (lk) plat_mutex_lock(&use_lock);
@@ -3424,11 +3434,13 @@ static void hot_check(void) {
if (mirror_hot < 0) {
mirror_hot = p < 0 ? 0 : p > 1000 ? 1000 : p;
if (mirror_hot) rt_log("mem: pools copied %d%% of their size a frame stay in system memory (NX2NAVI_MIRROR_HOT)", mirror_hot);
}
if (!mirror_hot) return;
uint64_t fr = mirror_frame;
if (!hot_frame0 || fr < hot_frame0) {
hot_frame0 = fr ? fr : 1;
return;
}
if (fr - hot_frame0 < HOT_FRAMES || npend || nwbl) return; // nsnl: the previous submission's, recorded already
double frames = (double)(fr - hot_frame0);
hot_frame0 = fr;
@@ -3442,6 +3454,7 @@ static void hot_check(void) {
r->hot_bytes = (uint64_t)per; // kept for the log line below
r->hot_bytes = 0, i++; // !twin with a table record: the table's writes are in the mirror alone (mirror_evict)
continue;
}
mem_lazy_release(r->import_lo, r->import_hi - r->import_lo); // what only the mirror holds comes down first
// as mirror_evict, under the exclusive lock (vk_pool_fini takes pools off the list under it), but for good:
// MS_REFUSED with no retry (retry_sub 0), so neither vk_mem_use nor the budget retries mirror it again
@@ -3456,6 +3469,7 @@ static void hot_check(void) {
dead[nout] = r->mbuf, dead_a[nout] = r->malloc_;
r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL;
out[nout++] = r;
}
plat_mutex_unlock(&pool_lock);
for (int k = 0; k < nout; k++) {
PoolRec *r = out[k];
@@ -3467,6 +3481,9 @@ static void hot_check(void) {
if (++n_demoted <= 32)
rt_log("mem: pool %#llx (%.1f MiB, %s) copied %.1f MiB a frame through its mirror: system memory from now on",
(unsigned long long)r->lo, n / 1048576.0, f, per / 1048576.0);
}
}
void vk_mem_mirror_begin(void) {
if (vk_mirror_on) hot_check();
if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions
@@ -3516,28 +3533,51 @@ void vk_mem_mirror_begin(void) {
int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; }
// NX2NAVI_MIRROR_CAPTURE_KB (default 0, off): the pages the CPU wrote since a VRAM mirror's last sync are copied out of
// guest memory by the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a
// host buffer the submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them
// from there into the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or
// more after the game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile,
// values meant for a later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest
// memory is still on the GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over
// the mirror's newer ones), and when its device address table writes reach guest memory as well (a twin: what they
// write is not recorded); so does a range with a page that cannot be read without a fault (mem_copy_readable: a
// lazily armed range). The whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU.
// NX2NAVI_MIRROR_CAPTURE_KB: the pages the CPU wrote since a VRAM mirror's last sync are copied out of guest memory by
// the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a host buffer the
// submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them from there into
// the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or more after the
// game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile, values meant for a
// later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest memory is still on the
// GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over the mirror's newer ones),
// and when its device address table writes reach guest memory as well (a twin: what they write is not recorded); so
// does a range with a page that cannot be read without a fault (mem_copy_readable: a lazily armed range). The
// whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU.
// Unset or 0 with NX2NAVI_MIRROR_STAGE=1 (the default here): CAPTURE_STAGE_KB. A submission whose prologue copied
// everything this way reads nothing of guest memory late, so its submit wait ends once the replay has submitted it
// (vk_sync_recorded_wait); one with a range left to the GPU-time copy, or a draw that reads a pool without a mirror,
// is noted (vk_mem_note_live) and its submit wait is the GPU's. The first form of this (0.8.0) copied while the replay
// recorded and left out neither the write-backs in flight nor the twins.
#define CAPTURE_STAGE_KB (64 << 10)
static volatile int32_t mirror_live; // the submission being recorded had the GPU read guest memory
static int stage_on(void) {
static int on = -1;
if (on < 0) on = rt_knob("MIRROR_STAGE", "1") != 0;
return on;
}
static uint64_t capture_cap(void) {
static int64_t cap = -1;
if (cap < 0) {
int kb = cfg_int("NX2NAVI_MIRROR_CAPTURE_KB");
if (kb <= 0 && stage_on()) kb = CAPTURE_STAGE_KB;
cap = kb > 0 ? (int64_t)kb << 10 : 0;
if (cap)
rt_log("mem: mirror pages the CPU wrote are copied out of guest memory when their part is handed over, up to "
"%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB)", kb);
"%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB, NX2NAVI_MIRROR_STAGE)", kb);
}
return (uint64_t)cap;
}
void vk_mem_note_live(void) { __atomic_store_n(&mirror_live, 1, __ATOMIC_RELAXED); }
int vk_mem_take_live(void) {
int live = __atomic_exchange_n(&mirror_live, 0, __ATOMIC_RELAXED);
if (!vk_mirror_on || shadow_on || !capture_cap()) live = 1;
return live;
}
static MRange *capm; // replay: mirror_capture's merged ranges
static uint64_t *caps; // replay: the chunk offset of each, UINT64_MAX for one left to the GPU-time copy
static uint8_t **capd, *capok; // replay: mem_copy_readable's destinations and results
@@ -3619,6 +3659,7 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb) {
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
mirror_capture(cb);
if (npend) vk_mem_note_live(); // left to the GPU-time copy: the GPU reads guest memory late
copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies);
if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote
if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,

View File

@@ -48,6 +48,10 @@ static PlatCond cv_work = PLAT_COND_INIT; // a submission, a callback or stop
static PlatCond cv_done = PLAT_COND_INIT; // done advanced
static uint64_t issued, done; // newest submitted value; newest value whose side effects ran
static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued)
// Staged mirror uploads (vk_memory.c "staged mirror uploads"): per submission value, whether its GPU still reads guest
// memory itself (value << 1 | live), under st_lock. A value not found counts as live.
#define LIVE_RING 4096
static uint64_t live_of[LIVE_RING];
static int stop;
static Pending *queue; // side effects in the order they were added
static int qhead, qn, qcap;
@@ -509,6 +513,7 @@ static void gw_wait(uint64_t value, uint64_t t0, uint64_t t1, uint64_t newest, u
static void gw_log(uint64_t frame, int frames) {
plat_mutex_lock(&gw_lock);
uint64_t n = gw_n, ns = gw_ns, rep = gw_rep_ns, gpu = gw_gpu_ns, lead = gw_lead_ns, age = gw_age, before = gw_before,
lr = gw_lat_rep, lg = gw_lat_gpu, ln = gw_lat_n, mx = gw_max_ns, unk = gw_unknown, rn = gw_rec_n,
gw_n = gw_ns = gw_rep_ns = gw_gpu_ns = gw_lead_ns = gw_age = gw_before = gw_lat_rep = gw_lat_gpu = gw_lat_n = 0;
plat_mutex_unlock(&gw_lock);
if (!n) {
@@ -963,6 +968,7 @@ static VkCommandPool pre_pool;
static CmdSlot *pre_slots;
static int npre, pre_pair = -1;
static VkCommandBuffer pre_cb;
VkCommandBuffer vk_sync_pre(void) {
if (pre_cb) return pre_cb;
VkCtx *c = vk_ctx();
@@ -1260,9 +1266,15 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) {
s->fence = sent ? fence : 0;
vk_mem_note_submit(fence);
gw_mark(fence, 1);
// after the last part's prologue: every staged copy of the submission is taken; a probe's submit leaves the note to
// the replay's submission it may have come in the middle of
int live = own ? 1 : vk_mem_take_live();
int res = own ? 0x1f : vk_exec_take_results();
plat_mutex_lock(&st_lock);
live_of[fence & (LIVE_RING - 1)] = fence << 1 | (uint64_t)(live != 0);
__atomic_store_n(&issued, fence, __ATOMIC_RELEASE);
plat_cond_broadcast(&cv_work);
plat_cond_broadcast(&cv_done); // vk_sync_recorded_wait
plat_mutex_unlock(&st_lock);
static unsigned subs;
if (++subs % 4096 == 0) {
@@ -1337,6 +1349,46 @@ static void results_run(uint64_t upto) {
static int fence_wait_impl(uint64_t value, uint64_t timeout_ns);
// The game's wait after a frame's first submission (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): until the replay has
// submitted value, which took every copy of guest memory the submission reads; then also until the GPU has run it when
// its GPU reads guest memory itself (vk_mem_take_live) or NX2NAVI_SUBMIT_FINISH_GPU=1. 1 when reached.
int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)) {
vk_sync_start();
if (gpu < 0) {
gpu = rt_knob("SUBMIT_FINISH_GPU", "0") != 0;
rt_log("vk: the wait after a frame's first submission ends %s (NX2NAVI_SUBMIT_FINISH_GPU=%d)",
gpu ? "when the GPU has run it" : "when the replay has submitted it, or the GPU has run it where it reads guest "
"memory itself", gpu);
}
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1;
if (gpu) return gpu_wait(value, timeout_ns);
uint64_t w0 = plat_now_ns(), newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE);
uint64_t sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE);
int live = 1, ok = 1;
plat_mutex_lock(&st_lock);
if (value > issued && value > reserved) ok = 0; // never handed over
while (ok && issued < value && done < value) {
if (timeout_ns < (1ull << 62)) {
uint64_t el = plat_now_ns() - w0;
if (el >= timeout_ns) { ok = 0; break; }
plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el);
} else {
plat_cond_wait(&cv_done, &st_lock);
}
}
uint64_t w1 = plat_now_ns();
if (w1 > w0) {
if (!live) gw_wait(value, w0, w1, newest > sent ? newest : sent, sent); // a GPU wait counts there
plat_mutex_lock(&gw_lock);
gw_rec_n++;
if (!live) gw_rec_done++;
plat_mutex_unlock(&gw_lock);
}
if (!ok) return 0;
return live ? gpu_wait(value, timeout_ns) : 1;
}
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) {
uint64_t p0 = perf_begin();
uint64_t newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE), sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE);