Keep a fault from releasing a lazily written range unseen while a part is handed over, merge what a part skipped and must write back after all, and leave the lazy write-back off on Linux unless set.

A review found a gap: a part checked what it had skipped at its end, and a fault between that check and the part's
hand-over to the queue fetched without the part's writes while nothing wrote them back afterwards. The replay now
opens a window from the check to the hand-over, per part, and a fault on an armed page waits for it to close (at most
20 ms; the replay thread's own faults and thaws do not wait and are counted). What a part skipped for a range that is
no longer armed goes back through the usual snapshot and merge, taken in that part's prologue, and not as a plain
copy, which would have put the mirror's words over what the CPU had just written there. A part split off earlier
makes the same check before its prologue. wb_lazy_touch is a test that reads armed ranges on purpose.

Dev build, Hyrule Field: 71.1 fps as before; standing, a fight, pause and unpause, and the touch test made 146 to 338
faults each, none inside a window, no fault of its own.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 05:23:43 -05:00
parent 17c13e49ff
commit bd9b48ff04
6 changed files with 93 additions and 25 deletions

View File

@@ -642,7 +642,7 @@
# wb_split = 1
# 1: what the game's shaders write into its memory is copied back from the video card once, and after that only when
# the game's CPU code touches it (most of it never is: 66 MiB a frame in Hyrule Field). 0: copied back at the end of
# every GPU submission, as before.
# every GPU submission, as before. On Linux it is off unless set here.
# wb_lazy = 1
# 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every
# page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in

View File

@@ -190,6 +190,7 @@ int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64
uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off);
int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max);
int mem_lazy_armed(uint64_t a, uint64_t n);
void mem_lazy_window(int open);
void mem_lazy_release(uint64_t a, uint64_t n);
void mem_lazy_report(uint64_t frame, uint64_t skipped);
int mem_lazy_pick(uint64_t *addr, int max); // test: addresses of armed pages

View File

@@ -119,6 +119,13 @@ static void unprobe_locked(uint64_t r) {
// being touched), written back every submission as before. Every thaw (before the kernel writes or reads guest
// memory, and before mapping changes) releases the armed ranges of its regions the same way.
// A region with armed pages is not frozen (freeze_locked wants every page read-write), and a frozen one is not armed.
// A fetch brings what the GPU work handed over so far wrote. The part of a submission that is being recorded has left
// its writes out as well, and makes up for it when it finds a range it skipped no longer armed (vk_memory.c
// lazy_part_check). Between that check and the part's hand-over a release would be seen by neither side, so the
// replay opens a window there (mem_lazy_window) and a fault on an armed page waits for it to close: the fetch then
// has the part's writes. Only a fault of the replay thread itself, a wait of 20 ms, or a release by a thaw goes ahead
// inside the window (lz_gaps counts the faults): that part's writes to the range then reach guest memory with the
// range's next write-back.
typedef struct { uint64_t lo, hi, h0, h1; int64_t delta; uint8_t stale; } LzSeg; // pages [lo, hi); buffer offset = address + delta
static LzSeg *lzs; // frz_lock
static int lz_n, lz_cap; // frz_lock
@@ -129,7 +136,8 @@ static volatile int32_t lz_on;
static int lz_want; // the setting, read in mem_init
static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size);
// frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time
static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns;
static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps;
static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else
static void lazy_flags_locked(uint64_t lo, uint64_t hi) {
for (uint64_t r = lo >> FRZ_SHIFT; r <= (hi - 1) >> FRZ_SHIFT; r++) {
@@ -290,14 +298,40 @@ int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64
return 1;
}
// Opens (under frz_lock, so no release is half done) or closes the window of the replay's part.
void mem_lazy_window(int open) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return;
if (!open) {
__atomic_store_n(&lz_win_tid, 0, __ATOMIC_RELEASE);
return;
}
plat_mutex_lock(&frz_lock);
__atomic_store_n(&lz_win_tid, plat_thread_id(), __ATOMIC_RELEASE);
plat_mutex_unlock(&frz_lock);
}
static int lazy_fault(uint64_t a, int write) {
if (a >= RT_ADDRESS_LIMIT || !__atomic_load_n(&lzr[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE)) return 0;
int hit = 0;
plat_mutex_lock(&frz_lock);
for (int i = 0; i < lz_n && !hit; i++)
if (a >= lzs[i].lo && a < lzs[i].hi) lazy_drop_locked(i, write != 0), hit = 1;
plat_mutex_unlock(&frz_lock);
return hit;
uint32_t me = plat_thread_id();
for (int tries = 0;; tries++) {
plat_mutex_lock(&frz_lock);
int i = 0;
while (i < lz_n && !(a >= lzs[i].lo && a < lzs[i].hi)) i++;
if (i == lz_n) {
plat_mutex_unlock(&frz_lock);
return 0;
}
uint32_t w = __atomic_load_n(&lz_win_tid, __ATOMIC_ACQUIRE);
if (w && w != me && tries < 400) { // up to 20 ms: after the hand-over the fetch has that part's writes
plat_mutex_unlock(&frz_lock);
plat_wait_ns(50000);
continue;
}
if (w) lz_gaps++;
lazy_drop_locked(i, write != 0);
plat_mutex_unlock(&frz_lock);
return 1;
}
}
// Test (NX2NAVI_WB_LAZY_TOUCH): the first addresses of up to max armed segments, another set each call, for a caller
@@ -318,15 +352,16 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return;
plat_mutex_lock(&frz_lock);
uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes,
fns = lz_fetch_ns;
fns = lz_fetch_ns, gaps = lz_gaps;
int n = lz_n;
for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo;
lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = 0;
lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = lz_gaps = 0;
plat_mutex_unlock(&frz_lock);
rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the "
"mirror, %llu read and %llu write faults, %llu other releases, %llu fetches of %.1f MiB in %.2f ms",
(unsigned long long)frame, n, bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr,
(unsigned long long)fw, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6);
"mirror, %llu read and %llu write faults (%llu inside a part's window), %llu other releases, %llu fetches of "
"%.1f MiB in %.2f ms", (unsigned long long)frame, n, bytes / 1048576.0, skipped / 1048576.0,
(unsigned long long)fr, (unsigned long long)fw, (unsigned long long)gaps, (unsigned long long)other,
(unsigned long long)nf, fb / 1048576.0, fns / 1e6);
}
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
@@ -583,7 +618,11 @@ void mem_init(void) {
if (cfg_bool("NX2NAVI_WB_PROBE")) { // a diagnosis: off, no access fault reaches probe_fault and nothing is probed
plat_mem_access_hook(access_fault);
__atomic_store_n(&prb_on, 1, __ATOMIC_RELEASE);
#ifdef _WIN32
} else if (cfg_bool("NX2NAVI_WB_LAZY")) { // armed only once the back end can fetch (mem_lazy_enable)
#else // the fetch runs inside the access-fault handler, which has not run on this platform yet: only when set
} else if (cfg_bool("NX2NAVI_WB_LAZY") && cfg_is_set("NX2NAVI_WB_LAZY")) {
#endif
plat_mem_access_hook(access_fault);
lz_want = 1;
}

View File

@@ -323,6 +323,8 @@ void vk_mem_mirror_begin(void);
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence);
int vk_mem_mirror_pending(void);
void vk_mem_mirror_prologue(VkCommandBuffer cb);
void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over
void vk_mem_lazy_handed(void);
// Shadow storage (vk_memory.c): vk_sync.c calls vk_mem_shadow_sync right before each submit; the replay notes every GPU
// write into CPU-visible memory at record time (vk_mem_gpu_pending) and reports it at the fence with vk_gpu_written,
// which copies the bytes back into guest memory before telling the texture cache (vk_guest_written).

View File

@@ -2115,17 +2115,20 @@ static void snap_free(void *arg) { // completion thread: the submission is don
// A range a shader wrote goes back to guest memory once and is then armed (vk_gpu_written_back, mem_lazy_arm). While it
// stays armed, gpu_wrote leaves the later writes of its pages in the mirror: no snapshot, no merge, no copy. When the
// CPU touches a page, memory.c calls lazy_fetch with the mirror and the imported buffer of the range: a copy of its
// own, sent after every submission handed over so far (the held ones first), and waited for. A submission that is
// still being recorded when that happens has already left its write out, so vk_mem_mirror_end copies the ranges it
// skipped that are no longer armed (lz_skip).
// own, sent after every part handed over so far (the held ones first), and waited for. The part that is still being
// recorded when that happens has already left its writes out: at its end (lazy_part_check, from vk_mem_mirror_end or
// from the prologue of a part split off earlier) it writes back, as ever, what it skipped and finds no longer armed;
// the snapshot for that merge is taken in the part's own prologue, before its GPU work. From that check to the part's
// hand-over (vk_mem_lazy_handed) memory.c's window keeps a fault from releasing a range unseen.
static VkCommandPool lz_pool;
static VkCommandBuffer lz_cb;
static VkFence lz_fence;
static volatile int32_t lz_state; // 0 not asked, 1 on, 2 off
static PlatMutex lz_init_lock = PLAT_MUTEX_INIT;
typedef struct { PoolRec *r; uint64_t off, size; } LzSkip;
static LzSkip *lz_skip; // replay: what this submission left in the mirror
typedef struct { VkMemSpan sp; uint64_t size; int shader; } LzSkip;
static LzSkip *lz_skip; // replay: what the part being recorded left in the mirror
static int lz_nskip, lz_cskip;
static int lz_win; // replay: the part's window is open (lazy_part_check)
static uint64_t lz_skipped; // replay: bytes left in the mirror since the last log line
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
@@ -2206,13 +2209,38 @@ static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) {
}
if (i < k) {
if (lz_nskip == lz_cskip) grow((void **)&lz_skip, &lz_cskip, sizeof *lz_skip);
lz_skip[lz_nskip++] = (LzSkip){r, sp->offset + (lo - g0), hi - lo};
LzSkip k = {*sp, hi - lo, shader};
k.sp.offset = sp->offset + (lo - g0);
lz_skip[lz_nskip++] = k;
lz_skipped += hi - lo;
}
run = hi;
}
}
// The end of the part being recorded: what it left in the mirror for a range that is no longer armed (the CPU touched
// it meanwhile) is written back after all. Opens memory.c's window, which vk_mem_lazy_handed closes.
static void lazy_part_check(void) {
if (lz_win || !lz_nskip) return;
mem_lazy_window(1);
lz_win = 1;
for (int i = 0; i < lz_nskip; i++) {
const PoolRec *r = lz_skip[i].sp.pool;
if (!mem_lazy_armed(r->import_lo + lz_skip[i].sp.offset, lz_skip[i].size))
gpu_wrote_range(&lz_skip[i].sp, lz_skip[i].size, lz_skip[i].shader);
}
}
// vk_sync.c, before a part's prologue is recorded: the check for a part that was split off (the last part's ran in
// vk_mem_mirror_end already).
void vk_mem_lazy_part_end(void) { lazy_part_check(); }
// vk_sync.c, once the part is held or handed to the queue: a fetch from here on has its writes.
void vk_mem_lazy_handed(void) {
lz_nskip = 0;
if (lz_win) mem_lazy_window(0), lz_win = 0;
}
static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return;
@@ -2482,12 +2510,7 @@ int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); }
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
if (!vk_mirror_on) return 0;
int rec = 0;
// What this submission left in the mirror for a range that was armed then and is not now (the CPU touched it while
// the submission was recorded): its bytes go down after all, as a copy.
for (int i = 0; i < lz_nskip; i++)
if (!mem_lazy_armed(lz_skip[i].r->import_lo + lz_skip[i].off, lz_skip[i].size))
range_add(&wbl, &nwbl, &cwbl, lz_skip[i].r, lz_skip[i].off, lz_skip[i].size);
lz_nskip = 0;
lazy_part_check(); // before the merges are recorded: what the last part skipped and has to write back after all
if (nsnl) {
// the merge reads the mirror the submission's shaders wrote and writes the imported buffers
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT,

View File

@@ -500,6 +500,7 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
// First brings the shadow storage up to date with the CPU's writes, which needs no command buffer.
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted
vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
int i = claim_slot(c, fence); // before reading slots: the claim may move it
VkCommandBuffer cb = slots[i].cb;
@@ -536,6 +537,7 @@ static VkCommandBuffer split_part(int exec) {
si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence;
ts_submitted();
queue_submit(c, &si, 0); // held or handed to the queue (submit batching)
vk_mem_lazy_handed();
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
@@ -607,6 +609,7 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
.pSignalSemaphoreInfos = sig};
ts_submitted();
queue_submit(c, &si, fence); // held or handed to the queue (submit batching)
vk_mem_lazy_handed();
s->fence = recorded ? fence : 0;
vk_mem_note_submit(fence);
plat_mutex_lock(&st_lock);