Copy what shaders write into the game's memory back only when the CPU touches it, which takes Hyrule Field from 51 to 71 fps.

Standing in Hyrule Field the GPU spent 13 ms a frame, of which 2.4 ms were the game's passes and 7 to 9 ms the copy
of 66 MiB of shader-written buffers back into guest memory at the end of every submission, in case the CPU reads
them. The write-back probe found no CPU access to them in 40 s. Now a range goes back once and its pages are made
inaccessible; while they stay so, later writes stay in the mirror. The first CPU access faults, the bytes are fetched
with a GPU copy of their own after everything submitted so far, the pages are given back, and the range is written
back every submission again for a while (1 s, doubling to 32 s while it keeps being touched). Thaws before kernel
reads and writes and before mapping changes release armed ranges the same way, and so does a mirror that goes away.

Dev build, same spot, 80 s each: 51.5 to 71.0 fps median, 1% low 43.5 to 56.3, GPU time 13.1 to 7.5 ms a frame,
fence waits 9.8 to 4.4 ms. In that run the game faulted on armed pages 226 times and 124 fetches took 51 ms in all;
a walk into a fight made 283 faults and 186 fetches, and a test that touches eight armed ranges every 60 frames
(wb_lazy_touch) 480 faults and 364 fetches, none with a fault of its own. wb_lazy = 0 is the old behaviour.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 04:36:46 -05:00
parent 1b9016896a
commit 17c13e49ff
5 changed files with 384 additions and 5 deletions

View File

@@ -640,6 +640,10 @@
# first submission ends when that submission's passes are through, without waiting for the copy back (3 to 4 ms of
# every frame). 0: the copy back is part of the submission, and the wait covers it.
# wb_split = 1
# 1: what the game's shaders write into its memory is copied back from the video card once, and after that only when
# the game's CPU code touches it (most of it never is: 66 MiB a frame in Hyrule Field). 0: copied back at the end of
# every GPU submission, as before.
# wb_lazy = 1
# 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every
# page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in
# Kokiri Forest.

View File

@@ -64,7 +64,7 @@ static const char *const known[] = {
"SWKBD_ANSWER",
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"ZSTD_CHECK", "ZSTD_NATIVE",
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP",
@@ -123,7 +123,7 @@ static const struct { const char *key, *section; } sections[] = {
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
{"VK_NAN_PRESERVE", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"WB_LAZY", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"},
};
@@ -244,7 +244,7 @@ static const CfgMeta metas[] = {
{"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag
// diagnostics
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", "1", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
{"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha

View File

@@ -185,6 +185,14 @@ void mem_thaw(uint64_t a, uint64_t n); // makes the frozen r
// before a kernel write (file read) into guest memory
void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws);
// NX2NAVI_WB_PROBE=1 (memory.c, "Probed regions"): who reads what the GPU wrote back into guest memory.
// NX2NAVI_WB_LAZY (memory.c, "Lazy write-back"): pages the GPU wrote that come down only when the CPU touches them.
int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size));
uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off);
int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max);
int mem_lazy_armed(uint64_t a, uint64_t n);
void mem_lazy_release(uint64_t a, uint64_t n);
void mem_lazy_report(uint64_t frame, uint64_t skipped);
int mem_lazy_pick(uint64_t *addr, int max); // test: addresses of armed pages
int mem_probe(uint64_t a, uint64_t n); // the 2 MiB regions inside [a, a+n) fault on their next access
int mem_probing(void); // the probe is on
void mem_probe_report(uint64_t frame); // logs the accesses since the last report

View File

@@ -108,9 +108,231 @@ static void unprobe_locked(uint64_t r) {
plat_dec32(&nprb);
}
// Lazy write-back (NX2NAVI_WB_LAZY, vk_memory.c "lazy write-back"). What a shader wrote into a mirrored pool used to be
// copied back into guest memory at the end of every submission, in case the CPU reads it. Ocarina's CPU does not read
// most of it: standing in Hyrule Field, 66 MiB a frame went back over PCIe, 7 to 9 ms of each frame's 13 to 15 ms of
// GPU time, and the write-back probe found no access to it in 40 s. So a range that went back once is armed here: its
// whole pages are made inaccessible (mem_lazy_arm), and while it stays armed the back end leaves its later writes in
// the mirror (mem_lazy_skip marks it stale). The first read or write of an armed page faults: lazy_fault has the
// back end copy the range's bytes down now (lz_fetch, a GPU copy after everything submitted so far), gives the pages
// back and runs the access again, and the range stays out for a while (lz_hot: 1 s, doubling to 32 s while it keeps
// being touched), written back every submission as before. Every thaw (before the kernel writes or reads guest
// memory, and before mapping changes) releases the armed ranges of its regions the same way.
// A region with armed pages is not frozen (freeze_locked wants every page read-write), and a frozen one is not armed.
typedef struct { uint64_t lo, hi, h0, h1; int64_t delta; uint8_t stale; } LzSeg; // pages [lo, hi); buffer offset = address + delta
static LzSeg *lzs; // frz_lock
static int lz_n, lz_cap; // frz_lock
static volatile uint8_t lzr[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // a segment reaches into the region
static uint32_t lz_hot[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 until which nothing is armed here
static uint8_t lz_back[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: seconds of the region's last hold
static volatile int32_t lz_on;
static int lz_want; // the setting, read in mem_init
static void (*lz_fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size);
// frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time
static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns;
static void lazy_flags_locked(uint64_t lo, uint64_t hi) {
for (uint64_t r = lo >> FRZ_SHIFT; r <= (hi - 1) >> FRZ_SHIFT; r++) {
int any = 0;
for (int j = 0; j < lz_n && !any; j++)
any = lzs[j].lo < ((r + 1) << FRZ_SHIFT) && lzs[j].hi > (r << FRZ_SHIFT);
__atomic_store_n(&lzr[r], (uint8_t)any, __ATOMIC_RELEASE);
}
}
// frz_lock held exclusively. Segment i goes: its bytes come down first when the mirror's are newer, then its pages
// are read-write again. why: 0 a read fault, 1 a write fault (both hold the regions back), else a thaw or the mirror
// going away.
static void lazy_drop_locked(int i, int why) {
LzSeg s = lzs[i];
if (s.stale && lz_fetch) {
uint64_t t0 = plat_now_ns();
lz_fetch(s.h0, s.h1, (uint64_t)((int64_t)s.lo + s.delta), s.hi - s.lo);
lz_fetches++, lz_fetch_bytes += s.hi - s.lo, lz_fetch_ns += plat_now_ns() - t0;
}
for (uint64_t a = s.lo, b; a < s.hi; a = b)
if (plat_mem_state(a, s.hi, &b) == PLAT_PROT_NONE && !plat_mem_protect(a, b - a, PLAT_PROT_RW))
rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)a,
(unsigned long long)(b - a));
lzs[i] = lzs[--lz_n];
lazy_flags_locked(s.lo, s.hi);
if (why > 1) { lz_other++; return; }
lz_faults[why]++;
uint32_t now = (uint32_t)plat_now_ms();
for (uint64_t r = s.lo >> FRZ_SHIFT; r <= (s.hi - 1) >> FRZ_SHIFT; r++) {
int again = lz_hot[r] && (int32_t)(now - lz_hot[r]) < 60000; // touched again within a minute of its last hold
uint8_t b = again ? (uint8_t)(lz_back[r] < 32 ? lz_back[r] * 2 : 32) : 1;
lz_back[r] = b ? b : 1;
lz_hot[r] = (now + (uint32_t)lz_back[r] * 1000u) | 1;
}
}
// frz_lock held exclusively: every segment that reaches into region r goes.
static void lazy_drop_region_locked(uint64_t r, int why) {
for (int i = lz_n; i-- > 0;)
if (lzs[i].lo < ((r + 1) << FRZ_SHIFT) && lzs[i].hi > (r << FRZ_SHIFT)) lazy_drop_locked(i, why);
}
static int lazy_covered_locked(uint64_t lo, uint64_t hi) {
for (int i = 0; i < lz_n; i++)
if (lzs[i].lo <= lo && lzs[i].hi >= hi) return 1;
return 0;
}
// frz_lock held exclusively: [x, e), inside one region, becomes inaccessible when every committed page of it is
// read-write, or inaccessible as part of a segment already; anything else (a read-only page, a hole, a page the game
// made inaccessible itself) leaves it as it is.
static int lazy_protect_locked(uint64_t x, uint64_t e) {
uint64_t run[16][2];
int k = 0;
for (uint64_t a = x, b; a < e; a = b) {
int st = plat_mem_state(a, e, &b);
if (st == PLAT_PROT_RW) {
if (k == 16) return 0;
run[k][0] = a, run[k][1] = b, k++;
} else if (st != PLAT_PROT_NONE || !lazy_covered_locked(a, b)) return 0;
}
for (int i = 0; i < k; i++)
if (!plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_NONE)) {
while (i--) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW);
return 0;
}
return 1;
}
// Arms the whole pages of [a, a + n), which the back end has just written back from the buffer h0 into h1 at offset
// off: they fault on their next access. Returns the bytes armed now. Not in a frozen, probed, held or thaw-held region.
uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t off) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || a >= RT_ADDRESS_LIMIT || !watched(a, n)) return 0;
uint64_t lo = (a + PAGE - 1) & ~(PAGE - 1), hi = (a + n) & ~(PAGE - 1), armed = 0;
if (hi <= lo) return 0;
int64_t delta = (int64_t)off - (int64_t)a;
plat_mutex_lock(&frz_lock);
if (lazy_covered_locked(lo, hi)) {
plat_mutex_unlock(&frz_lock);
return 0;
}
uint32_t now = (uint32_t)plat_now_ms();
for (uint64_t x = lo, e; x < hi; x = e) {
uint64_t r = x >> FRZ_SHIFT;
e = (r + 1) << FRZ_SHIFT;
if (e > hi) e = hi;
if (frz[r] || prb[r] || (lz_hot[r] && (int32_t)(lz_hot[r] - now) > 0) ||
(thaw_ms[r] && (int32_t)(now - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) || !lazy_protect_locked(x, e))
continue;
uint64_t slo = x, shi = e;
uint8_t stale = 0;
for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer
if (lzs[j].h0 == h0 && lzs[j].delta == delta && lzs[j].lo <= shi && lzs[j].hi >= slo) {
if (lzs[j].lo < slo) slo = lzs[j].lo;
if (lzs[j].hi > shi) shi = lzs[j].hi;
stale |= lzs[j].stale;
lzs[j] = lzs[--lz_n];
j = -1;
}
if (lz_n == lz_cap) {
lz_cap = lz_cap ? lz_cap * 2 : 256;
lzs = realloc(lzs, sizeof *lzs * (size_t)lz_cap);
if (!lzs) rt_fatal("memory: out of memory for the lazily written ranges");
}
lzs[lz_n++] = (LzSeg){slo, shi, h0, h1, delta, stale};
__atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE);
armed += e - x;
}
plat_mutex_unlock(&frz_lock);
return armed;
}
// The armed parts of [a, a + n), sorted, into out (at most max); they count as stale from here on: the caller leaves
// the bytes it is about to write there in the mirror. Returns how many.
int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t out[][2], int max) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || !n || a >= RT_ADDRESS_LIMIT || a + n > RT_ADDRESS_LIMIT) return 0;
int any = 0, k = 0;
for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = lzr[r];
if (!any) return 0;
plat_mutex_lock(&frz_lock);
for (int i = 0; i < lz_n && k < max; i++) {
uint64_t lo = lzs[i].lo > a ? lzs[i].lo : a, hi = lzs[i].hi < a + n ? lzs[i].hi : a + n;
if (lo >= hi) continue;
lzs[i].stale = 1;
int at = k++;
for (; at > 0 && out[at - 1][0] > lo; at--) out[at][0] = out[at - 1][0], out[at][1] = out[at - 1][1];
out[at][0] = lo, out[at][1] = hi;
}
plat_mutex_unlock(&frz_lock);
return k;
}
// [a, a + n) is armed as a whole still.
int mem_lazy_armed(uint64_t a, uint64_t n) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0;
plat_mutex_lock(&frz_lock);
int ok = lazy_covered_locked(a, a + n);
plat_mutex_unlock(&frz_lock);
return ok;
}
// Releases what is armed in [a, a + n): before the buffers a fetch would read go away.
void mem_lazy_release(uint64_t a, uint64_t n) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED) || !n) return;
plat_mutex_lock(&frz_lock);
for (int i = lz_n; i-- > 0;)
if (lzs[i].lo < a + n && lzs[i].hi > a) lazy_drop_locked(i, 4);
plat_mutex_unlock(&frz_lock);
}
// The back end is ready to fetch: arming starts. 0 when the setting is off (or the probe is on, which watches the
// same pages).
int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size)) {
if (!lz_want) return 0;
lz_fetch = fetch;
__atomic_store_n(&lz_on, 1, __ATOMIC_RELEASE);
return 1;
}
static int lazy_fault(uint64_t a, int write) {
if (a >= RT_ADDRESS_LIMIT || !__atomic_load_n(&lzr[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE)) return 0;
int hit = 0;
plat_mutex_lock(&frz_lock);
for (int i = 0; i < lz_n && !hit; i++)
if (a >= lzs[i].lo && a < lzs[i].hi) lazy_drop_locked(i, write != 0), hit = 1;
plat_mutex_unlock(&frz_lock);
return hit;
}
// Test (NX2NAVI_WB_LAZY_TOUCH): the first addresses of up to max armed segments, another set each call, for a caller
// that reads them to make the faults a game's access would.
int mem_lazy_pick(uint64_t *addr, int max) {
static int next;
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0;
int k = 0;
plat_mutex_lock(&frz_lock);
for (; k < max && k < lz_n; k++) addr[k] = lzs[(next + k) % lz_n].lo;
next += k;
plat_mutex_unlock(&frz_lock);
return k;
}
// One log line since the last one; skipped: the bytes the back end left in the mirror since then.
void mem_lazy_report(uint64_t frame, uint64_t skipped) {
if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return;
plat_mutex_lock(&frz_lock);
uint64_t bytes = 0, fr = lz_faults[0], fw = lz_faults[1], other = lz_other, nf = lz_fetches, fb = lz_fetch_bytes,
fns = lz_fetch_ns;
int n = lz_n;
for (int i = 0; i < lz_n; i++) bytes += lzs[i].hi - lzs[i].lo;
lz_faults[0] = lz_faults[1] = lz_other = lz_fetches = lz_fetch_bytes = lz_fetch_ns = 0;
plat_mutex_unlock(&frz_lock);
rt_log("mem: lazy write-back at frame %llu: %d ranges armed, %.1f MiB; since the last line %.1f MiB left in the "
"mirror, %llu read and %llu write faults, %llu other releases, %llu fetches of %.1f MiB in %.2f ms",
(unsigned long long)frame, n, bytes / 1048576.0, skipped / 1048576.0, (unsigned long long)fr,
(unsigned long long)fw, (unsigned long long)other, (unsigned long long)nf, fb / 1048576.0, fns / 1e6);
}
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
static void thaw_locked(uint64_t r, int why) {
if (why == 2) thaw_ms[r] = (uint32_t)plat_now_ms() | 1;
if (lzr[r]) lazy_drop_region_locked(r, why + 1); // not a fault on an armed page: no hold
if (prb[r]) unprobe_locked(r);
if (!frz[r]) return;
// The read-only runs only: freeze_locked left reserved holes (heap pages given back) as they were. Every committed
@@ -212,7 +434,8 @@ int mem_probe(uint64_t a, uint64_t n) {
return armed;
}
int mem_probing(void) { return __atomic_load_n(&prb_on, __ATOMIC_RELAXED); }
// Pages of guest memory may be inaccessible (the probe, or the lazy write-back): a kernel read of them has to thaw first.
int mem_probing(void) { return __atomic_load_n(&prb_on, __ATOMIC_RELAXED) || __atomic_load_n(&lz_on, __ATOMIC_RELAXED); }
// The platform's access-fault hook while the probe is on: the first access of a probed region ends its probe and is
// noted; 1 runs the access again. A read that faulted in a region another thread gave back meanwhile retries too; a
@@ -234,6 +457,16 @@ static int probe_fault(uint64_t a, int write, uint64_t ip) {
return plat_mem_state(a & ~(PAGE - 1), (a & ~(PAGE - 1)) + PAGE, &end) >= PLAT_PROT_READ;
}
// The platform's access-fault hook: an armed page of the lazy write-back first, then the probe. A read that faulted on
// a page another thread gave back meanwhile runs again; a write goes on to thaw_fault.
static int access_fault(uint64_t a, int write, uint64_t ip) {
if (__atomic_load_n(&lz_on, __ATOMIC_RELAXED) && lazy_fault(a, write)) return 1;
if (__atomic_load_n(&prb_on, __ATOMIC_RELAXED)) return probe_fault(a, write, ip);
if (write || a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return 0;
uint64_t end;
return plat_mem_state(a & ~(PAGE - 1), (a & ~(PAGE - 1)) + PAGE, &end) >= PLAT_PROT_READ;
}
// One log block for the probe since the last one: how many regions were armed, how many probes a read or a write
// ended, and the first accesses by instruction (an address inside the executable as nx2navi+offset, for the linker
// map or nm). Nothing while the probe is off.
@@ -348,8 +581,11 @@ void mem_init(void) {
// depends on NX2NAVI_APP_POOL and on the resolution settings (resolution.c decides them once, on this first call).
cfg_init();
if (cfg_bool("NX2NAVI_WB_PROBE")) { // a diagnosis: off, no access fault reaches probe_fault and nothing is probed
plat_mem_access_hook(probe_fault);
plat_mem_access_hook(access_fault);
__atomic_store_n(&prb_on, 1, __ATOMIC_RELEASE);
} else if (cfg_bool("NX2NAVI_WB_LAZY")) { // armed only once the back end can fetch (mem_lazy_enable)
plat_mem_access_hook(access_fault);
lz_want = 1;
}
const char *pool = cfg_get("NX2NAVI_APP_POOL");
L = (RtGuestLayout){

View File

@@ -1149,9 +1149,17 @@ static void gpu_written(uint64_t addr, uint64_t size, int back) {
}
void vk_gpu_written(uint64_t addr, uint64_t size) { gpu_written(addr, size, 0); }
static int wb_lazy(void);
void vk_gpu_written_back(uint64_t addr, uint64_t size) {
gpu_written(addr, size, 1);
mem_probe(addr, size); // NX2NAVI_WB_PROBE: the bytes are in guest memory now; note who reads them next
VkMemSpan sp;
if (size && wb_lazy() && vk_mem_resolve(addr, size, &sp) && sp.pool) { // or newer ones stay in the mirror from here on
const PoolRec *r = sp.pool;
if (r->mstate == MS_ON && r->mbuf && r->buf && addr >= r->import_lo && addr + size <= r->import_hi)
mem_lazy_arm(addr, size, (uint64_t)(uintptr_t)r->mbuf, (uint64_t)(uintptr_t)r->buf, addr - r->import_lo);
}
}
int vk_mem_mirrored(uint64_t addr, uint64_t size) {
@@ -1566,6 +1574,7 @@ static uint64_t mirror_evict(uint64_t need, uint32_t min_idle, const char *why)
for (; n < nv && freed < need; n++) {
PoolRec *r = v[n];
idle_for[n] = msub - r->acc_sub;
mem_lazy_release(r->import_lo, r->import_hi - r->import_lo); // what only the mirror holds comes down first
mirror_forget(r); // off the list, mirror_gen bumped: no buffer use gets the mirror from here on
r->mstate = MS_NONE, r->acc_sub = 0, r->retry_sub = 0;
dead[n] = r->mbuf, dead_a[n] = r->malloc_;
@@ -2102,7 +2111,109 @@ static void snap_free(void *arg) { // completion thread: the submission is don
free(f);
}
// ---- lazy write-back (NX2NAVI_WB_LAZY, default 1; memory.c "Lazy write-back")
// A range a shader wrote goes back to guest memory once and is then armed (vk_gpu_written_back, mem_lazy_arm). While it
// stays armed, gpu_wrote leaves the later writes of its pages in the mirror: no snapshot, no merge, no copy. When the
// CPU touches a page, memory.c calls lazy_fetch with the mirror and the imported buffer of the range: a copy of its
// own, sent after every submission handed over so far (the held ones first), and waited for. A submission that is
// still being recorded when that happens has already left its write out, so vk_mem_mirror_end copies the ranges it
// skipped that are no longer armed (lz_skip).
static VkCommandPool lz_pool;
static VkCommandBuffer lz_cb;
static VkFence lz_fence;
static volatile int32_t lz_state; // 0 not asked, 1 on, 2 off
static PlatMutex lz_init_lock = PLAT_MUTEX_INIT;
typedef struct { PoolRec *r; uint64_t off, size; } LzSkip;
static LzSkip *lz_skip; // replay: what this submission left in the mirror
static int lz_nskip, lz_cskip;
static uint64_t lz_skipped; // replay: bytes left in the mirror since the last log line
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
VkAccessFlags2 da);
// memory.c's fetch (frz_lock held there, so one at a time): [off, off + size) of the mirror h0 into the imported
// buffer h1, after all the GPU work handed over so far.
static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) {
VkCtx *c = vk_ctx();
VkBufferCopy reg = {off, off, size};
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
vkResetCommandPool(c->device, lz_pool, 0);
vkBeginCommandBuffer(lz_cb, &bi);
mbarrier(lz_cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
vkCmdCopyBuffer(lz_cb, (VkBuffer)(uintptr_t)h0, (VkBuffer)(uintptr_t)h1, 1, &reg);
mbarrier(lz_cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_2_HOST_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_HOST_READ_BIT | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
vkEndCommandBuffer(lz_cb);
VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &lz_cb};
vk_queue_lock();
vk_sync_flush_held(); // the held submissions wrote first
VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, lz_fence);
vk_queue_unlock();
if (r == VK_SUCCESS) r = vkWaitForFences(c->device, 1, &lz_fence, VK_TRUE, 10000000000ull);
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
rt_fatal("vk: bringing %llu bytes the GPU wrote down to guest memory failed (%d)", (unsigned long long)size, (int)r);
}
vkResetFences(c->device, 1, &lz_fence);
}
static int wb_lazy(void) {
int s = __atomic_load_n(&lz_state, __ATOMIC_ACQUIRE);
if (s) return s == 1;
plat_mutex_lock(&lz_init_lock);
if (!lz_state) {
VkCtx *c = vk_ctx();
int on = vk_mirror_on && c && cfg_bool("NX2NAVI_WB_LAZY");
if (on) {
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.queueFamilyIndex = c->gfx_family};
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
on = vkCreateCommandPool(c->device, &pi, NULL, &lz_pool) == VK_SUCCESS;
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = lz_pool,
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
on = on && vkAllocateCommandBuffers(c->device, &ai, &lz_cb) == VK_SUCCESS &&
vkCreateFence(c->device, &fi, NULL, &lz_fence) == VK_SUCCESS;
on = on && mem_lazy_enable(lazy_fetch);
}
rt_log("mem: what shaders write into mirrored pools %s (NX2NAVI_WB_LAZY)",
on ? "goes back to guest memory once and then only when the CPU touches it"
: "goes back to guest memory at the end of every submission");
__atomic_store_n(&lz_state, on ? 1 : 2, __ATOMIC_RELEASE);
}
plat_mutex_unlock(&lz_init_lock);
return lz_state == 1;
}
static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader);
static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size || !wb_lazy()) {
gpu_wrote_range(sp, size, shader);
return;
}
uint64_t g0 = r->import_lo + sp->offset, g1 = g0 + size, run = g0, out[16][2];
int k = mem_lazy_skip(g0, size, out, 16);
for (int i = 0; i <= k; i++) {
uint64_t lo = i < k ? out[i][0] : g1, hi = i < k ? out[i][1] : g1;
if (lo > run) { // not armed: written back as ever
VkMemSpan part = *sp;
part.offset = sp->offset + (run - g0);
gpu_wrote_range(&part, lo - run, shader);
}
if (i < k) {
if (lz_nskip == lz_cskip) grow((void **)&lz_skip, &lz_cskip, sizeof *lz_skip);
lz_skip[lz_nskip++] = (LzSkip){r, sp->offset + (lo - g0), hi - lo};
lz_skipped += hi - lo;
}
run = hi;
}
}
static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return;
r->vd_wrote_sub = msub, r->vd_wrote_n++;
@@ -2371,6 +2482,12 @@ int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); }
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
if (!vk_mirror_on) return 0;
int rec = 0;
// What this submission left in the mirror for a range that was armed then and is not now (the CPU touched it while
// the submission was recorded): its bytes go down after all, as a copy.
for (int i = 0; i < lz_nskip; i++)
if (!mem_lazy_armed(lz_skip[i].r->import_lo + lz_skip[i].off, lz_skip[i].size))
range_add(&wbl, &nwbl, &cwbl, lz_skip[i].r, lz_skip[i].off, lz_skip[i].size);
lz_nskip = 0;
if (nsnl) {
// the merge reads the mirror the submission's shaders wrote and writes the imported buffers
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT,
@@ -2467,6 +2584,20 @@ static void mirror_log(uint64_t frame) {
uint64_t fz, ff, ft, fo;
mem_freeze_stats(&fz, &ff, &ft, &fo);
mem_probe_report(frame);
mem_lazy_report(frame, lz_skipped);
lz_skipped = 0;
// NX2NAVI_WB_LAZY_TOUCH=<n> (test, 0 off): at each of these lines this thread reads a byte of n armed ranges, as
// the game's CPU might, so the fault, the fetch and the hold run without waiting for the game to do it.
static int touch = -1;
if (touch < 0) touch = cfg_int("NX2NAVI_WB_LAZY_TOUCH");
if (touch > 0) {
uint64_t at[16];
int k = mem_lazy_pick(at, touch > 16 ? 16 : touch);
for (int i = 0; i < k; i++) {
volatile uint8_t v = *(volatile const uint8_t *)(uintptr_t)at[i];
(void)v;
}
}
if (rt_trace & RT_TRACE_PERF)
rt_log("mem: write-watch frame %llu: per frame, every consumer: %.1f queries over %.1f MiB, %.3f ms (%.1f found "
"%.0f written pages, %.3f ms); frozen %.0f MiB; since the last line: freezes %llu, thaws by a write %llu, by a file read "