mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 21:29:58 -04:00
Merge the shadow storage flicker fix: the wait after a frame's first submission covers the write-backs there.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -272,7 +272,10 @@ writes come back at its fence. Pages both sides write keep a base copy, so each
|
||||
a VRAM mirror's write-back merge marks the pages it changed, and only those are compared at the fence; a lookup that
|
||||
reads guest memory itself (the uniform snapshot, `vk_mem_lookup`) does not queue its pool for the copy-in. In Kokiri
|
||||
Forest (1.1.1, RTX 4090) that costs about 1 ms a frame (copy-in 0.5 ms, write-back 0.6 ms), and the copies hold about
|
||||
5 GB. `NX2NAVI_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as
|
||||
5 GB. The GPU's bytes reach guest memory in the fence's completion callback, later than an import's, so the wait
|
||||
after a frame's first submission covers the write-backs there (`runtime/vk_sync.c`, "the executed timeline"): the game
|
||||
rewrites what those passes wrote without a fence, and a write-back that landed after the rewrite left the GPU adding
|
||||
light onto its own last output, the whole scene brighter for a frame. `NX2NAVI_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as
|
||||
well: the shadow copies are chosen for the measured cost above, not for a missing extension.
|
||||
|
||||
The device address table (`runtime/vk_gpa.c`) names a pool's shadow copy as it names an import, so shaders follow guest
|
||||
@@ -280,9 +283,20 @@ pointers into either. A shader that reads through the table resolves nothing, so
|
||||
the copy-in: a part whose pipelines declare the table is marked at its draws and dispatches, and its prologue brings
|
||||
the shadow of every pool in the table up to date with the pages the CPU wrote and queues the same pages for those
|
||||
pools' VRAM mirrors (`vk_memory.c` `vk_mem_gpa_sync`). That copies the written pages of every CPU-visible pool in
|
||||
each such part, which the cost above does not include. What a shader stores through the table into a shadow copy does
|
||||
not come back to guest memory. `--probe-vkmem` checks the read: a shader follows the table to a word of a pool that is
|
||||
never bound, and must read `0x13579BDF` and, after a plain CPU write, `0x2468ACE0`. On Linux, run it for each backing:
|
||||
each such part, which the cost above does not include. What a shader stores through the table lands in the shadow copy
|
||||
and in the pool's VRAM mirror, and no binding names it, so nothing notes it as a GPU write: it does not come back to
|
||||
guest memory, and the next whole copy-in of its page (the CPU rewrites other words of the same records) puts guest
|
||||
memory's stale bytes back over it in both copies. In the runs measured so far that copy-in always came before the
|
||||
frame's stores, which does no harm. `NX2NAVI_SHADOW_TABLE_STORES=1` keeps the stores instead: the copy-ins then keep a
|
||||
hash of each page they write, a page whose shadow no longer matches its hash was stored into, keeps a base copy from
|
||||
then on like a page with a noted GPU write, and has its stores written back to guest memory at the fence of every
|
||||
submission that uses the table. It is off by default (`runtime/vk_memory.c`, "Pages a shader stores into", says why).
|
||||
`NX2NAVI_SHADOW_CHECK=1` logs every 60 frames how often a copy-in overwrote such a store and for which pools, how many
|
||||
write-backs landed on pages the CPU had written since their copy-in, and what the copy-ins cost and where they went.
|
||||
`--probe-vkmem` checks the read: a shader follows the table to a word of a pool that is never bound, and must read
|
||||
`0x13579BDF` and, after a plain CPU write, `0x2468ACE0`; and with `NX2NAVI_SHADOW_TABLE_STORES=1` the store: a GPU
|
||||
write into a shadow page that nothing notes must survive the CPU's writes of that page and reach guest memory. On
|
||||
Linux, run it for each backing:
|
||||
|
||||
```sh
|
||||
for shadow in 0 1; do for memory in import mirror; do
|
||||
|
||||
@@ -60,7 +60,7 @@ static const char *const known[] = {
|
||||
"PRESNAP_PLACE",
|
||||
"REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES",
|
||||
"ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE",
|
||||
"SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_BULK_WRITE", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR",
|
||||
"SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_BULK_WRITE", "SHADOW_CHECK", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHADOW_TABLE_STORES", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR",
|
||||
"SVC_SLOW_MS",
|
||||
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS",
|
||||
"SWKBD_ANSWER",
|
||||
@@ -192,6 +192,8 @@ static const CfgMeta metas[] = {
|
||||
{"SHADOW_RES", "game", "game|256|512|768|1024|1536|2048", RESTART},
|
||||
{"SHADOW_SCALE", "1", "1..4", RESTART}, {"SHADOW_COMPARE", "1", "0|1", RESTART}, // vk_texture.c
|
||||
{"SHADOW_BULK_WRITE", "0", "0|1", RESTART}, // vk_memory.c shadow_write_back
|
||||
// vk_memory.c "Pages a shader stores into through the device address table"
|
||||
{"SHADOW_TABLE_STORES", "0", "0|1", RESTART}, {"SHADOW_CHECK", "0", "0|1", RESTART},
|
||||
{"SVC_SLOW_MS", "0", "0..60000", RESTART}, // kernel.c kern_svc: log services slower than this
|
||||
{"SHADOW_RANGE", "100", "50..400", RESTART}, {"CONTACT_SHADOWS", "0", "0..100", RESTART},
|
||||
{"PCSS", "0", "0..200", RESTART},
|
||||
@@ -279,7 +281,7 @@ static const CfgMeta metas[] = {
|
||||
{"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag
|
||||
// diagnostics
|
||||
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
|
||||
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
|
||||
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
|
||||
{"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame
|
||||
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
|
||||
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha
|
||||
|
||||
@@ -352,7 +352,11 @@ static int test_import(pool_t *pool) {
|
||||
vk_mem_resolve(a + 0x1000, 0x20000, &sp);
|
||||
mem_written_pages(a, pool->size, pages, 64);
|
||||
vk_mem_gpu_pending(a + 0x1000, 0x20000);
|
||||
vk_mem_shadow_sync(); // as a submit does: the part's CPU inputs go in before its GPU work
|
||||
// As a submit does: a new write-watch epoch (the copy-in reads each region's snapshot once per submission, so a
|
||||
// second sync in the same epoch would not see the CPU writes made since the first), then the part's CPU inputs
|
||||
// go in before its GPU work.
|
||||
vk_texture_new_submit();
|
||||
vk_mem_shadow_sync();
|
||||
copy_wait(sb, 0, sp.buffer, sp.offset, 0x20000);
|
||||
vk_gpu_written(a + 0x1000, 0x20000);
|
||||
GCHECK(!memcmp(m + 0x1000, s, 0x20000), "GPU copy into the imported heap range differs from the source");
|
||||
@@ -370,6 +374,7 @@ static int test_import(pool_t *pool) {
|
||||
// A GPU write noted for the part being submitted must not keep the part's CPU inputs out: a buffer the shaders
|
||||
// add to and the CPU resets (the game's lighting accumulators) depends on the reset reaching the GPU first.
|
||||
vk_mem_gpu_pending(a + 0x30000, 0x4000);
|
||||
vk_texture_new_submit();
|
||||
vk_mem_shadow_sync();
|
||||
copy_wait(sp.buffer, sp.offset, sb, 0, 0x4000);
|
||||
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
|
||||
@@ -515,6 +520,100 @@ done:
|
||||
return bad;
|
||||
}
|
||||
|
||||
// test_table_store: a submission whose one part uses the device address table and records nothing else, as a part of
|
||||
// the replay with such a shader ends: its prologue brings every pool of the table up to date (vk_mem_gpa_sync).
|
||||
static void table_part_record(VkCommandBuffer cb, void *arg) {
|
||||
(void)arg;
|
||||
vk_texture_new_submit();
|
||||
vk_mem_mirror_begin();
|
||||
vk_mem_gpa_use();
|
||||
vk_mem_mirror_end(cb, vk_sync_issued() + 1);
|
||||
}
|
||||
|
||||
static int table_part(void) {
|
||||
uint64_t fence = vk_sync_probe_submit(table_part_record, NULL);
|
||||
return vk_sync_fence_wait(fence, 5000000000ull); // its completion callbacks have run when the wait returns
|
||||
}
|
||||
|
||||
// Shadow storage: what a shader stores through the device address table lands in the pool's shadow copy and nothing
|
||||
// notes it (vk_memory.c "Pages a shader stores into"). A GPU copy into the shadow buffer that is not noted stands in
|
||||
// for the store. Word X of a page takes the stores, the CPU rewrites word Y of the same page before every part:
|
||||
// - the store made after the page's first whole copy-in is overwritten by the second, which finds it (nothing can
|
||||
// keep it);
|
||||
// - every later store stays in the shadow through the copy-ins that bring Y, and is in guest memory after the fence;
|
||||
// - a CPU write of X after that reaches the shadow.
|
||||
static int test_table_store(uint8_t *guest) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!vk_mem_shadow_storage() || !vk_gpa_wanted() || !vk_mem_shadow_table_stores()) {
|
||||
printf("probe-vkmem: stores through the device address table into a shadow copy: skipped (%s)\n",
|
||||
!vk_mem_shadow_storage() ? "no shadow storage" : !vk_gpa_wanted() ? "no device address table"
|
||||
: "NX2NAVI_SHADOW_TABLE_STORES is not 1");
|
||||
return 0;
|
||||
}
|
||||
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xB00000, .size = 0x10000,
|
||||
.game_mem = guest + 0xB00000};
|
||||
volatile uint32_t *x = (volatile uint32_t *)(pool.mem + 0x200), *y = (volatile uint32_t *)(pool.mem + 0x100);
|
||||
*x = 0, *y = 1;
|
||||
GCHECK(vk_gpa_on() && vk_pool_init(&pool), "a CPU-visible pool in the device address table");
|
||||
VkBuffer sb;
|
||||
VmaAllocation sa;
|
||||
volatile uint32_t *s = vk_staging(0x1000, &sb, &sa);
|
||||
VkMemSpan sp;
|
||||
int bad = 1;
|
||||
#define TS_CHECK(cond, ...) do { if (!(cond)) { printf("probe-vkmem: FAIL table store: "); printf(__VA_ARGS__); printf("\n"); goto done; } } while (0)
|
||||
#define TS_STORE(v) \
|
||||
do { \
|
||||
s[0] = (v); \
|
||||
vmaFlushAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); \
|
||||
TS_CHECK(vk_mem_lookup((uintptr_t)x, 4, &sp) && sp.buffer, "the pool has no buffer"); \
|
||||
copy_wait(sb, 0, sp.buffer, sp.offset, 4); \
|
||||
} while (0)
|
||||
#define TS_SHADOW(off, into) \
|
||||
do { \
|
||||
TS_CHECK(vk_mem_lookup((uintptr_t)pool.mem + (off), 4, &sp) && sp.buffer, "the pool has no buffer"); \
|
||||
copy_wait(sp.buffer, sp.offset, sb, 16, 4); \
|
||||
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); \
|
||||
(into) = s[4]; \
|
||||
} while (0)
|
||||
uint32_t sx, sy;
|
||||
*y = 2;
|
||||
TS_CHECK(table_part(), "fence timed out"); // the page's first whole copy-in: its mark
|
||||
TS_STORE(0xAAAA0001u);
|
||||
*y = 3;
|
||||
TS_CHECK(table_part(), "fence timed out"); // finds the store, overwrites it, the page keeps a base copy
|
||||
TS_SHADOW(0x200, sx);
|
||||
TS_CHECK(sx == 0, "the copy-in that found the store left X %#010x in the shadow, expected guest memory's 0", sx);
|
||||
TS_STORE(0xAAAA0002u);
|
||||
*y = 4;
|
||||
TS_CHECK(table_part(), "fence timed out"); // brings Y, keeps X; its fence writes X back
|
||||
TS_SHADOW(0x200, sx);
|
||||
TS_SHADOW(0x100, sy);
|
||||
TS_CHECK(sx == 0xAAAA0002u && sy == 4, "after a CPU write of the page the shadow holds X %#010x Y %u, expected "
|
||||
"0xaaaa0002 and 4", sx, sy);
|
||||
TS_CHECK(*x == 0xAAAA0002u && *y == 4, "guest memory holds X %#010x Y %u after the fence, expected 0xaaaa0002 and 4",
|
||||
(uint32_t)*x, (uint32_t)*y);
|
||||
*x = 0; // the CPU resets the stored word
|
||||
TS_CHECK(table_part(), "fence timed out");
|
||||
TS_SHADOW(0x200, sx);
|
||||
TS_CHECK(sx == 0 && *x == 0, "after the CPU reset X the shadow holds %#010x and guest memory %#010x", sx, (uint32_t)*x);
|
||||
TS_STORE(0xAAAA0003u); // and a store after the reset comes back as well
|
||||
TS_CHECK(table_part(), "fence timed out");
|
||||
TS_CHECK(*x == 0xAAAA0003u && *y == 4, "guest memory holds X %#010x Y %u after the second fence", (uint32_t)*x,
|
||||
(uint32_t)*y);
|
||||
g_gpu_checks += 5;
|
||||
printf("probe-vkmem: a store into a shadow page that nothing notes survives the CPU's writes of that page, reaches "
|
||||
"guest memory at the fence, and gives way to a CPU write of the same word\n");
|
||||
bad = 0;
|
||||
done:
|
||||
vk_sync_finish();
|
||||
vmaDestroyBuffer(c->vma, sb, sa);
|
||||
vk_pool_fini(&pool);
|
||||
#undef TS_CHECK
|
||||
#undef TS_STORE
|
||||
#undef TS_SHADOW
|
||||
return bad;
|
||||
}
|
||||
|
||||
// ReadFile into a watched page, buffered and unbuffered: timed, and whether write watch saw the kernel's write.
|
||||
static int test_readfile(uint8_t *dst) {
|
||||
char dir[PLAT_PATH_MAX], path[PLAT_PATH_MAX + 64];
|
||||
@@ -912,7 +1011,7 @@ static int test_gpu(void) {
|
||||
vk_mem_stats(&st);
|
||||
GCHECK(!st.imported_pools && !st.physical_pools && !st.virtual_pools, "pools left after fini");
|
||||
g_gpu_checks += 2;
|
||||
return test_gpa_read(h);
|
||||
return test_gpa_read(h) || test_table_store(h);
|
||||
}
|
||||
|
||||
int probe_vkmem(void) {
|
||||
|
||||
@@ -348,10 +348,12 @@ void vk_mem_fatal_note(char *out, size_t n); // the memory mode and the lazy w
|
||||
void vk_mem_shadow_sync(void);
|
||||
// A shader of the part being recorded reads through the device address table (vk_gpa.c), also when its descriptors
|
||||
// are reused: with shadow storage the part's prologue (vk_mem_gpa_sync) then brings every pool the table names up to
|
||||
// date with the CPU's writes, since the pools such a shader reaches are never resolved.
|
||||
// date with the CPU's writes, since the pools such a shader reaches are never resolved. fence: the submission's value,
|
||||
// at which what the shaders stored through the table goes back to guest memory.
|
||||
void vk_mem_gpa_use(void);
|
||||
void vk_mem_gpa_sync(void);
|
||||
void vk_mem_gpa_sync(uint64_t fence);
|
||||
int vk_mem_shadow_storage(void); // CPU-visible pools are shadow copies of guest memory, not imports (NX2NAVI_SHADOW)
|
||||
int vk_mem_shadow_table_stores(void); // shadow storage keeps what shaders store through the table (NX2NAVI_SHADOW_TABLE_STORES)
|
||||
void vk_mem_gpu_pending(uint64_t addr, uint64_t size);
|
||||
void vk_gpu_written(uint64_t addr, uint64_t size);
|
||||
// The same for a GPU write into a VRAM mirror, which went back to guest memory with the submission: the texture checks
|
||||
|
||||
@@ -37,8 +37,11 @@
|
||||
// NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis).
|
||||
// Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). With shadow storage the
|
||||
// pools are copies of guest memory: a part whose shaders use the table is marked (vk_exec_draw.c bind_resources) and
|
||||
// its prologue copies the pages the CPU wrote into the copies the table names (vk_memory.c vk_mem_gpa_sync). What a
|
||||
// shader stores through the table into a shadow copy is not noted as a GPU write and does not come back to guest memory.
|
||||
// its prologue copies the pages the CPU wrote into the copies the table names (vk_memory.c vk_mem_gpa_sync). There
|
||||
// the twin of a mirrored pool is its shadow copy, not guest memory, and what a shader stores through the table is not
|
||||
// noted as a GPU write: it does not come back to guest memory, and the next whole copy-in of its page overwrites it.
|
||||
// vk_memory.c "Pages a shader stores into through the device address table" has the sequence, what was measured, and
|
||||
// the switch that keeps the stores (NX2NAVI_SHADOW_TABLE_STORES).
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
|
||||
@@ -87,6 +87,16 @@ typedef struct PoolRec {
|
||||
// csb a power of two, page 0 an empty slot
|
||||
struct SBase { uint64_t page; uint8_t *base; } *sb;
|
||||
uint32_t nsb, csb;
|
||||
// shadow_lock ("Pages a shader stores into through the device address table" below): a mark per page of
|
||||
// [import_lo, import_hi), the hash of the shadow page as its last whole copy-in left it, 0 not known; the pages
|
||||
// found stored into (guest addresses), and whether the pool is on the list of pools that have such pages
|
||||
uint32_t *smark;
|
||||
uint64_t *tpg;
|
||||
uint32_t ntpg, ctpg;
|
||||
uint8_t tlisted;
|
||||
uint64_t t_over; // whole copy-ins that found such a store in the page they overwrote, so far
|
||||
uint64_t t_over_log; // t_over at the last log line
|
||||
uint64_t s_in; // NX2NAVI_SHADOW_CHECK: bytes copied in whole since the last log line
|
||||
// NX2NAVI_TRACE=perf per-pool mirror traffic since the last mirror_pool_log, replay thread (the log reads and clears
|
||||
// them without a lock: a count lost to the race only blurs a diagnostic)
|
||||
uint64_t st_pages, st_flush, st_wb, st_merge; // bytes: written pages and flush ranges copied up, GPU writes back
|
||||
@@ -592,6 +602,7 @@ static void dirty_frame(uint64_t frame);
|
||||
static void mirror_log(uint64_t frame);
|
||||
static void mirror_pool_log(uint64_t frame);
|
||||
static void shadow_log(uint64_t frame);
|
||||
static int shadow_check_on(void);
|
||||
static int dead_pending(void);
|
||||
|
||||
static void vram_log(uint64_t frame);
|
||||
@@ -607,6 +618,8 @@ void vk_mem_census(uint64_t frame) {
|
||||
if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame);
|
||||
if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800)
|
||||
mirror_log(frame), shadow_log(frame); // mirror_log also logs the write-watch line
|
||||
else if (shadow_check_on() && frame % 60 == 0)
|
||||
shadow_log(frame); // NX2NAVI_SHADOW_CHECK: the shadow lines at that rate without the perf trace
|
||||
if (frame == 1200) { // loading is over: the first window starts here
|
||||
for (int c = 0; c < VK_MCLS_N; c++)
|
||||
for (int u = 0; u < VK_MUSE_N; u++) plat_xchg64(&traffic[c][u], 0);
|
||||
@@ -1030,6 +1043,46 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
|
||||
// GPU's when the fence does. Whole-range copies both ways lost the CPU's bytes: Ocarina writes a lighting parameter
|
||||
// into a buffer whose binding its compute passes write every frame, the GPU only ever read zero there, the lighting
|
||||
// coefficients and then the volumetric fog came out NaN, and the fog pass drew the 3D scene black.
|
||||
// - The game must not get ahead of the write-backs. With an import the GPU's bytes are in guest memory when the GPU
|
||||
// has stored them; here they get there in a completion callback, after the fence. A CPU write that should follow
|
||||
// them and lands before that callback is overtaken: the write-back puts the GPU's older bytes over it and moves the
|
||||
// base copy, and a rewrite with the base copy's value (a reset to zero) is no difference at any time. So the wait
|
||||
// after a frame's first submission covers the write-backs with shadow storage (vk_sync.c "the executed timeline",
|
||||
// which tells what the game rewrites there and what a frame looked like without it).
|
||||
// - Pages a shader stores into through the device address table (vk_gpa.c): the store goes to the pool's VRAM mirror
|
||||
// and to its twin, which here is the shadow, not guest memory. No binding names the range, so nothing notes it
|
||||
// (vk_mem_gpu_pending): the page has no base copy and guest memory never gets the bytes. The game's culling passes
|
||||
// store counts and records this way into pages whose other words the CPU rewrites. For one such page P, a word X the
|
||||
// shaders store and a word Y the CPU writes:
|
||||
// part A (the pass that stores): the CPU wrote Y, so A's prologue copies P in whole (shadow_copy) and queues it
|
||||
// for the mirror (mirror_sync). X in the shadow and in the mirror is now guest memory's X, which no store ever
|
||||
// reached. A runs and stores X = v into both.
|
||||
// part B (a later part that reads X through the table), the CPU has not written P since A's write-watch snapshot:
|
||||
// B's prologue leaves P alone and its draws read v.
|
||||
// part B, the CPU has written Y again between the two snapshots: B's prologue copies P in whole and uploads it, X
|
||||
// is guest memory's stale word again in both copies, and the draws read that.
|
||||
// An import does not have the fault: its twin is guest memory, and a page copied up from there holds the store.
|
||||
// Measured (NX2NAVI_SHADOW_CHECK, a standing run outdoors, RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10): the stores
|
||||
// go into one page each of ten pools of 20 and 44 KiB, and whole copy-ins overwrite one exactly four times a frame
|
||||
// (240 in every 60 frames): part A's case, the copy that precedes the frame's stores, which does no harm. Part B's
|
||||
// second case would add to the four and was not seen in 1,800 frames; nothing proves it cannot happen where the
|
||||
// frame's timing differs.
|
||||
// NX2NAVI_SHADOW_TABLE_STORES=1 keeps the stores. The pages are found by their content: each whole copy-in leaves a
|
||||
// hash of the page it wrote (smark), and the next one hashes the shadow page first. A different hash means something
|
||||
// stored into the shadow in between, and with no base copy on the page that was a store through the table. That
|
||||
// copy-in still overwrites the store (nothing tells its bytes from the CPU's), and from then on the page keeps a
|
||||
// base copy: copy-ins take the CPU's bytes only, and the fence of every submission with a part that uses the table
|
||||
// writes the stored bytes of all such pages back to guest memory (table_wb_cb), as the fence of a noted write does.
|
||||
// Guest memory needs them: a word the CPU resets each frame to the value it had before the stores would otherwise
|
||||
// never differ from its base, and the reset would never reach the GPU.
|
||||
// It is off by default, for three reasons. The fault it removes was not seen. Its hashes cost 1.8 ms a frame of
|
||||
// the replay there (copy_marked). And it puts these pages under the rule of the bullet above: a count the CPU
|
||||
// resets before that write-back has landed keeps counting from its old value, where the whole copy-in of today
|
||||
// resets it every time; whether the game resets any of these words from the CPU is not known.
|
||||
// NX2NAVI_SHADOW_CHECK=1 counts, with the shadow line and at its rate every 60 frames (shadow_table_log): the whole
|
||||
// copy-ins that overwrote a store through the table, per pool; the write-backs that moved bytes into a page the CPU
|
||||
// had written since its copy-in (cpu_wrote_since); the bytes both sides changed since their base copy (taken from the
|
||||
// CPU at a copy-in, from the GPU at a fence); what the copy-ins cost by kind.
|
||||
// NX2NAVI_SHADOW=1/0 forces the mode on or off; the default is on for Linux and Wine.
|
||||
#define SB_PAGE 4096ull
|
||||
static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock
|
||||
@@ -1041,6 +1094,22 @@ static uint64_t s_syncs, s_copied, s_queries, s_untracked, s_wb_bytes, s_ticks;
|
||||
static uint64_t s_wb_seen, s_wb_ticks; // shadow_lock: bytes the fence write-backs compared, their time
|
||||
static uint64_t s_q_ticks; // shadow_lock: the copy-ins' write-watch queries
|
||||
static int wb_no_reup(void);
|
||||
// Pages a shader stores into through the device address table. The two switches are read in mirror_mode.
|
||||
static int s_learn; // NX2NAVI_SHADOW_TABLE_STORES: such pages get a base copy and a write-back
|
||||
static int s_check; // NX2NAVI_SHADOW_CHECK: the counts below marked "check"
|
||||
static PoolRec **tpools; // shadow_lock: the pools that have such pages
|
||||
static int ntpools, ctpools;
|
||||
static uint64_t s_tab_pages; // shadow_lock: such pages, all pools
|
||||
// shadow_lock, since the last log:
|
||||
static uint64_t s_tab_over; // whole copy-ins that overwrote a store through the table
|
||||
static uint64_t s_tab_wb, s_tab_wb_seen, s_tab_wb_ticks, s_tab_wb_jobs; // table_wb_cb: bytes moved and compared, time, runs
|
||||
static uint64_t s_marked, s_mark_ticks; // pages copied in whole with a mark; check: the time of those copies
|
||||
static uint64_t s_plain_ticks; // check: the time of the whole copies made without marks
|
||||
static uint64_t s_both_cpu, s_both_gpu; // check: bytes both sides changed, taken from the CPU / from the GPU
|
||||
static uint64_t s_in_merge, s_in_nomirror; // check: bytes copied in by merge_page; whole, into pools without a mirror
|
||||
static uint64_t s_gpa_ticks, s_gpa_syncs; // vk_mem_gpa_sync: its time (part of s_ticks) and its runs
|
||||
static uint64_t s_t_over_all; // shadow_lock: s_tab_over since the start
|
||||
static uint64_t s_late_pages, s_late_bytes; // shadow_lock, check: write-backs into pages the CPU wrote since their copy-in
|
||||
|
||||
static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
|
||||
r->import_lo = addr & ~0xfffull;
|
||||
@@ -1184,12 +1253,91 @@ static uint64_t merge_page(uint8_t *dst, const uint8_t *src, uint8_t *b, uint64_
|
||||
return moved;
|
||||
}
|
||||
|
||||
// ---- pages a shader stores into through the device address table ("shadow storage" above). shadow_lock held.
|
||||
|
||||
// The mark of a 4 KiB page: a hash of its bytes, never 0. Four sums that do not wait for each other.
|
||||
static uint32_t page_mark(const uint8_t *p) {
|
||||
uint64_t h[4] = {0x9E3779B97F4A7C15ull, 0xC2B2AE3D27D4EB4Full, 0x165667B19E3779F9ull, 0x27D4EB2F165667C5ull};
|
||||
for (uint64_t i = 0; i < SB_PAGE; i += 32) {
|
||||
uint64_t w[4];
|
||||
memcpy(w, p + i, 32);
|
||||
for (int k = 0; k < 4; k++) {
|
||||
h[k] = (h[k] ^ w[k]) * 0xFF51AFD7ED558CCDull;
|
||||
h[k] = h[k] << 29 | h[k] >> 35;
|
||||
}
|
||||
}
|
||||
uint64_t x = (h[0] ^ h[1] << 13 ^ h[2] << 27 ^ h[3] << 41 ^ h[1] >> 51 ^ h[2] >> 37 ^ h[3] >> 23) * 0xC4CEB9FE1A85EC53ull;
|
||||
uint32_t m = (uint32_t)(x >> 32);
|
||||
return m ? m : 1;
|
||||
}
|
||||
|
||||
// Guest page pg of r was stored into through the table: it keeps a base copy from here on, taken from the shadow as
|
||||
// the whole copy-in just left it (guest memory's bytes), and table_wb_cb writes its stores back.
|
||||
static void table_page_add(PoolRec *r, uint64_t pg) {
|
||||
sbase_make(r, pg);
|
||||
if (r->ntpg == r->ctpg) {
|
||||
uint32_t cap = r->ctpg ? r->ctpg * 2 : 16;
|
||||
uint64_t *t = realloc(r->tpg, cap * sizeof *t);
|
||||
if (!t) rt_fatal("vk: out of memory for the shadow's table pages");
|
||||
r->tpg = t, r->ctpg = cap;
|
||||
}
|
||||
r->tpg[r->ntpg++] = pg;
|
||||
s_tab_pages++;
|
||||
if (r->tlisted) return;
|
||||
if (ntpools == ctpools) grow((void **)&tpools, &ctpools, sizeof *tpools);
|
||||
tpools[ntpools++] = r;
|
||||
r->tlisted = 1;
|
||||
}
|
||||
|
||||
// NX2NAVI_SHADOW_CHECK: the bytes of [0, n) that both sides changed since the base copy b and that now differ, which a
|
||||
// merge of src into dst is about to settle in src's favour.
|
||||
static uint64_t both_changed(const uint8_t *dst, const uint8_t *src, const uint8_t *b, uint64_t n) {
|
||||
uint64_t k = 0;
|
||||
for (uint64_t i = 0; i < n; i++) k += (src[i] != b[i]) & (dst[i] != b[i]) & (dst[i] != src[i]);
|
||||
return k;
|
||||
}
|
||||
|
||||
// Copies the whole guest pages [a, e) of r into its shadow, none of which has a base copy. Each page's mark tells
|
||||
// whether the shadow page still holds what the last such copy left: if not, a shader stored into it through the table
|
||||
// meanwhile, and this copy overwrites that store ("Pages a shader stores into" above says why it cannot be kept, and
|
||||
// what the page gets from here on).
|
||||
// Only with NX2NAVI_SHADOW_TABLE_STORES or NX2NAVI_SHADOW_CHECK (shadow_copy). A mark is two hashes of the page, one
|
||||
// of them of bytes the copy would not have read: outdoors, where 2,400 pages a frame are copied in whole, the marks
|
||||
// cost 1.8 ms a frame of the replay (3.09 against 1.32 ms of copy-ins, 67 against 75 fps, RTX 5090 with
|
||||
// NX2NAVI_SHADOW=1, 2026-11-10).
|
||||
static void copy_marked(PoolRec *r, uint64_t a, uint64_t e) {
|
||||
if (!r->smark) {
|
||||
r->smark = calloc((r->import_hi - r->import_lo) / SB_PAGE, sizeof *r->smark);
|
||||
if (!r->smark) rt_fatal("vk: out of memory for the shadow's page marks");
|
||||
}
|
||||
for (uint64_t pg = a; pg < e; pg += SB_PAGE) {
|
||||
uint8_t *s = r->smap + (pg - r->import_lo);
|
||||
uint32_t *m = &r->smark[(pg - r->import_lo) / SB_PAGE];
|
||||
int stored = *m && page_mark(s) != *m;
|
||||
memcpy(s, (const void *)(uintptr_t)pg, SB_PAGE);
|
||||
if (stored) {
|
||||
s_tab_over++, s_t_over_all++, r->t_over++;
|
||||
if (s_learn) { // merged from now on: its mark is not read again
|
||||
table_page_add(r, pg);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
*m = page_mark(s); // of the shadow, not of guest memory, which the CPU may be writing
|
||||
}
|
||||
s_marked += (e - a) / SB_PAGE;
|
||||
}
|
||||
|
||||
// vk_pool_fini, after the pool left the index: off the touched list before its record goes.
|
||||
static void shadow_forget(PoolRec *r) {
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
for (int i = 0; i < nstouch; i++)
|
||||
if (stouch[i] == r) { stouch[i] = stouch[--nstouch]; break; }
|
||||
for (int i = 0; r->tlisted && i < ntpools; i++)
|
||||
if (tpools[i] == r) { tpools[i] = tpools[--ntpools]; break; }
|
||||
s_live_n--, s_live_bytes -= (int64_t)(r->import_hi - r->import_lo);
|
||||
s_tab_pages -= r->ntpg;
|
||||
free(r->tpg), free(r->smark);
|
||||
r->tpg = NULL, r->smark = NULL, r->ntpg = r->ctpg = 0, r->tlisted = 0;
|
||||
sbase_free(r);
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
@@ -1205,12 +1353,25 @@ static void shadow_copy(PoolRec *r, uint64_t a, uint64_t b) {
|
||||
if (e > b) e = b;
|
||||
if (e == a) { // a starts on such a page: its part of [a, b)
|
||||
uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < b ? pg + SB_PAGE : b;
|
||||
s_copied += merge_page(r->smap + (a - r->import_lo), (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a);
|
||||
uint8_t *s = r->smap + (a - r->import_lo);
|
||||
if (s_check) s_both_cpu += both_changed(s, (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a);
|
||||
uint64_t n = merge_page(s, (const uint8_t *)(uintptr_t)a, base + (a - pg), pe - a);
|
||||
s_copied += n, s_in_merge += n;
|
||||
a = pe;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, e - a);
|
||||
// The copies are of whole pages (shadow_sync_pool); the marks need the device address table, whose stores
|
||||
// they find, and one of the two switches.
|
||||
uint64_t t0 = s_check ? qpc() : 0;
|
||||
int marks = (s_learn || s_check) && r->gpa_rec && !((a | e) & (SB_PAGE - 1));
|
||||
if (marks) copy_marked(r, a, e);
|
||||
else memcpy(r->smap + (a - r->import_lo), (const void *)(uintptr_t)a, e - a);
|
||||
if (s_check) {
|
||||
*(marks ? &s_mark_ticks : &s_plain_ticks) += qpc() - t0;
|
||||
if (r->mstate != MS_ON) s_in_nomirror += e - a;
|
||||
r->s_in += e - a;
|
||||
}
|
||||
s_copied += e - a;
|
||||
a = e;
|
||||
}
|
||||
@@ -1334,6 +1495,18 @@ static int shadow_bulk_write(void) {
|
||||
return on;
|
||||
}
|
||||
|
||||
// NX2NAVI_SHADOW_CHECK: the CPU wrote guest page pg after the copy-in that sgen stands for: a write-watch snapshot
|
||||
// stamped it later, or its write-watch bit is set and no snapshot has read it yet. A write-back that moves bytes into
|
||||
// such a page may have come after a CPU write it should have come before ("The game must not get ahead of the
|
||||
// write-backs" above); a page of which the CPU wrote other bytes counts too.
|
||||
// No gen_lock, as lazy_clean: a stamp that lands meanwhile only blurs a count.
|
||||
static int cpu_wrote_since(uint64_t pg, uint32_t sgen) {
|
||||
const uint32_t *ch = __atomic_load_n(&cgen_chunk[pg >> GEN_CHUNK_SHIFT], __ATOMIC_ACQUIRE);
|
||||
if (ch && __atomic_load_n(&ch[(pg >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)], __ATOMIC_RELAXED) > sgen) return 1;
|
||||
uint64_t p[1], n = mem_written_peek(pg, SB_PAGE, p, 1);
|
||||
return n && n != UINT64_MAX;
|
||||
}
|
||||
|
||||
// Completion thread: the GPU's bytes of [addr, addr+size) from the shadow into guest memory (merge_page against the
|
||||
// base copies; a page without one whole). Returns the bytes moved.
|
||||
static uint64_t shadow_write_back(uint64_t addr, uint64_t size) {
|
||||
@@ -1350,8 +1523,12 @@ static uint64_t shadow_write_back(uint64_t addr, uint64_t size) {
|
||||
for (uint64_t a = addr, end = addr + size; a < end;) {
|
||||
uint64_t pg = a & ~(SB_PAGE - 1), pe = pg + SB_PAGE < end ? pg + SB_PAGE : end;
|
||||
uint8_t *base = sbase_find(r, pg), *s = r->smap + (a - r->import_lo);
|
||||
if (base) moved += merge_page((uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a);
|
||||
if (base && s_check) s_both_gpu += both_changed((const uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a);
|
||||
int late = base && s_check && cpu_wrote_since(pg, r->sgen); // before the merge's own stores into the page
|
||||
uint64_t n = 0;
|
||||
if (base) moved += n = merge_page((uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a);
|
||||
else memcpy((void *)(uintptr_t)a, s, pe - a), moved += pe - a; // a write never noted as pending
|
||||
if (late && n) s_late_pages++, s_late_bytes += n;
|
||||
a = pe;
|
||||
}
|
||||
s_wb_bytes += moved, s_wb_seen += size, s_wb_ticks += qpc() - t0;
|
||||
@@ -1361,6 +1538,47 @@ static uint64_t shadow_write_back(uint64_t addr, uint64_t size) {
|
||||
return moved;
|
||||
}
|
||||
|
||||
// Completion thread, at the fence of a submission with a part that uses the device address table (vk_mem_gpa_sync):
|
||||
// what the shaders stored into the pages found so far ("Pages a shader stores into" above) goes from the shadow into
|
||||
// guest memory, the bytes that differ from the page's base copy. A page whose shadow equals its base is passed over
|
||||
// before guest memory is touched, so a frozen region stays frozen. The pages that changed are stamped as a mirror's
|
||||
// write-back is (swb_run): the stores are in the mirror already.
|
||||
static void table_wb_cb(void *arg) {
|
||||
(void)arg;
|
||||
static uint64_t *wrote; // this thread only: the pages that changed
|
||||
static int cwrote;
|
||||
int nwrote = 0, bulk = shadow_bulk_write(), back = wb_no_reup(); // before the locks: their first calls log
|
||||
uint64_t t0 = qpc(), moved = 0, seen = 0;
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
for (int i = 0; i < ntpools; i++) {
|
||||
PoolRec *r = tpools[i];
|
||||
for (uint32_t k = 0; k < r->ntpg; k++) {
|
||||
uint64_t pg = r->tpg[k];
|
||||
uint8_t *base = sbase_find(r, pg), *s = r->smap + (pg - r->import_lo);
|
||||
seen += SB_PAGE;
|
||||
if (!base || !memcmp(s, base, SB_PAGE)) continue;
|
||||
if (mem_frozen(pg)) mem_thaw(pg, SB_PAGE);
|
||||
if (bulk) plat_mem_prepare_write(pg, SB_PAGE);
|
||||
if (s_check) s_both_gpu += both_changed((const uint8_t *)(uintptr_t)pg, s, base, SB_PAGE);
|
||||
uint64_t n = merge_page((uint8_t *)(uintptr_t)pg, s, base, SB_PAGE);
|
||||
if (!n) continue; // the store came after the compare above: the next fence takes it
|
||||
moved += n;
|
||||
if (nvn_gpu_only(pg)) continue;
|
||||
if (nwrote == cwrote) grow((void **)&wrote, &cwrote, sizeof *wrote);
|
||||
wrote[nwrote++] = pg;
|
||||
}
|
||||
}
|
||||
s_tab_wb += moved, s_tab_wb_seen += seen, s_tab_wb_jobs++;
|
||||
if (nwrote) {
|
||||
plat_mutex_lock(&gen_lock);
|
||||
uint32_t g = ++gen_now;
|
||||
for (int i = 0; i < nwrote; i++) stamp_page_as(wrote[i], g, back);
|
||||
plat_mutex_unlock(&gen_lock);
|
||||
}
|
||||
s_tab_wb_ticks += qpc() - t0;
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
|
||||
static void gpu_written(uint64_t addr, uint64_t size, int back) {
|
||||
// back: the bytes are in a VRAM mirror, and its write-back brings them into the shadow; that submission's
|
||||
// shadow_wb_cb, after the replay's own callbacks, then copies them into guest memory.
|
||||
@@ -1398,8 +1616,92 @@ int vk_mem_mirrored(uint64_t addr, uint64_t size) {
|
||||
return r->mstate == MS_ON && addr >= r->import_lo && addr + size <= r->import_hi;
|
||||
}
|
||||
|
||||
// shadow_lock held, with the shadow line: the pages a shader stores into through the device address table ("shadow
|
||||
// storage" above), and with NX2NAVI_SHADOW_CHECK (pool_lock held shared too, for the index) what the copy-ins did and
|
||||
// cost and the pools whose stores were overwritten most since the last line.
|
||||
#define TABLE_LOG_TOP 6
|
||||
static void shadow_table_log(uint64_t frame) {
|
||||
rt_log("mem: shadow table stores at frame %llu (%s): %llu pages in %d pools keep a base copy; since the last: %llu whole "
|
||||
"copy-ins overwrote a store (%llu so far), %llu pages copied in with a mark; written back %.3f MiB of %.1f "
|
||||
"compared in %llu fence jobs, %.2f ms",
|
||||
(unsigned long long)frame, s_learn ? "NX2NAVI_SHADOW_TABLE_STORES=1: kept from the second copy-in on" : "overwritten",
|
||||
(unsigned long long)s_tab_pages, ntpools, (unsigned long long)s_tab_over, (unsigned long long)s_t_over_all,
|
||||
(unsigned long long)s_marked, s_tab_wb / 1048576.0, s_tab_wb_seen / 1048576.0, (unsigned long long)s_tab_wb_jobs,
|
||||
qpc_ms(s_tab_wb_ticks));
|
||||
if (s_check) {
|
||||
rt_log("mem: shadow check at frame %llu: copied in whole %.2f ms with marks, %.2f ms without; %.1f MiB by merge, "
|
||||
"%.1f MiB whole into pools without a mirror; %llu table syncs %.2f ms; bytes both sides changed: %llu taken "
|
||||
"from the CPU at a copy-in, %llu from the GPU at a fence; write-backs into %llu pages the CPU had written "
|
||||
"since their copy-in, %llu bytes",
|
||||
(unsigned long long)frame, qpc_ms(s_mark_ticks), qpc_ms(s_plain_ticks), s_in_merge / 1048576.0,
|
||||
s_in_nomirror / 1048576.0, (unsigned long long)s_gpa_syncs, qpc_ms(s_gpa_ticks),
|
||||
(unsigned long long)s_both_cpu, (unsigned long long)s_both_gpu, (unsigned long long)s_late_pages,
|
||||
(unsigned long long)s_late_bytes);
|
||||
PoolRec *top[TABLE_LOG_TOP];
|
||||
int nt = 0;
|
||||
for (int i = 0; i < pool_index.n; i++) {
|
||||
PoolRec *r = pool_index.e[i].buf;
|
||||
uint64_t d = r->kind == POOL_IMPORT ? r->t_over - r->t_over_log : 0;
|
||||
int k = nt;
|
||||
while (k > 0 && top[k - 1]->t_over - top[k - 1]->t_over_log < d) k--;
|
||||
if (!d || k >= TABLE_LOG_TOP) continue;
|
||||
int last = nt < TABLE_LOG_TOP ? nt : TABLE_LOG_TOP - 1;
|
||||
memmove(top + k + 1, top + k, (size_t)(last - k) * sizeof *top);
|
||||
top[k] = r;
|
||||
if (nt < TABLE_LOG_TOP) nt++;
|
||||
}
|
||||
for (int i = 0; i < nt; i++) {
|
||||
char f[160];
|
||||
flag_names(top[i]->flags, f, sizeof f);
|
||||
rt_log("mem: shadow check: pool %#llx+%#llx (%s, %s): %llu whole copy-ins overwrote a store (%llu so far), "
|
||||
"%u of its pages keep a base copy for it",
|
||||
(unsigned long long)top[i]->lo, (unsigned long long)top[i]->size, f,
|
||||
top[i]->mstate == MS_ON ? "mirrored" : "no mirror",
|
||||
(unsigned long long)(top[i]->t_over - top[i]->t_over_log), (unsigned long long)top[i]->t_over, top[i]->ntpg);
|
||||
}
|
||||
// the pools that took the most whole pages, and what all pools of each flag set took
|
||||
struct { uint32_t flags; uint64_t bytes; int n; } cls[16];
|
||||
int ncls = 0;
|
||||
nt = 0;
|
||||
for (int i = 0; i < pool_index.n; i++) {
|
||||
PoolRec *r = pool_index.e[i].buf;
|
||||
if (r->kind != POOL_IMPORT || !r->s_in) continue;
|
||||
int c = 0, k = nt;
|
||||
while (c < ncls && cls[c].flags != r->flags) c++;
|
||||
if (c == ncls && ncls < 16) cls[ncls++] = (__typeof__(cls[0])){r->flags, 0, 0};
|
||||
if (c < ncls) cls[c].bytes += r->s_in, cls[c].n++;
|
||||
while (k > 0 && top[k - 1]->s_in < r->s_in) k--;
|
||||
if (k >= TABLE_LOG_TOP) continue;
|
||||
int last = nt < TABLE_LOG_TOP ? nt : TABLE_LOG_TOP - 1;
|
||||
memmove(top + k + 1, top + k, (size_t)(last - k) * sizeof *top);
|
||||
top[k] = r;
|
||||
if (nt < TABLE_LOG_TOP) nt++;
|
||||
}
|
||||
for (int i = 0; i < ncls; i++) {
|
||||
char f[160];
|
||||
flag_names(cls[i].flags, f, sizeof f);
|
||||
rt_log("mem: shadow check: copied in whole into %d pools %s: %.2f MiB", cls[i].n, f, cls[i].bytes / 1048576.0);
|
||||
}
|
||||
for (int i = 0; i < nt; i++) {
|
||||
char f[160];
|
||||
flag_names(top[i]->flags, f, sizeof f);
|
||||
rt_log("mem: shadow check: pool %#llx+%#llx (%s, %s, %s): %.2f MiB copied in whole",
|
||||
(unsigned long long)top[i]->lo, (unsigned long long)top[i]->size, f,
|
||||
top[i]->mstate == MS_ON ? "mirrored" : "no mirror", top[i]->used ? "resolved by the GPU" : "never resolved",
|
||||
top[i]->s_in / 1048576.0);
|
||||
}
|
||||
for (int i = 0; i < pool_index.n; i++) {
|
||||
PoolRec *r = pool_index.e[i].buf;
|
||||
if (r->kind == POOL_IMPORT) r->t_over_log = r->t_over, r->s_in = 0;
|
||||
}
|
||||
}
|
||||
s_tab_over = s_tab_wb = s_tab_wb_seen = s_tab_wb_ticks = s_tab_wb_jobs = s_marked = s_mark_ticks = s_plain_ticks = 0;
|
||||
s_both_cpu = s_both_gpu = s_in_merge = s_in_nomirror = s_gpa_ticks = s_gpa_syncs = s_late_pages = s_late_bytes = 0;
|
||||
}
|
||||
|
||||
static void shadow_log(uint64_t frame) {
|
||||
if (!shadow_on) return;
|
||||
if (s_check) plat_mutex_lock_shared(&pool_lock); // before shadow_lock: shadow_table_log walks the index
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
rt_log("mem: shadow at frame %llu: %lld pools %.1f MiB; since the last: %llu pool syncs, %llu queries, %.1f MiB "
|
||||
"copied in (%llu untracked), %.2f ms (queries %.2f); %.1f MiB written back of %.1f MiB the GPU may have written, %.2f ms; %llu "
|
||||
@@ -1408,7 +1710,9 @@ static void shadow_log(uint64_t frame) {
|
||||
(unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, qpc_ms(s_ticks),
|
||||
qpc_ms(s_q_ticks), s_wb_bytes / 1048576.0, s_wb_seen / 1048576.0, qpc_ms(s_wb_ticks), (unsigned long long)s_sb_pages);
|
||||
s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = s_wb_seen = s_wb_ticks = s_q_ticks = 0;
|
||||
if (s_learn || s_check) shadow_table_log(frame);
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
if (s_check) plat_mutex_unlock_shared(&pool_lock);
|
||||
}
|
||||
|
||||
void vk_mem_dirty_submit(void) {
|
||||
@@ -1657,6 +1961,14 @@ static void mirror_mode(int discrete) {
|
||||
#endif
|
||||
if (shadow_on) rt_log("mem: shadow storage: CPU-visible pools are host-visible copies of guest memory, not imports%s",
|
||||
sh && *sh ? " (NX2NAVI_SHADOW)" : "");
|
||||
s_learn = shadow_on && rt_knob("SHADOW_TABLE_STORES", "0") == 1;
|
||||
s_check = shadow_on && cfg_bool("NX2NAVI_SHADOW_CHECK");
|
||||
if (shadow_on)
|
||||
rt_log("mem: what shaders store through the device address table into a shadow copy %s "
|
||||
"(NX2NAVI_SHADOW_TABLE_STORES)%s",
|
||||
s_learn ? "is kept through the copy-ins and written back to guest memory, from the second copy-in of its page on"
|
||||
: "is overwritten by the next whole copy-in of its page",
|
||||
s_check ? "; NX2NAVI_SHADOW_CHECK counts the overwrites" : "");
|
||||
rt_log("mem: mode %s (%s): %s", mode == 2 ? "verify" : mode ? "mirror" : "import", why,
|
||||
mode ? "pools used as buffers get a device-local mirror" : "every CPU-visible pool is imported host memory");
|
||||
// Import is the integrated-GPU mode. On a discrete card every draw reads the pools over PCIe (the comment above
|
||||
@@ -2071,13 +2383,39 @@ static void mirror_sync(PoolRec *r) {
|
||||
static int gpa_part_used; // replay thread: the part being recorded uses the table
|
||||
void vk_mem_gpa_use(void) { gpa_part_used = 1; }
|
||||
int vk_mem_shadow_storage(void) { return shadow_on; }
|
||||
static int shadow_check_on(void) { return s_check; }
|
||||
int vk_mem_shadow_table_stores(void) { return s_learn; }
|
||||
|
||||
static int gpa_shadow_pool(const PoolRec *r) { return r->kind == POOL_IMPORT && r->gpa_rec && r->smap; }
|
||||
|
||||
void vk_mem_gpa_sync(void) {
|
||||
// gen_lock held: nothing of the pool was stamped since its shadow's last copy-in. Its regions have this epoch's
|
||||
// write-watch snapshot (vk_mem_gpa_sync takes them for all pools at once), none of them is untracked, and none holds a
|
||||
// stamp newer than the shadow's sgen, which then stays as it is. With cpu the stamps are the CPU-side tables', as
|
||||
// shadow_sync_pool reads them. A table sync ran shadow_sync_pool for every pool at every part that uses the table,
|
||||
// 10,700 times a frame outdoors (3,772 pools, about three such parts), each with its own timer reads, snapshot walk
|
||||
// and region loop; with this test 1,500 pool syncs a frame are left, and the copy-ins went from 1.32 to 1.01 ms a
|
||||
// frame (RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10).
|
||||
static int shadow_pool_clean(const PoolRec *r, uint32_t epoch, int cpu) {
|
||||
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
|
||||
if (!epoch || r1 >= SNAP_REGIONS) return 0;
|
||||
const uint32_t *mx = cpu ? creg_max : reg_max;
|
||||
for (uint64_t k = r0; k <= r1; k++)
|
||||
if (snap_at[k] != epoch || snap_untracked[k] || mx[k] > r->sgen) return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
void vk_mem_gpa_sync(uint64_t fence) {
|
||||
if (!gpa_part_used) return;
|
||||
gpa_part_used = 0;
|
||||
if (!shadow_on) return;
|
||||
// What this part's shaders store through the table goes back to guest memory at the submission's fence, once a
|
||||
// submission however many of its parts use the table (table_wb_cb). Asked before the copy-ins below: the part
|
||||
// whose copy-ins find the first such page leaves it to the next part that uses the table.
|
||||
static uint64_t wb_fence; // replay thread: the last fence that got the job
|
||||
if (s_learn && fence && fence != wb_fence && __atomic_load_n(&ntpools, __ATOMIC_RELAXED)) {
|
||||
wb_fence = fence;
|
||||
vk_completion_add(fence, table_wb_cb, NULL);
|
||||
}
|
||||
uint64_t t0 = qpc(), run0 = 0, run1 = 0;
|
||||
uint32_t epoch = vk_texture_epoch();
|
||||
int cpu = vk_mirror_on && wb_no_reup(), have = 0; // before the locks: its first call logs
|
||||
@@ -2101,7 +2439,7 @@ void vk_mem_gpa_sync(void) {
|
||||
if (have) s_queries += snap_regions(run0, run1, epoch);
|
||||
for (int i = 0; i < pool_index.n; i++) {
|
||||
PoolRec *r = pool_index.e[i].buf;
|
||||
if (gpa_shadow_pool(r)) shadow_sync_pool(r, epoch, cpu), s_syncs++;
|
||||
if (gpa_shadow_pool(r) && !shadow_pool_clean(r, epoch, cpu)) shadow_sync_pool(r, epoch, cpu), s_syncs++;
|
||||
}
|
||||
plat_mutex_unlock(&gen_lock);
|
||||
// The uploads read the shadows just brought up to date, and the prologue records them ahead of the part, after
|
||||
@@ -2110,6 +2448,7 @@ void vk_mem_gpa_sync(void) {
|
||||
PoolRec *r = pool_index.e[i].buf;
|
||||
if (gpa_shadow_pool(r) && r->mstate == MS_ON && r->acc_sub != msub) mirror_sync(r);
|
||||
}
|
||||
s_gpa_ticks += qpc() - t0, s_gpa_syncs++;
|
||||
s_ticks += qpc() - t0;
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
plat_mutex_unlock_shared(&pool_lock);
|
||||
|
||||
@@ -82,12 +82,35 @@ static int hold_max = -1;
|
||||
// write-backs follow in a part of their own, which signals the timeline as before. A submission without that split
|
||||
// signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among
|
||||
// them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's.
|
||||
// Not with shadow storage (vk_memory.c): there the game does need the write-backs before it goes on. With an import a
|
||||
// write-back stores into guest memory on the GPU, right behind the passes. With a shadow copy it stores into the copy,
|
||||
// and the bytes reach guest memory in the fence's completion callback (shadow_wb_cb), after the write-back part, the
|
||||
// fence and the callbacks queued before it, while the game is already rewriting for its next frame what those passes
|
||||
// wrote. A write-back that lands after such a rewrite puts the GPU's older bytes over it in guest memory and moves the
|
||||
// page's base copy along, so the next copy-in finds nothing the CPU changed; and a rewrite with the value the base
|
||||
// copy holds (a reset to zero) is no difference at any time. The GPU then goes on from its own last output, as the
|
||||
// game's lighting accumulators do, which the shaders add to and the CPU resets.
|
||||
// Measured: with the split the whole 3D scene came out brighter for single frames (3.8 of 255, the interface as it
|
||||
// was), each of them a frame whose main work was held at its CPU event wait and replayed late, together with the next
|
||||
// frame's: standing outdoors, 695, 624, 443, 323 and 2 of 1,800 frames in five runs, and none in eight runs without
|
||||
// the split (RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10). A player on Linux (RX 7800 XT, RADV) saw the open field lit
|
||||
// in one frame and flat in the next; that it is the same fault is not confirmed on their machine. Which words the
|
||||
// write-backs overtook was not traced: NX2NAVI_SHADOW_CHECK counts no byte that both sides changed to different
|
||||
// values, which fits a reset to the base copy's value.
|
||||
// NX2NAVI_WB_SPLIT=2 keeps the split with shadow storage too, to measure that fault.
|
||||
static VkSemaphore exec_tl;
|
||||
static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl
|
||||
static int wb_split = -1;
|
||||
|
||||
int vk_sync_wb_split(void) {
|
||||
if (wb_split < 0) wb_split = cfg_bool("NX2NAVI_WB_SPLIT");
|
||||
if (wb_split < 0) {
|
||||
const char *e = cfg_get("NX2NAVI_WB_SPLIT");
|
||||
int forced = e && !strcmp(e, "2");
|
||||
wb_split = cfg_bool("NX2NAVI_WB_SPLIT") && (forced || !vk_mem_shadow_storage());
|
||||
if (vk_mem_shadow_storage())
|
||||
rt_log("vk: the wait after a frame's first submission %s (shadow storage; NX2NAVI_WB_SPLIT=2 leaves them out)",
|
||||
wb_split ? "leaves out its write-backs" : "covers its write-backs and their copies into guest memory");
|
||||
}
|
||||
return wb_split && exec_tl != VK_NULL_HANDLE;
|
||||
}
|
||||
|
||||
@@ -508,7 +531,7 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
|
||||
// First brings the shadow storage up to date with the CPU's writes, which needs no command buffer.
|
||||
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
|
||||
vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted
|
||||
vk_mem_gpa_sync(); // a part whose shaders read through the device address table never resolved the pools they reach
|
||||
vk_mem_gpa_sync(fence); // a part whose shaders read through the device address table never resolved the pools they reach
|
||||
vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots
|
||||
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
|
||||
int i = claim_slot(c, fence); // before reading slots: the claim may move it
|
||||
|
||||
Reference in New Issue
Block a user