Hand a pool's device address record to its new mirror by a store the GPU runs after the mirror's first copy.

The table still named a new mirror at record time, as the pages' twin: parts already handed to the queue wrote
through the table into the new block before its first copy, the same early access the delayed read was made to stop.
The reads then moved to the mirror some submissions later by a second change that took a second record, cleared its
note before it knew the record was there, and until it ran left the table's readers on guest memory, without what
other shaders had written into the mirror through a buffer binding.

Nothing is changed when the mirror is made now. The prologue that holds the mirror's first copy also holds a store
into the pool's record, after the copy: what was queued before it reads and writes guest memory through the table as
if no mirror existed, what comes after reads the mirror and writes both. No second record, no second change, no
window in which the table is behind. gpa_late = 0 still changes the table at once, as up to 0.9.2.

--probe-vkmem makes a mirror in a recorded submission and reads the table's own memory back: the page names the
mirror with guest memory as the twin, and the mirror holds the pool's bytes. tests\gpa_check.py runs the store as the
GPU would, before and after. A table with no record left now says so once in the log.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-12 22:18:14 -05:00
parent a8639f1cb6
commit e548180da2
6 changed files with 227 additions and 88 deletions

View File

@@ -483,6 +483,76 @@ static int test_remade(uint8_t *guest) {
return 0;
}
// A pool in the device address table gets its VRAM mirror while a guest submission is recorded, as on a draw's first
// use of it as a buffer (vk_memory.c mirror_create). The table takes the mirror up by a store the GPU runs after the
// mirror's first copy (vk_gpa.c vk_gpa_flip_cmd), so nothing is changed when the mirror is made; read from the table's
// own memory once the submission is done, the pool's page reads the mirror and writes guest memory as well, and the
// mirror holds the pool's bytes.
static void mirror_use_record(VkCommandBuffer cb, void *arg) {
vk_texture_new_submit();
vk_mem_mirror_begin();
vk_mem_use(arg);
vk_mem_mirror_end(cb, vk_sync_issued() + 1);
}
static int test_table_mirror(uint8_t *guest) {
VkCtx *c = vk_ctx();
if (vk_mem_shadow_storage() || !vk_gpa_wanted()) {
printf("probe-vkmem: a new mirror in the device address table: skipped (%s)\n",
vk_mem_shadow_storage() ? "shadow storage, no mirrors" : "no device address table");
return 0;
}
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xD00000, .size = 0x10000,
.game_mem = guest + 0xD00000};
volatile uint32_t *w = (volatile uint32_t *)(pool.mem + 0x3000);
uint64_t a = (uintptr_t)w, f0 = 0, t0 = 0, f1 = 0, t1 = 0, fm = 0, tm = 0;
*w = 0x600DF00Du;
GCHECK(vk_gpa_on() && vk_pool_init(&pool), "a CPU-visible pool in the device address table");
VkBuffer sb;
VmaAllocation sa;
volatile uint32_t *s = vk_staging(0x1000, &sb, &sa);
VkMemSpan sp;
int bad = 1;
#define TM_CHECK(cond, ...) do { if (!(cond)) { printf("probe-vkmem: FAIL table mirror: "); printf(__VA_ARGS__); printf("\n"); goto done; } } while (0)
TM_CHECK(vk_mem_lookup(a, 4, &sp) && sp.buffer && sp.pool, "the pool has no buffer");
VkBuffer imported = sp.buffer;
TM_CHECK(vk_gpa_probe(a, &f0, &t0) && f0 == t0 && f0 == vk_gpa_device_address(imported) + sp.offset,
"before a mirror the table gives %#llx / %#llx for the pool's page, expected its imported buffer twice",
(unsigned long long)f0, (unsigned long long)t0);
uint64_t fence = vk_sync_submit_own(mirror_use_record, &sp);
// read while the submission may still be in flight: the table as the CPU left it when the mirror was made
int mid = vk_gpa_probe(a, &fm, &tm);
TM_CHECK(vk_sync_fence_wait(fence, 5000000000ull), "fence timed out");
if (sp.buffer == imported) {
printf("probe-vkmem: a new mirror in the device address table: skipped (the pool got no mirror)\n");
bad = 0;
goto done;
}
TM_CHECK(vk_gpa_probe(a, &f1, &t1), "the pool's page left the table");
if (vk_mem_gpa_late()) // else the table changes when the mirror is made, as up to 0.9.2
TM_CHECK(mid && (fm == f0 || fm == f1) && tm == f0, "while the submission ran the table gave %#llx / %#llx: neither "
"the state before the store (%#llx twice) nor after it", (unsigned long long)fm, (unsigned long long)tm,
(unsigned long long)f0);
TM_CHECK(f1 == vk_gpa_device_address(sp.buffer) + sp.offset, "after the submission the page reads %#llx, expected the "
"mirror's %#llx", (unsigned long long)f1, (unsigned long long)(vk_gpa_device_address(sp.buffer) + sp.offset));
TM_CHECK(t1 == f0, "after the submission the page's twin is %#llx, expected the imported buffer's %#llx",
(unsigned long long)t1, (unsigned long long)f0);
copy_wait(sp.buffer, sp.offset, sb, 0, 4);
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
TM_CHECK(s[0] == 0x600DF00Du, "the mirror holds %#010x, expected the pool's 0x600df00d", s[0]);
g_gpu_checks += 4;
printf("probe-vkmem: a pool that gets its mirror in a recorded submission: the device address table names the mirror "
"after it, with guest memory as the twin (%s)\n",
vk_mem_gpa_late() ? "by a store in queue order, after the mirror's first copy" : "changed at once, NX2NAVI_GPA_LATE=0");
bad = 0;
done:
vk_sync_finish();
vmaDestroyBuffer(c->vma, sb, sa);
vk_pool_fini(&pool);
#undef TM_CHECK
return bad;
}
// test_table_sync, test_table_store: a guest submission of one part that records nothing. bound: its record binds the
// device address table, as the replay of a draw whose shader reads through the table does (vk_exec_draw.c ssbo_info),
// so its prologue brings every shadow of the table up to date (vk_memory.c vk_mem_shadow_sync).
@@ -1366,7 +1436,7 @@ static int test_gpu(void) {
vk_mem_stats(&st);
GCHECK(!st.imported_pools && !st.physical_pools && !st.virtual_pools, "pools left after fini");
g_gpu_checks += 2;
return test_table_sync(h) || test_table_store(h) || test_remade(h);
return test_table_sync(h) || test_table_store(h) || test_remade(h) || test_table_mirror(h);
}
int probe_vkmem(void) {

View File

@@ -377,6 +377,7 @@ void vk_mvprev_frame(void); // each present, beside vk_dlaa_frame
void vk_mem_mirror_begin(void);
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence);
int vk_mem_mirror_pending(void);
int vk_mem_gpa_late(void); // --probe-vkmem: NX2NAVI_GPA_LATE, the table takes a new mirror up in queue order
void vk_mem_mirror_prologue(VkCommandBuffer cb);
// Staged mirror uploads (vk_memory.c): the GPU of the submission being recorded reads guest memory itself (noted by
// vk_mem_note_live); vk_mem_take_live returns that and starts the next submission's note.

View File

@@ -19,8 +19,11 @@
// writes (vk_exec_draw.c ssbo_info), so with the mirror alone the CPU never saw them, and a sync of a page the CPU wrote
// copied the stale guest page over them. The game's compute passes write tree and foliage records this way, and a
// record overwritten so drew a tree stretched into a column reaching the sky (Hyrule Field, Kakariko, Death Mountain).
// A record a page entry names is never written: a new mapping takes a new record, and a record goes back to the free
// list only once the GPU finished every submission issued while a page could still name it (vk_completion_add). Page
// A record a page entry names is never written by the CPU: a new mapping takes a new record, and a record goes back
// to the free list only once the GPU finished every submission issued while a page could still name it
// (vk_completion_add). The one write to a record in use is the GPU's own, in queue order, when a pool's new mirror
// takes its record over (vk_gpa_flip_cmd): what was queued before it reads the old addresses, what comes after it
// the new ones. Page
// and L1 entries are single 32-bit stores, so a shader that reads the table while it changes sees the old mapping or
// the new one, never half of each. The CPU keeps a copy (shadow) for its reads: the GPU copy is often device memory
// behind the PCIe BAR, slow to read back.
@@ -110,7 +113,8 @@ static int create(void) {
G.free_recs = malloc(RECS * sizeof *G.free_recs);
if (!G.cpu || !G.free_recs) return 0;
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)WORDS * 4,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, // vk_gpa_flip_cmd
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT,
.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
VmaAllocationInfo info;
@@ -352,23 +356,39 @@ static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf,
// the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too.
uint64_t twin = old_rec && twins_on() && keep_twin ? rec_first(old_rec) : dev - buf_lo;
uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0;
if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; }
if (!r) { // no record left: the pages keep the old one, and the caller its number
static int said;
plat_mutex_unlock(&gpa_lock);
if (!said++)
rt_log("vk_gpa: WARNING: no record left for a buffer that takes over %llu pages (%u records in all): they "
"keep naming the buffer they named", (unsigned long long)(p1 > p0 ? p1 - p0 : 0), RECS - 1);
return old_rec;
}
pages_move(p0, p1, old_rec, r);
plat_mutex_unlock(&gpa_lock);
if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec);
return r;
}
uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) {
if (!old_rec || !buf || !vk_gpa_on() || !twins_on()) return 0;
// Records into cb the store that hands record rec over to buf, a pool's new VRAM mirror: from that point of the queue
// on the pages that name rec read buf, and their twin is the buffer they read until then (keep_twin, with twins on) or
// buf again. A change of the table made on the CPU is seen at once by every shader still running, the ones queued
// before the mirror's first copy included, which would then read or write the mirror before anything had filled it;
// this store runs where the command stands, after that copy (vk_memory.c vk_mem_mirror_prologue). The caller puts
// barriers around it. No new record is taken, so there is none to run out of. The shadow takes the new words now:
// the CPU never reads a record's addresses back for a record handed over this way. 0 when nothing was recorded.
int vk_gpa_flip_cmd(VkCommandBuffer cb, uint32_t rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) {
if (!rec || !buf || !vk_gpa_on()) return 0;
uint64_t dev = vk_gpa_device_address(buf);
uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT;
if (!dev) return 0;
uint32_t at = REC_WORD + rec * REC_WORDS, w[REC_WORDS];
plat_mutex_lock(&gpa_lock);
uint32_t r = dev ? rec_alloc(rec_first(old_rec), dev - buf_lo) : 0;
if (r) pages_move(p0, p1, old_rec, r);
uint64_t first = dev - buf_lo, twin = keep_twin && twins_on() ? rec_first(rec) : first;
w[0] = (uint32_t)first, w[1] = (uint32_t)(first >> 32), w[2] = (uint32_t)twin, w[3] = (uint32_t)(twin >> 32);
memcpy(G.cpu + at, w, sizeof w);
plat_mutex_unlock(&gpa_lock);
if (r) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec);
return r;
vkCmdUpdateBuffer(cb, G.buf, (VkDeviceSize)at * 4, sizeof w, w);
return 1;
}
uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) {
@@ -379,6 +399,21 @@ uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer
return remap(lo, hi, old_rec, buf, buf_lo, 0);
}
// --probe-vkmem: what the table's own memory, as the GPU has it, resolves addr to: the device address a shader reads
// there and the one it writes as well. 0 when no page maps addr. Slow (the table may be device memory), and only
// meaningful with no submission in flight.
int vk_gpa_probe(uint64_t addr, uint64_t *first, uint64_t *twin) {
if (G.state <= 0 || addr >> 40) return 0;
vmaInvalidateAllocation(vk_ctx()->vma, G.alloc, 0, VK_WHOLE_SIZE);
uint32_t blk = G.gpu[L1_WORD + (addr >> 24)];
uint32_t rec = blk ? G.gpu[L2_WORD + blk * L2_N + ((addr >> PAGE_SHIFT) & (L2_N - 1))] : 0;
if (!rec) return 0;
const volatile uint32_t *r = G.gpu + REC_WORD + rec * REC_WORDS;
*first = ((uint64_t)r[0] | (uint64_t)r[1] << 32) + addr;
*twin = ((uint64_t)r[2] | (uint64_t)r[3] << 32) + addr;
return 1;
}
void vk_gpa_report(void) {
if (G.state <= 0) return;
plat_mutex_lock(&gpa_lock);

View File

@@ -24,9 +24,12 @@ uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf,
// The same without the twin: the pages name buf alone (a pool whose VRAM mirror is evicted goes back to its imported
// buffer).
uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo);
// The pages of [lo, hi) that name old_rec keep reading old_rec's buffer and get buf as their twin: shaders read the
// pool where they did and write buf too (a VRAM mirror until its first copy has run, vk_memory.c mirror_create).
// Returns the new record, or 0 when there is none to give or twins are off (old_rec then stays as it is).
uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo);
// Records into cb the store that hands record rec over to buf (a pool's new VRAM mirror) in queue order: what is
// queued after it reads buf through rec's pages, with the buffer they read before as the twin (keep_twin) or buf
// again; what was queued before it sees no change. 0 when nothing was recorded (no record, no table).
int vk_gpa_flip_cmd(VkCommandBuffer cb, uint32_t rec, VkBuffer buf, uint64_t buf_lo, int keep_twin);
// --probe-vkmem: the device addresses the table, read from its own memory, gives a shader for addr: the one it
// reads and the one it writes as well. 0 when no page maps addr.
int vk_gpa_probe(uint64_t addr, uint64_t *first, uint64_t *twin);
void vk_gpa_report(void);
void vk_gpa_shutdown(void);

View File

@@ -61,8 +61,8 @@ typedef struct PoolRec {
// VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only.
uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED
uint32_t gpa_rec; // POOL_IMPORT: its record in the device address table (vk_gpa.c), 0 none
uint64_t gpa_at; // pool_lock: the submission after which the table reads the mirror (gpa_switch);
// 0 when it does already, or there is no mirror
uint8_t gpa_flip; // as pend: the next prologue records the store that hands its table record to
// the mirror (gpa_flip_cmds); 0 when the table names the mirror, or none exists
struct GpaMap { uint64_t lo, hi; uint32_t rec; } *vmaps; // POOL_VIRT: its mapped ranges in the table
int nvmaps, cvmaps;
VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf
@@ -2758,50 +2758,58 @@ static int twin_wanted(const PoolRec *r) {
return !twin_max || (int64_t)(r->import_hi - r->import_lo) <= twin_max;
}
// ---- when the device address table starts reading a new mirror (NX2NAVI_GPA_LATE, default 1; 0 = at once, as before)
// A mirror is made while the replay records a submission, and its first copy of the pool is in that submission's
// prologue. The table is memory the shaders read as they run, so with its pages moved to the mirror at once, every
// part already handed to the queue (the submissions before, and the earlier parts of this one) read the mirror through
// the table before that copy had run: memory nothing had written yet, and on a card over its budget a block the
// driver had not made resident for those parts. A GTX 1650 4 GB lost its device that way at area loads in every
// session: six reports of a GPU read fault at byte 0 of a mirror made in the faulting submission, each the first byte
// of a new 2 MiB-aligned block, with mirrors being made over the budget; with the table kept empty the same routes ran
// without a loss.
// Now the pages keep reading the imported buffer and name the mirror as their twin (vk_gpa_remap_writes): what shaders
// write through the table goes to both, so the mirror misses nothing, and what they read is guest memory, as before
// the mirror existed. Once the GPU has finished the submission whose prologue holds the copy, the pages move to the
// mirror with the imported buffer as the twin (gpa_switch, at the start of a later submission). In between, a shader
// that reads through the table what another wrote through a buffer binding sees guest memory, which gets such a write
// at the write-back; data written through the table itself, as the game's foliage, instance and light records are,
// is read where it was written.
static int n_gpa_wait; // pool_lock: mirrors whose pages still read the imported buffer (PoolRec.gpa_at)
static uint64_t n_gpa_late; // replay: pages moved to their mirror after its first copy, so far
// ---- when the device address table takes up a new mirror (NX2NAVI_GPA_LATE, default 1; 0 = at once, as before)
// A mirror is made while the replay records a part of a submission, and its first copy of the pool is in that part's
// prologue. The table is memory the shaders read as they run, so a change of it made here, at record time, reaches
// every part already handed to the queue (the submissions before, and the earlier parts of this one), which run
// before that copy: they read the mirror through the table, memory nothing had written yet, and wrote into it, and on
// a card over its budget the block was one the driver had not made resident for them. A GTX 1650 4 GB lost its device
// that way at area loads in every session: six reports of a GPU read fault at byte 0 of a mirror made in the faulting
// submission, each the first byte of a new 2 MiB-aligned block, with mirrors being made over the budget; with the
// table kept empty the same routes ran without a loss.
// So the table is not touched here. The prologue that holds the mirror's first copy also holds, right after it, a
// store into the pool's record (vk_gpa_flip_cmd): the GPU runs it in its place in the queue, so the parts before it
// read and write guest memory through the table as if no mirror existed, and the parts from this one on read the
// mirror and write both. What the earlier parts wrote through the table is in guest memory when the first copy reads
// it there (a twinned pool's first copy is always the GPU's own copy from the imported buffer, mirror_capture), so
// the mirror misses none of it, and no shader reads through the table a state older than the one its place in the
// queue gives it.
static int n_gpa_flip; // as pend: mirrors whose record the next prologue hands over (PoolRec.gpa_flip)
static uint64_t n_gpa_late; // replay: records handed to their mirror in queue order, so far
static int gpa_late(void);
int vk_mem_gpa_late(void) { return gpa_late(); } // --probe-vkmem
static int gpa_late(void) {
static int on = -1;
if (on < 0) {
on = rt_knob("GPA_LATE", "1") == 1;
rt_log("mem: the device address table reads a new VRAM mirror %s (NX2NAVI_GPA_LATE)",
on ? "once the mirror's first copy has run" : "at once, as before");
rt_log("mem: the device address table takes up a new VRAM mirror %s (NX2NAVI_GPA_LATE)",
on ? "in queue order, after the mirror's first copy" : "at once, as before");
}
return on;
}
// Replay thread, at the start of a submission: the mirrors whose first copy the GPU has finished take the table's reads.
static void gpa_switch(void) {
if (!__atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) return;
uint64_t done = vk_sync_fence_done();
static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds,
VkAccessFlags2 da);
// Prologue, after the copies into the mirrors: the stores that hand the new mirrors' records over. Everything queued
// before is done with the table before the stores run, and everything after sees them.
static void gpa_flip_cmds(VkCommandBuffer cb) {
if (!n_gpa_flip) return;
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
plat_mutex_lock(&pool_lock);
for (int i = 0; i < nm && n_gpa_wait; i++) {
for (int i = 0; i < nm && n_gpa_flip; i++) {
PoolRec *r = mlist[i];
if (!r->gpa_at || done < r->gpa_at) continue;
r->gpa_at = 0, n_gpa_wait--;
if (!r->gpa_rec || !r->mbuf) continue;
r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo)
: vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo);
n_gpa_late++;
if (!r->gpa_flip) continue;
r->gpa_flip = 0, n_gpa_flip--;
if (r->gpa_rec && r->mbuf && vk_gpa_flip_cmd(cb, r->gpa_rec, r->mbuf, r->import_lo, r->twin)) n_gpa_late++;
}
n_gpa_flip = 0;
plat_mutex_unlock(&pool_lock);
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
}
// Replay thread: the pool's mirror on its first buffer use.
@@ -2897,25 +2905,19 @@ static void mirror_create(PoolRec *r) {
r->mstate = MS_ON;
r->asked_room = 0;
// The device address table names the mirror: the GPU's writes go there, and the sync copies the CPU's. With a
// twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. Reads
// move to the mirror once its first copy has run (gpa_late, below).
// twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. The
// table takes the mirror up in queue order, after its first copy (gpa_late, above).
r->twin = (uint8_t)twin_wanted(r);
if (!r->twin) {
plat_inc64(&m_untwinned_n);
plat_add64(&m_untwinned_bytes, (int64_t)n);
}
if (r->gpa_rec) {
if (r->gpa_rec && gpa_late()) {
r->gpa_flip = 1, n_gpa_flip++; // vk_mem_mirror_prologue, with the copy range_add queues below
} else if (r->gpa_rec) {
plat_mutex_lock(&pool_lock);
uint32_t late = gpa_late() ? vk_gpa_remap_writes(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) : 0;
if (late) {
uint64_t f = vk_exec_recording_fence();
r->gpa_rec = late;
r->gpa_at = f ? f : vk_sync_issued() + 1; // the submission being recorded: its prologue holds the copy
n_gpa_wait++;
} else {
r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo)
: vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo);
}
r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo)
: vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo);
plat_mutex_unlock(&pool_lock);
}
range_add(&pend, &npend, &cpend, r, 0, n);
@@ -2926,7 +2928,7 @@ static void mirror_create(PoolRec *r) {
static void mirror_forget(PoolRec *r) {
__atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE);
if (r->gpa_at) r->gpa_at = 0, n_gpa_wait--; // the caller takes the table's pages off the mirror
if (r->gpa_flip) r->gpa_flip = 0, n_gpa_flip--; // gone before its first copy: its record stays as it is
for (int i = 0; i < nm; i++)
if (mlist[i] == r) {
memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist);
@@ -3993,7 +3995,7 @@ static void hot_check(void) {
}
void vk_mem_mirror_begin(void) {
if (vk_mirror_on) gpa_switch(), hot_check();
if (vk_mirror_on) hot_check();
if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions
if (!freeze_idle) return;
plat_mutex_lock(&gen_lock);
@@ -4039,7 +4041,7 @@ void vk_mem_mirror_begin(void) {
slow_mem_call("mirror begin (its queries and freezes)", 0, ww_bytes - b0, q, qpc() - t0);
}
int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; }
int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl || n_gpa_flip; }
// NX2NAVI_MIRROR_CAPTURE_KB: the pages the CPU wrote since a VRAM mirror's last sync are copied out of guest memory by
// the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a host buffer the
@@ -4163,7 +4165,7 @@ static void mirror_capture(VkCommandBuffer cb) {
}
void vk_mem_mirror_prologue(VkCommandBuffer cb) {
if (!npend && snl_copied == nsnl) return;
if (!npend && snl_copied == nsnl && !n_gpa_flip) return;
mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
mirror_capture(cb);
@@ -4176,6 +4178,7 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb) {
}
mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
gpa_flip_cmds(cb); // after the new mirrors' first copies
npend = 0;
mrep.prologues++;
}
@@ -4448,10 +4451,9 @@ static void mirror_log(uint64_t frame) {
"%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; flushed and not copied (write-watched) %.3f MiB%s",
(unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f,
a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M, a.flush_skipped / f / M, capl);
if (n_gpa_late || __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED))
rt_log("mem: mirror frame %llu: the device address table took up %llu mirrors after their first copy so far, %d "
"wait for theirs (NX2NAVI_GPA_LATE)", (unsigned long long)frame, (unsigned long long)n_gpa_late,
__atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED));
if (n_gpa_late)
rt_log("mem: mirror frame %llu: the device address table took up %llu mirrors in queue order, after their first "
"copy, so far (NX2NAVI_GPA_LATE)", (unsigned long long)frame, (unsigned long long)n_gpa_late);
}
// After vram_log: every heap's usage as the driver reports it (VK_EXT_memory_budget) beside VMA's share of it, VMA by
@@ -5562,7 +5564,7 @@ void vk_mem_shutdown(void) {
plat_mutex_lock(&pool_lock);
for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf);
pool_index.n = 0;
nm = npend = nwbl = nsnl = snl_copied = nswl = 0; // mirrors went with their pools
nm = npend = nwbl = nsnl = snl_copied = nswl = n_gpa_flip = 0; // mirrors went with their pools
swl_marks = -1;
plat_mutex_lock(&snap_pool_lock);
for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc);

View File

@@ -49,6 +49,10 @@ VkResult vmaCreateBuffer(VmaAllocator a, const VkBufferCreateInfo *bi, const Vma
return VK_SUCCESS;
}
void vmaDestroyBuffer(VmaAllocator a, VkBuffer buf, VmaAllocation alloc) { (void)a, (void)buf; free(alloc); }
VkResult vmaInvalidateAllocation(VmaAllocator a, VmaAllocation alloc, VkDeviceSize off, VkDeviceSize n) {
(void)a, (void)alloc, (void)off, (void)n;
return VK_SUCCESS;
}
// A buffer's device address: its handle far above the GPU addresses the tests map.
static uint64_t dev_of(VkBuffer b) { return (uint64_t)(uintptr_t)b << 28; }
@@ -73,6 +77,22 @@ static void gpu_done(void) { // the submission being recorded is issued and fi
nqueue = k;
}
// A store the GPU runs in queue order (vk_gpa_flip_cmd): kept until gpu_runs_stores, as a command buffer keeps it
// until the GPU reaches it.
static struct { VkDeviceSize off, n; uint8_t data[32]; } stores[8];
static int nstores;
void vkCmdUpdateBuffer(VkCommandBuffer cb, VkBuffer b, VkDeviceSize off, VkDeviceSize n, const void *data) {
(void)cb, (void)b;
if (nstores == 8 || n > sizeof stores[0].data || (off | n) & 3) { printf("FAIL: a store the stand-in cannot hold\n"); exit(1); }
stores[nstores].off = off, stores[nstores].n = n;
memcpy(stores[nstores].data, data, (size_t)n);
nstores++;
}
static void gpu_runs_stores(void) {
for (int i = 0; i < nstores; i++) memcpy((uint8_t *)(uintptr_t)table + stores[i].off, stores[i].data, (size_t)stores[i].n);
nstores = 0;
}
// ---- the table as a shader reads it (the layout at the top of vk_gpa.c)
enum { L1_WORD = 16, L1_N = 65536, L2_WORD = L1_WORD + L1_N, L2_N = 4096, BLOCKS = 1024 };
static uint32_t block_of(uint64_t addr) { return table[L1_WORD + (addr >> 24)]; }
@@ -125,27 +145,35 @@ int main(void) {
CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "a plain page is its own twin");
gpu_done();
// a new mirror takes the writes first and the reads once its first copy has run (vk_memory.c mirror_create,
// gpa_switch): until then the pages read the imported buffer and the mirror is their twin
// a new mirror takes the pool's record over by a store the GPU runs after the mirror's first copy (vk_memory.c
// mirror_create, vk_mem_mirror_prologue): what is queued before the store reads and writes the imported buffer
// alone, what comes after it reads the mirror and writes both
VkBuffer m1 = new_buf();
uint32_t w0 = vk_gpa_remap_writes(a0, a1, r0, m1, a0);
CHECK(w0 && w0 != r0, "a new mirror gets a record for the writes");
expect(a0, a1, b0, a0, "a new mirror before its first copy: reads stay on the imported buffer");
CHECK(read_through(a0 + 4096, 1) == dev_of(m1) + 4096 && read_through(a1 - 4096, 1) == dev_of(m1) + (a1 - 4096 - a0),
"a new mirror before its first copy is the twin, which takes the writes");
CHECK(vk_gpa_flip_cmd(VK_NULL_HANDLE, r0, m1, a0, 1), "a new mirror's store is recorded");
expect(a0, a1, b0, a0, "before the GPU reaches the store: reads stay on the imported buffer");
CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096 && read_through(a1 - 4096, 1) == dev_of(b0) + (a1 - 4096 - a0),
"before the GPU reaches the store nothing is written into the mirror");
gpu_runs_stores();
expect(a0, a1, m1, a0, "after the store: reads are on the mirror");
CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096 && read_through(a1 - 4096, 1) == dev_of(b0) + (a1 - 4096 - a0),
"after the store the imported buffer is the twin");
gpu_done();
r0 = vk_gpa_remap(a0, a1, w0, m1, a0);
expect(a0, a1, m1, a0, "a new mirror after its first copy: reads move to it");
CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "and the imported buffer is the twin again");
// evicted while it still waits for its first copy: the pages go back to the imported buffer alone
// evicted: the pages go back to the imported buffer alone, on a record of their own
uint32_t was = r0;
r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0);
gpu_done();
VkBuffer m2 = new_buf();
w0 = vk_gpa_remap_writes(a0, a1, r0, m2, a0);
r0 = vk_gpa_remap_plain(a0, a1, w0, b0, a0);
expect(a0, a1, b0, a0, "a mirror gone before its first copy");
CHECK(r0 && r0 != was, "an evicted mirror's pages take a new record");
expect(a0, a1, b0, a0, "an evicted mirror");
CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "leaves the imported buffer as its own twin");
CHECK(!vk_gpa_remap_writes(a0, a1, 0, m2, a0), "a pool without a record gets none for a mirror's writes");
gpu_done();
// a mirror without a twin: the store names it twice
VkBuffer m2 = new_buf();
CHECK(vk_gpa_flip_cmd(VK_NULL_HANDLE, r0, m2, a0, 0), "an untwinned mirror's store is recorded");
gpu_runs_stores();
expect(a0, a1, m2, a0, "an untwinned mirror");
CHECK(read_through(a0 + 4096, 1) == dev_of(m2) + 4096, "is its own twin");
r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0);
expect(a0, a1, b0, a0, "and goes back to the imported buffer");
CHECK(!vk_gpa_flip_cmd(VK_NULL_HANDLE, 0, m2, a0, 1) && !nstores, "a pool without a record gets no store");
gpu_done();
// an unmapped range reads the zero buffer at once, and its blocks serve no other range while the GPU may still read