diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 2f2d3110..11b6a88b 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -483,6 +483,76 @@ static int test_remade(uint8_t *guest) { return 0; } +// A pool in the device address table gets its VRAM mirror while a guest submission is recorded, as on a draw's first +// use of it as a buffer (vk_memory.c mirror_create). The table takes the mirror up by a store the GPU runs after the +// mirror's first copy (vk_gpa.c vk_gpa_flip_cmd), so nothing is changed when the mirror is made; read from the table's +// own memory once the submission is done, the pool's page reads the mirror and writes guest memory as well, and the +// mirror holds the pool's bytes. +static void mirror_use_record(VkCommandBuffer cb, void *arg) { + vk_texture_new_submit(); + vk_mem_mirror_begin(); + vk_mem_use(arg); + vk_mem_mirror_end(cb, vk_sync_issued() + 1); +} + +static int test_table_mirror(uint8_t *guest) { + VkCtx *c = vk_ctx(); + if (vk_mem_shadow_storage() || !vk_gpa_wanted()) { + printf("probe-vkmem: a new mirror in the device address table: skipped (%s)\n", + vk_mem_shadow_storage() ? "shadow storage, no mirrors" : "no device address table"); + return 0; + } + pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xD00000, .size = 0x10000, + .game_mem = guest + 0xD00000}; + volatile uint32_t *w = (volatile uint32_t *)(pool.mem + 0x3000); + uint64_t a = (uintptr_t)w, f0 = 0, t0 = 0, f1 = 0, t1 = 0, fm = 0, tm = 0; + *w = 0x600DF00Du; + GCHECK(vk_gpa_on() && vk_pool_init(&pool), "a CPU-visible pool in the device address table"); + VkBuffer sb; + VmaAllocation sa; + volatile uint32_t *s = vk_staging(0x1000, &sb, &sa); + VkMemSpan sp; + int bad = 1; +#define TM_CHECK(cond, ...) do { if (!(cond)) { printf("probe-vkmem: FAIL table mirror: "); printf(__VA_ARGS__); printf("\n"); goto done; } } while (0) + TM_CHECK(vk_mem_lookup(a, 4, &sp) && sp.buffer && sp.pool, "the pool has no buffer"); + VkBuffer imported = sp.buffer; + TM_CHECK(vk_gpa_probe(a, &f0, &t0) && f0 == t0 && f0 == vk_gpa_device_address(imported) + sp.offset, + "before a mirror the table gives %#llx / %#llx for the pool's page, expected its imported buffer twice", + (unsigned long long)f0, (unsigned long long)t0); + uint64_t fence = vk_sync_submit_own(mirror_use_record, &sp); + // read while the submission may still be in flight: the table as the CPU left it when the mirror was made + int mid = vk_gpa_probe(a, &fm, &tm); + TM_CHECK(vk_sync_fence_wait(fence, 5000000000ull), "fence timed out"); + if (sp.buffer == imported) { + printf("probe-vkmem: a new mirror in the device address table: skipped (the pool got no mirror)\n"); + bad = 0; + goto done; + } + TM_CHECK(vk_gpa_probe(a, &f1, &t1), "the pool's page left the table"); + if (vk_mem_gpa_late()) // else the table changes when the mirror is made, as up to 0.9.2 + TM_CHECK(mid && (fm == f0 || fm == f1) && tm == f0, "while the submission ran the table gave %#llx / %#llx: neither " + "the state before the store (%#llx twice) nor after it", (unsigned long long)fm, (unsigned long long)tm, + (unsigned long long)f0); + TM_CHECK(f1 == vk_gpa_device_address(sp.buffer) + sp.offset, "after the submission the page reads %#llx, expected the " + "mirror's %#llx", (unsigned long long)f1, (unsigned long long)(vk_gpa_device_address(sp.buffer) + sp.offset)); + TM_CHECK(t1 == f0, "after the submission the page's twin is %#llx, expected the imported buffer's %#llx", + (unsigned long long)t1, (unsigned long long)f0); + copy_wait(sp.buffer, sp.offset, sb, 0, 4); + vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); + TM_CHECK(s[0] == 0x600DF00Du, "the mirror holds %#010x, expected the pool's 0x600df00d", s[0]); + g_gpu_checks += 4; + printf("probe-vkmem: a pool that gets its mirror in a recorded submission: the device address table names the mirror " + "after it, with guest memory as the twin (%s)\n", + vk_mem_gpa_late() ? "by a store in queue order, after the mirror's first copy" : "changed at once, NX2NAVI_GPA_LATE=0"); + bad = 0; +done: + vk_sync_finish(); + vmaDestroyBuffer(c->vma, sb, sa); + vk_pool_fini(&pool); +#undef TM_CHECK + return bad; +} + // test_table_sync, test_table_store: a guest submission of one part that records nothing. bound: its record binds the // device address table, as the replay of a draw whose shader reads through the table does (vk_exec_draw.c ssbo_info), // so its prologue brings every shadow of the table up to date (vk_memory.c vk_mem_shadow_sync). @@ -1366,7 +1436,7 @@ static int test_gpu(void) { vk_mem_stats(&st); GCHECK(!st.imported_pools && !st.physical_pools && !st.virtual_pools, "pools left after fini"); g_gpu_checks += 2; - return test_table_sync(h) || test_table_store(h) || test_remade(h); + return test_table_sync(h) || test_table_store(h) || test_remade(h) || test_table_mirror(h); } int probe_vkmem(void) { diff --git a/runtime/vk.h b/runtime/vk.h index 6fe33822..bc297b29 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -377,6 +377,7 @@ void vk_mvprev_frame(void); // each present, beside vk_dlaa_frame void vk_mem_mirror_begin(void); int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); int vk_mem_mirror_pending(void); +int vk_mem_gpa_late(void); // --probe-vkmem: NX2NAVI_GPA_LATE, the table takes a new mirror up in queue order void vk_mem_mirror_prologue(VkCommandBuffer cb); // Staged mirror uploads (vk_memory.c): the GPU of the submission being recorded reads guest memory itself (noted by // vk_mem_note_live); vk_mem_take_live returns that and starts the next submission's note. diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index 9daf0a14..8439b812 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -19,8 +19,11 @@ // writes (vk_exec_draw.c ssbo_info), so with the mirror alone the CPU never saw them, and a sync of a page the CPU wrote // copied the stale guest page over them. The game's compute passes write tree and foliage records this way, and a // record overwritten so drew a tree stretched into a column reaching the sky (Hyrule Field, Kakariko, Death Mountain). -// A record a page entry names is never written: a new mapping takes a new record, and a record goes back to the free -// list only once the GPU finished every submission issued while a page could still name it (vk_completion_add). Page +// A record a page entry names is never written by the CPU: a new mapping takes a new record, and a record goes back +// to the free list only once the GPU finished every submission issued while a page could still name it +// (vk_completion_add). The one write to a record in use is the GPU's own, in queue order, when a pool's new mirror +// takes its record over (vk_gpa_flip_cmd): what was queued before it reads the old addresses, what comes after it +// the new ones. Page // and L1 entries are single 32-bit stores, so a shader that reads the table while it changes sees the old mapping or // the new one, never half of each. The CPU keeps a copy (shadow) for its reads: the GPU copy is often device memory // behind the PCIe BAR, slow to read back. @@ -110,7 +113,8 @@ static int create(void) { G.free_recs = malloc(RECS * sizeof *G.free_recs); if (!G.cpu || !G.free_recs) return 0; VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)WORDS * 4, - .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, // vk_gpa_flip_cmd + .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; VmaAllocationInfo info; @@ -352,23 +356,39 @@ static uint32_t remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, // the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too. uint64_t twin = old_rec && twins_on() && keep_twin ? rec_first(old_rec) : dev - buf_lo; uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0; - if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } + if (!r) { // no record left: the pages keep the old one, and the caller its number + static int said; + plat_mutex_unlock(&gpa_lock); + if (!said++) + rt_log("vk_gpa: WARNING: no record left for a buffer that takes over %llu pages (%u records in all): they " + "keep naming the buffer they named", (unsigned long long)(p1 > p0 ? p1 - p0 : 0), RECS - 1); + return old_rec; + } pages_move(p0, p1, old_rec, r); plat_mutex_unlock(&gpa_lock); if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); return r; } -uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { - if (!old_rec || !buf || !vk_gpa_on() || !twins_on()) return 0; +// Records into cb the store that hands record rec over to buf, a pool's new VRAM mirror: from that point of the queue +// on the pages that name rec read buf, and their twin is the buffer they read until then (keep_twin, with twins on) or +// buf again. A change of the table made on the CPU is seen at once by every shader still running, the ones queued +// before the mirror's first copy included, which would then read or write the mirror before anything had filled it; +// this store runs where the command stands, after that copy (vk_memory.c vk_mem_mirror_prologue). The caller puts +// barriers around it. No new record is taken, so there is none to run out of. The shadow takes the new words now: +// the CPU never reads a record's addresses back for a record handed over this way. 0 when nothing was recorded. +int vk_gpa_flip_cmd(VkCommandBuffer cb, uint32_t rec, VkBuffer buf, uint64_t buf_lo, int keep_twin) { + if (!rec || !buf || !vk_gpa_on()) return 0; uint64_t dev = vk_gpa_device_address(buf); - uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + if (!dev) return 0; + uint32_t at = REC_WORD + rec * REC_WORDS, w[REC_WORDS]; plat_mutex_lock(&gpa_lock); - uint32_t r = dev ? rec_alloc(rec_first(old_rec), dev - buf_lo) : 0; - if (r) pages_move(p0, p1, old_rec, r); + uint64_t first = dev - buf_lo, twin = keep_twin && twins_on() ? rec_first(rec) : first; + w[0] = (uint32_t)first, w[1] = (uint32_t)(first >> 32), w[2] = (uint32_t)twin, w[3] = (uint32_t)(twin >> 32); + memcpy(G.cpu + at, w, sizeof w); plat_mutex_unlock(&gpa_lock); - if (r) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); - return r; + vkCmdUpdateBuffer(cb, G.buf, (VkDeviceSize)at * 4, sizeof w, w); + return 1; } uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { @@ -379,6 +399,21 @@ uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer return remap(lo, hi, old_rec, buf, buf_lo, 0); } +// --probe-vkmem: what the table's own memory, as the GPU has it, resolves addr to: the device address a shader reads +// there and the one it writes as well. 0 when no page maps addr. Slow (the table may be device memory), and only +// meaningful with no submission in flight. +int vk_gpa_probe(uint64_t addr, uint64_t *first, uint64_t *twin) { + if (G.state <= 0 || addr >> 40) return 0; + vmaInvalidateAllocation(vk_ctx()->vma, G.alloc, 0, VK_WHOLE_SIZE); + uint32_t blk = G.gpu[L1_WORD + (addr >> 24)]; + uint32_t rec = blk ? G.gpu[L2_WORD + blk * L2_N + ((addr >> PAGE_SHIFT) & (L2_N - 1))] : 0; + if (!rec) return 0; + const volatile uint32_t *r = G.gpu + REC_WORD + rec * REC_WORDS; + *first = ((uint64_t)r[0] | (uint64_t)r[1] << 32) + addr; + *twin = ((uint64_t)r[2] | (uint64_t)r[3] << 32) + addr; + return 1; +} + void vk_gpa_report(void) { if (G.state <= 0) return; plat_mutex_lock(&gpa_lock); diff --git a/runtime/vk_gpa.h b/runtime/vk_gpa.h index 29df805e..18d778e5 100644 --- a/runtime/vk_gpa.h +++ b/runtime/vk_gpa.h @@ -24,9 +24,12 @@ uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, // The same without the twin: the pages name buf alone (a pool whose VRAM mirror is evicted goes back to its imported // buffer). uint32_t vk_gpa_remap_plain(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); -// The pages of [lo, hi) that name old_rec keep reading old_rec's buffer and get buf as their twin: shaders read the -// pool where they did and write buf too (a VRAM mirror until its first copy has run, vk_memory.c mirror_create). -// Returns the new record, or 0 when there is none to give or twins are off (old_rec then stays as it is). -uint32_t vk_gpa_remap_writes(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); +// Records into cb the store that hands record rec over to buf (a pool's new VRAM mirror) in queue order: what is +// queued after it reads buf through rec's pages, with the buffer they read before as the twin (keep_twin) or buf +// again; what was queued before it sees no change. 0 when nothing was recorded (no record, no table). +int vk_gpa_flip_cmd(VkCommandBuffer cb, uint32_t rec, VkBuffer buf, uint64_t buf_lo, int keep_twin); +// --probe-vkmem: the device addresses the table, read from its own memory, gives a shader for addr: the one it +// reads and the one it writes as well. 0 when no page maps addr. +int vk_gpa_probe(uint64_t addr, uint64_t *first, uint64_t *twin); void vk_gpa_report(void); void vk_gpa_shutdown(void); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 998ef77c..fa75347d 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -61,8 +61,8 @@ typedef struct PoolRec { // VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only. uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED uint32_t gpa_rec; // POOL_IMPORT: its record in the device address table (vk_gpa.c), 0 none - uint64_t gpa_at; // pool_lock: the submission after which the table reads the mirror (gpa_switch); - // 0 when it does already, or there is no mirror + uint8_t gpa_flip; // as pend: the next prologue records the store that hands its table record to + // the mirror (gpa_flip_cmds); 0 when the table names the mirror, or none exists struct GpaMap { uint64_t lo, hi; uint32_t rec; } *vmaps; // POOL_VIRT: its mapped ranges in the table int nvmaps, cvmaps; VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf @@ -2758,50 +2758,58 @@ static int twin_wanted(const PoolRec *r) { return !twin_max || (int64_t)(r->import_hi - r->import_lo) <= twin_max; } -// ---- when the device address table starts reading a new mirror (NX2NAVI_GPA_LATE, default 1; 0 = at once, as before) -// A mirror is made while the replay records a submission, and its first copy of the pool is in that submission's -// prologue. The table is memory the shaders read as they run, so with its pages moved to the mirror at once, every -// part already handed to the queue (the submissions before, and the earlier parts of this one) read the mirror through -// the table before that copy had run: memory nothing had written yet, and on a card over its budget a block the -// driver had not made resident for those parts. A GTX 1650 4 GB lost its device that way at area loads in every -// session: six reports of a GPU read fault at byte 0 of a mirror made in the faulting submission, each the first byte -// of a new 2 MiB-aligned block, with mirrors being made over the budget; with the table kept empty the same routes ran -// without a loss. -// Now the pages keep reading the imported buffer and name the mirror as their twin (vk_gpa_remap_writes): what shaders -// write through the table goes to both, so the mirror misses nothing, and what they read is guest memory, as before -// the mirror existed. Once the GPU has finished the submission whose prologue holds the copy, the pages move to the -// mirror with the imported buffer as the twin (gpa_switch, at the start of a later submission). In between, a shader -// that reads through the table what another wrote through a buffer binding sees guest memory, which gets such a write -// at the write-back; data written through the table itself, as the game's foliage, instance and light records are, -// is read where it was written. -static int n_gpa_wait; // pool_lock: mirrors whose pages still read the imported buffer (PoolRec.gpa_at) -static uint64_t n_gpa_late; // replay: pages moved to their mirror after its first copy, so far +// ---- when the device address table takes up a new mirror (NX2NAVI_GPA_LATE, default 1; 0 = at once, as before) +// A mirror is made while the replay records a part of a submission, and its first copy of the pool is in that part's +// prologue. The table is memory the shaders read as they run, so a change of it made here, at record time, reaches +// every part already handed to the queue (the submissions before, and the earlier parts of this one), which run +// before that copy: they read the mirror through the table, memory nothing had written yet, and wrote into it, and on +// a card over its budget the block was one the driver had not made resident for them. A GTX 1650 4 GB lost its device +// that way at area loads in every session: six reports of a GPU read fault at byte 0 of a mirror made in the faulting +// submission, each the first byte of a new 2 MiB-aligned block, with mirrors being made over the budget; with the +// table kept empty the same routes ran without a loss. +// So the table is not touched here. The prologue that holds the mirror's first copy also holds, right after it, a +// store into the pool's record (vk_gpa_flip_cmd): the GPU runs it in its place in the queue, so the parts before it +// read and write guest memory through the table as if no mirror existed, and the parts from this one on read the +// mirror and write both. What the earlier parts wrote through the table is in guest memory when the first copy reads +// it there (a twinned pool's first copy is always the GPU's own copy from the imported buffer, mirror_capture), so +// the mirror misses none of it, and no shader reads through the table a state older than the one its place in the +// queue gives it. +static int n_gpa_flip; // as pend: mirrors whose record the next prologue hands over (PoolRec.gpa_flip) +static uint64_t n_gpa_late; // replay: records handed to their mirror in queue order, so far + +static int gpa_late(void); +int vk_mem_gpa_late(void) { return gpa_late(); } // --probe-vkmem static int gpa_late(void) { static int on = -1; if (on < 0) { on = rt_knob("GPA_LATE", "1") == 1; - rt_log("mem: the device address table reads a new VRAM mirror %s (NX2NAVI_GPA_LATE)", - on ? "once the mirror's first copy has run" : "at once, as before"); + rt_log("mem: the device address table takes up a new VRAM mirror %s (NX2NAVI_GPA_LATE)", + on ? "in queue order, after the mirror's first copy" : "at once, as before"); } return on; } -// Replay thread, at the start of a submission: the mirrors whose first copy the GPU has finished take the table's reads. -static void gpa_switch(void) { - if (!__atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) return; - uint64_t done = vk_sync_fence_done(); +static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, + VkAccessFlags2 da); + +// Prologue, after the copies into the mirrors: the stores that hand the new mirrors' records over. Everything queued +// before is done with the table before the stores run, and everything after sees them. +static void gpa_flip_cmds(VkCommandBuffer cb) { + if (!n_gpa_flip) return; + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); plat_mutex_lock(&pool_lock); - for (int i = 0; i < nm && n_gpa_wait; i++) { + for (int i = 0; i < nm && n_gpa_flip; i++) { PoolRec *r = mlist[i]; - if (!r->gpa_at || done < r->gpa_at) continue; - r->gpa_at = 0, n_gpa_wait--; - if (!r->gpa_rec || !r->mbuf) continue; - r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) - : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); - n_gpa_late++; + if (!r->gpa_flip) continue; + r->gpa_flip = 0, n_gpa_flip--; + if (r->gpa_rec && r->mbuf && vk_gpa_flip_cmd(cb, r->gpa_rec, r->mbuf, r->import_lo, r->twin)) n_gpa_late++; } + n_gpa_flip = 0; plat_mutex_unlock(&pool_lock); + mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); } // Replay thread: the pool's mirror on its first buffer use. @@ -2897,25 +2905,19 @@ static void mirror_create(PoolRec *r) { r->mstate = MS_ON; r->asked_room = 0; // The device address table names the mirror: the GPU's writes go there, and the sync copies the CPU's. With a - // twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. Reads - // move to the mirror once its first copy has run (gpa_late, below). + // twin they also go to the imported buffer; without one (twin_wanted) the record names the mirror twice. The + // table takes the mirror up in queue order, after its first copy (gpa_late, above). r->twin = (uint8_t)twin_wanted(r); if (!r->twin) { plat_inc64(&m_untwinned_n); plat_add64(&m_untwinned_bytes, (int64_t)n); } - if (r->gpa_rec) { + if (r->gpa_rec && gpa_late()) { + r->gpa_flip = 1, n_gpa_flip++; // vk_mem_mirror_prologue, with the copy range_add queues below + } else if (r->gpa_rec) { plat_mutex_lock(&pool_lock); - uint32_t late = gpa_late() ? vk_gpa_remap_writes(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) : 0; - if (late) { - uint64_t f = vk_exec_recording_fence(); - r->gpa_rec = late; - r->gpa_at = f ? f : vk_sync_issued() + 1; // the submission being recorded: its prologue holds the copy - n_gpa_wait++; - } else { - r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) - : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); - } + r->gpa_rec = r->twin ? vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo) + : vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); plat_mutex_unlock(&pool_lock); } range_add(&pend, &npend, &cpend, r, 0, n); @@ -2926,7 +2928,7 @@ static void mirror_create(PoolRec *r) { static void mirror_forget(PoolRec *r) { __atomic_add_fetch(&mirror_gen, 1, __ATOMIC_RELEASE); - if (r->gpa_at) r->gpa_at = 0, n_gpa_wait--; // the caller takes the table's pages off the mirror + if (r->gpa_flip) r->gpa_flip = 0, n_gpa_flip--; // gone before its first copy: its record stays as it is for (int i = 0; i < nm; i++) if (mlist[i] == r) { memmove(mlist + i, mlist + i + 1, (size_t)(nm - i - 1) * sizeof *mlist); @@ -3993,7 +3995,7 @@ static void hot_check(void) { } void vk_mem_mirror_begin(void) { - if (vk_mirror_on) gpa_switch(), hot_check(); + if (vk_mirror_on) hot_check(); if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions if (!freeze_idle) return; plat_mutex_lock(&gen_lock); @@ -4039,7 +4041,7 @@ void vk_mem_mirror_begin(void) { slow_mem_call("mirror begin (its queries and freezes)", 0, ww_bytes - b0, q, qpc() - t0); } -int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; } +int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl || n_gpa_flip; } // NX2NAVI_MIRROR_CAPTURE_KB: the pages the CPU wrote since a VRAM mirror's last sync are copied out of guest memory by // the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a host buffer the @@ -4163,7 +4165,7 @@ static void mirror_capture(VkCommandBuffer cb) { } void vk_mem_mirror_prologue(VkCommandBuffer cb) { - if (!npend && snl_copied == nsnl) return; + if (!npend && snl_copied == nsnl && !n_gpa_flip) return; mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); mirror_capture(cb); @@ -4176,6 +4178,7 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb) { } mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); + gpa_flip_cmds(cb); // after the new mirrors' first copies npend = 0; mrep.prologues++; } @@ -4448,10 +4451,9 @@ static void mirror_log(uint64_t frame) { "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; flushed and not copied (write-watched) %.3f MiB%s", (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M, a.flush_skipped / f / M, capl); - if (n_gpa_late || __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)) - rt_log("mem: mirror frame %llu: the device address table took up %llu mirrors after their first copy so far, %d " - "wait for theirs (NX2NAVI_GPA_LATE)", (unsigned long long)frame, (unsigned long long)n_gpa_late, - __atomic_load_n(&n_gpa_wait, __ATOMIC_RELAXED)); + if (n_gpa_late) + rt_log("mem: mirror frame %llu: the device address table took up %llu mirrors in queue order, after their first " + "copy, so far (NX2NAVI_GPA_LATE)", (unsigned long long)frame, (unsigned long long)n_gpa_late); } // After vram_log: every heap's usage as the driver reports it (VK_EXT_memory_budget) beside VMA's share of it, VMA by @@ -5562,7 +5564,7 @@ void vk_mem_shutdown(void) { plat_mutex_lock(&pool_lock); for (int i = 0; i < pool_index.n; i++) destroy_rec(c, pool_index.e[i].buf); pool_index.n = 0; - nm = npend = nwbl = nsnl = snl_copied = nswl = 0; // mirrors went with their pools + nm = npend = nwbl = nsnl = snl_copied = nswl = n_gpa_flip = 0; // mirrors went with their pools swl_marks = -1; plat_mutex_lock(&snap_pool_lock); for (int i = 0; i < nsnap_pool; i++) vmaDestroyBuffer(c->vma, snap_pool[i].buf, snap_pool[i].alloc); diff --git a/tests/gpa_check.c b/tests/gpa_check.c index 7bda3d39..47a1c24d 100644 --- a/tests/gpa_check.c +++ b/tests/gpa_check.c @@ -49,6 +49,10 @@ VkResult vmaCreateBuffer(VmaAllocator a, const VkBufferCreateInfo *bi, const Vma return VK_SUCCESS; } void vmaDestroyBuffer(VmaAllocator a, VkBuffer buf, VmaAllocation alloc) { (void)a, (void)buf; free(alloc); } +VkResult vmaInvalidateAllocation(VmaAllocator a, VmaAllocation alloc, VkDeviceSize off, VkDeviceSize n) { + (void)a, (void)alloc, (void)off, (void)n; + return VK_SUCCESS; +} // A buffer's device address: its handle far above the GPU addresses the tests map. static uint64_t dev_of(VkBuffer b) { return (uint64_t)(uintptr_t)b << 28; } @@ -73,6 +77,22 @@ static void gpu_done(void) { // the submission being recorded is issued and fi nqueue = k; } +// A store the GPU runs in queue order (vk_gpa_flip_cmd): kept until gpu_runs_stores, as a command buffer keeps it +// until the GPU reaches it. +static struct { VkDeviceSize off, n; uint8_t data[32]; } stores[8]; +static int nstores; +void vkCmdUpdateBuffer(VkCommandBuffer cb, VkBuffer b, VkDeviceSize off, VkDeviceSize n, const void *data) { + (void)cb, (void)b; + if (nstores == 8 || n > sizeof stores[0].data || (off | n) & 3) { printf("FAIL: a store the stand-in cannot hold\n"); exit(1); } + stores[nstores].off = off, stores[nstores].n = n; + memcpy(stores[nstores].data, data, (size_t)n); + nstores++; +} +static void gpu_runs_stores(void) { + for (int i = 0; i < nstores; i++) memcpy((uint8_t *)(uintptr_t)table + stores[i].off, stores[i].data, (size_t)stores[i].n); + nstores = 0; +} + // ---- the table as a shader reads it (the layout at the top of vk_gpa.c) enum { L1_WORD = 16, L1_N = 65536, L2_WORD = L1_WORD + L1_N, L2_N = 4096, BLOCKS = 1024 }; static uint32_t block_of(uint64_t addr) { return table[L1_WORD + (addr >> 24)]; } @@ -125,27 +145,35 @@ int main(void) { CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "a plain page is its own twin"); gpu_done(); - // a new mirror takes the writes first and the reads once its first copy has run (vk_memory.c mirror_create, - // gpa_switch): until then the pages read the imported buffer and the mirror is their twin + // a new mirror takes the pool's record over by a store the GPU runs after the mirror's first copy (vk_memory.c + // mirror_create, vk_mem_mirror_prologue): what is queued before the store reads and writes the imported buffer + // alone, what comes after it reads the mirror and writes both VkBuffer m1 = new_buf(); - uint32_t w0 = vk_gpa_remap_writes(a0, a1, r0, m1, a0); - CHECK(w0 && w0 != r0, "a new mirror gets a record for the writes"); - expect(a0, a1, b0, a0, "a new mirror before its first copy: reads stay on the imported buffer"); - CHECK(read_through(a0 + 4096, 1) == dev_of(m1) + 4096 && read_through(a1 - 4096, 1) == dev_of(m1) + (a1 - 4096 - a0), - "a new mirror before its first copy is the twin, which takes the writes"); + CHECK(vk_gpa_flip_cmd(VK_NULL_HANDLE, r0, m1, a0, 1), "a new mirror's store is recorded"); + expect(a0, a1, b0, a0, "before the GPU reaches the store: reads stay on the imported buffer"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096 && read_through(a1 - 4096, 1) == dev_of(b0) + (a1 - 4096 - a0), + "before the GPU reaches the store nothing is written into the mirror"); + gpu_runs_stores(); + expect(a0, a1, m1, a0, "after the store: reads are on the mirror"); + CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096 && read_through(a1 - 4096, 1) == dev_of(b0) + (a1 - 4096 - a0), + "after the store the imported buffer is the twin"); gpu_done(); - r0 = vk_gpa_remap(a0, a1, w0, m1, a0); - expect(a0, a1, m1, a0, "a new mirror after its first copy: reads move to it"); - CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "and the imported buffer is the twin again"); - // evicted while it still waits for its first copy: the pages go back to the imported buffer alone + // evicted: the pages go back to the imported buffer alone, on a record of their own + uint32_t was = r0; r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0); - gpu_done(); - VkBuffer m2 = new_buf(); - w0 = vk_gpa_remap_writes(a0, a1, r0, m2, a0); - r0 = vk_gpa_remap_plain(a0, a1, w0, b0, a0); - expect(a0, a1, b0, a0, "a mirror gone before its first copy"); + CHECK(r0 && r0 != was, "an evicted mirror's pages take a new record"); + expect(a0, a1, b0, a0, "an evicted mirror"); CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "leaves the imported buffer as its own twin"); - CHECK(!vk_gpa_remap_writes(a0, a1, 0, m2, a0), "a pool without a record gets none for a mirror's writes"); + gpu_done(); + // a mirror without a twin: the store names it twice + VkBuffer m2 = new_buf(); + CHECK(vk_gpa_flip_cmd(VK_NULL_HANDLE, r0, m2, a0, 0), "an untwinned mirror's store is recorded"); + gpu_runs_stores(); + expect(a0, a1, m2, a0, "an untwinned mirror"); + CHECK(read_through(a0 + 4096, 1) == dev_of(m2) + 4096, "is its own twin"); + r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0); + expect(a0, a1, b0, a0, "and goes back to the imported buffer"); + CHECK(!vk_gpa_flip_cmd(VK_NULL_HANDLE, 0, m2, a0, 1) && !nstores, "a pool without a record gets no store"); gpu_done(); // an unmapped range reads the zero buffer at once, and its blocks serve no other range while the GPU may still read