diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 9d5249c3..0581860d 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -396,6 +396,33 @@ static int test_import(pool_t *pool) { return 0; } +// A GPU write is noted on a pool, and the pool is finalized and made again at its address before the write is +// reported: the report must leave alone what the CPU has written to the new pool since. With shadow storage the new +// pool's shadow is a copy of guest memory from before those writes and holds no byte of the old write +// (vk_memory.c shadow_write_back); with an import the report copies nothing. +static int test_remade(uint8_t *guest) { + pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xc00000, .size = 0x10000, + .game_mem = guest + 0xc00000}; + uint8_t *m = pool.mem; + uint64_t a = (uintptr_t)m; + memset(m, 0x11, pool.size); + GCHECK(vk_pool_init(&pool), "vk_pool_init of the pool to be made again"); + VkMemSpan sp; + GCHECK(vk_mem_resolve(a, 0x2000, &sp) && sp.buffer, "resolve inside the pool to be made again"); + vk_mem_gpu_pending(a, 0x2000); // the submission being recorded writes its first two pages + vk_pool_fini(&pool); + GCHECK(vk_pool_init(&pool), "vk_pool_init of the pool made again"); + memset(m, 0x22, 0x1000); // the CPU writes the first page of the new pool + vk_gpu_written(a, 0x2000); // the old submission's fence + GCHECK(m[0] == 0x22 && m[0xfff] == 0x22, "the report of a write noted on a finalized pool put %#x over the CPU's " + "bytes in the pool made at its address", m[0]); + GCHECK(m[0x1000] == 0x11 && m[0x1fff] == 0x11, "the page the CPU left alone changed (%#x)", m[0x1000]); + vk_pool_fini(&pool); + printf("probe-vkmem: a write reported for a pool that was made again leaves the CPU's newer bytes alone\n"); + g_gpu_checks += 2; + return 0; +} + // test_gpa_read's shader (runtime/shaders/gpa_read.spvasm): one invocation follows the device address table from a // guest address (the push constant) to the buffer that holds it and stores the 32-bit word found there in the result // buffer. Storage bindings: 0 the table itself, 1 the result. The pool that holds the word is never a descriptor, a @@ -1053,7 +1080,7 @@ static int test_gpu(void) { vk_mem_stats(&st); GCHECK(!st.imported_pools && !st.physical_pools && !st.virtual_pools, "pools left after fini"); g_gpu_checks += 2; - return test_gpa_read(h) || test_table_store(h); + return test_gpa_read(h) || test_table_store(h) || test_remade(h); } int probe_vkmem(void) { diff --git a/runtime/vk.h b/runtime/vk.h index db3304ea..3d2c4981 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -681,6 +681,7 @@ int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs // completion thread's time by callback. void vk_sync_exec_stats(uint64_t *waits, uint64_t *early, uint64_t *lag_ns, uint64_t *lag_max_ns); int vk_sync_effect_late(void); +uint64_t vk_sync_cb_value(void); // the value whose side effect the calling thread is running; 0: not in one void vk_sync_cb_stats(char *out, size_t n); void vk_exec_late_stats(uint64_t *events, uint64_t *counters, uint64_t *counts); // vk_exec_xfer.c: such callbacks since the last call VkCommandBuffer vk_sync_split_exec(void); // vk_sync_split for the part before a submission's write-backs diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 519de236..1ea01c1e 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -84,6 +84,7 @@ typedef struct PoolRec { VmaAllocation salloc; uint8_t *smap; // its mapping; offsets as in buf (from import_lo) uint32_t sgen; // gen_lock: every page write stamped up to this generation is in the shadow + uint64_t sborn; // the value of the first submission that can have used the shadow (shadow_pool) volatile uint8_t stouched; // on the touched list: the GPU may use it in the submission being recorded // shadow_lock: base copies of the pages a noted GPU write covers (merge_page), open addressing by page address; // csb a power of two, page 0 an empty slot @@ -1134,6 +1135,7 @@ static uint64_t s_in_merge, s_in_nomirror; // check: bytes copied in by m static uint64_t s_gpa_ticks, s_gpa_syncs; // vk_mem_gpa_sync: its time (part of s_ticks) and its runs static uint64_t s_t_over_all; // shadow_lock: s_tab_over since the start static uint64_t s_late_pages, s_late_bytes; // shadow_lock, check: write-backs into pages the CPU wrote since their copy-in +static uint64_t s_stale_pages; // shadow_lock: write-backs left out, the shadow could not hold them (shadow_write_back) static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) { r->import_lo = addr & ~0xfffull; @@ -1154,6 +1156,7 @@ static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) { return 0; } r->smap = info.pMappedData; + r->sborn = vk_sync_issued() + 1; // the submission being recorded now, or the next one plat_mutex_lock(&gen_lock); r->sgen = gen_now; // the copy below holds every write stamped so far; later stamps are copied again plat_mutex_unlock(&gen_lock); @@ -1533,9 +1536,19 @@ static int cpu_wrote_since(uint64_t pg, uint32_t sgen) { // Completion thread: the GPU's bytes of [addr, addr+size) from the shadow into guest memory (merge_page against the // base copies; a page without one whole). Returns the bytes moved. +// A page without a base copy is one no write was noted on in this record (vk_mem_gpu_pending notes every write the +// replay records). It gets here when the pool the write was noted on was finalized before the write's fence and +// another was made at its address: shadow_forget dropped the base copies, and the new record's shadow is a copy of +// guest memory from its own start that holds no byte of that write. Copied whole, it put those older bytes over +// everything the CPU had written to the page since, for no gain. So the page is left alone when the record is younger +// than the submission whose fence this is (sborn), or when the CPU has written the page since the shadow's last +// copy-in (cpu_wrote_since: a pool made again while that submission was still being recorded has the same value). +// Any other such page goes whole as before: a write nothing noted, into a shadow that holds it. static uint64_t shadow_write_back(uint64_t addr, uint64_t size) { uint64_t moved = 0; int bulk = shadow_bulk_write(); // before the locks: its first call logs + uint64_t value = vk_sync_cb_value(); // the fence being completed; a call from elsewhere reports submitted work + if (!value) value = vk_sync_issued(); plat_mutex_lock_shared(&pool_lock); const pidx_ent *e = pidx_find(&pool_index, addr, size); PoolRec *r = e ? e->buf : NULL; @@ -1551,6 +1564,7 @@ static uint64_t shadow_write_back(uint64_t addr, uint64_t size) { int late = base && s_check && cpu_wrote_since(pg, r->sgen); // before the merge's own stores into the page uint64_t n = 0; if (base) moved += n = merge_page((uint8_t *)(uintptr_t)a, s, base + (a - pg), pe - a); + else if (r->sborn > value || cpu_wrote_since(pg, r->sgen)) s_stale_pages++; // the shadow cannot hold the write else memcpy((void *)(uintptr_t)a, s, pe - a), moved += pe - a; // a write never noted as pending if (late && n) s_late_pages++, s_late_bytes += n; a = pe; @@ -1729,11 +1743,14 @@ static void shadow_log(uint64_t frame) { plat_mutex_lock(&shadow_lock); rt_log("mem: shadow at frame %llu: %lld pools %.1f MiB; since the last: %llu pool syncs, %llu queries, %.1f MiB " "copied in (%llu untracked), %.2f ms (queries %.2f); %.1f MiB written back of %.1f MiB the GPU may have written, %.2f ms; %llu " - "pages the GPU writes keep a base copy", + "pages the GPU writes keep a base copy; %llu pages not written back, their pool was made again or the CPU " + "wrote them since", (unsigned long long)frame, (long long)s_live_n, s_live_bytes / 1048576.0, (unsigned long long)s_syncs, (unsigned long long)s_queries, s_copied / 1048576.0, (unsigned long long)s_untracked, qpc_ms(s_ticks), - qpc_ms(s_q_ticks), s_wb_bytes / 1048576.0, s_wb_seen / 1048576.0, qpc_ms(s_wb_ticks), (unsigned long long)s_sb_pages); + qpc_ms(s_q_ticks), s_wb_bytes / 1048576.0, s_wb_seen / 1048576.0, qpc_ms(s_wb_ticks), (unsigned long long)s_sb_pages, + (unsigned long long)s_stale_pages); s_syncs = s_queries = s_copied = s_untracked = s_wb_bytes = s_ticks = s_wb_seen = s_wb_ticks = s_q_ticks = 0; + s_stale_pages = 0; if (s_learn || s_check) shadow_table_log(frame); plat_mutex_unlock(&shadow_lock); if (s_check) plat_mutex_unlock_shared(&pool_lock); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index b94a1705..f11d8261 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -147,6 +147,14 @@ static volatile int32_t cb_late; // the callback being run belongs int vk_sync_effect_late(void) { return __atomic_load_n(&cb_late, __ATOMIC_RELAXED); } +// The value whose side effect the calling thread is running, 0 when it is not the completion thread inside one. +static volatile uint64_t cb_value; +static volatile uint32_t cb_tid; + +uint64_t vk_sync_cb_value(void) { + return __atomic_load_n(&cb_tid, __ATOMIC_ACQUIRE) == plat_thread_id() ? __atomic_load_n(&cb_value, __ATOMIC_RELAXED) : 0; +} + void vk_sync_exec_stats(uint64_t *waits, uint64_t *early, uint64_t *lag_ns, uint64_t *lag_max_ns) { *waits = __atomic_exchange_n(&exw_n, 0, __ATOMIC_RELAXED), *early = __atomic_exchange_n(&exw_early, 0, __ATOMIC_RELAXED); *lag_ns = __atomic_exchange_n(&exw_lag_ns, 0, __ATOMIC_RELAXED); @@ -482,6 +490,7 @@ WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint static void completion_main(void *arg) { VkCtx *c = arg; + __atomic_store_n(&cb_tid, plat_thread_id(), __ATOMIC_RELEASE); plat_mutex_lock(&st_lock); for (;;) { while (!stop && done == issued && !(qn && queue[qhead].fence <= done)) @@ -528,7 +537,9 @@ static void completion_main(void *arg) { plat_mutex_unlock(&st_lock); __atomic_store_n(&cb_late, p.fence <= __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE), __ATOMIC_RELAXED); uint64_t c0 = (rt_trace & RT_TRACE_PERF) ? kern_now_ns() : 0; + __atomic_store_n(&cb_value, p.fence, __ATOMIC_RELAXED); p.fn(p.arg); + __atomic_store_n(&cb_value, 0, __ATOMIC_RELAXED); uint64_t c1 = c0 ? kern_now_ns() : 0; __atomic_store_n(&cb_late, 0, __ATOMIC_RELAXED); plat_mutex_lock(&st_lock);