diff --git a/runtime/vk.h b/runtime/vk.h index dc625bce..90408b1f 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -253,6 +253,7 @@ void vk_mem_dirty_submit(void); // vk_sync_submit, under its l extern int vk_mirror_on; void vk_mem_use(VkMemSpan *sp); void vk_mem_pick(VkMemSpan *sp); +VkBuffer vk_mem_guest_twin(const VkMemSpan *sp); // a VRAM mirror span: the pool's imported buffer (guest memory), same offset; else none #define VK_MEM_USE(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_use(&(sp)); } while (0) #define VK_MEM_PICK(sp) do { if (vk_mirror_on && (sp).pool) vk_mem_pick(&(sp)); } while (0) void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size); diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 0a7120d1..aebc29d3 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -700,6 +700,10 @@ static void clear_buf(VkExec *ex, const p_clear_buf *p) { if (!started) vk_exec_pass_end(ex), vk_exec_barrier(ex), started = 1; VK_MEM_PICK(d); vkCmdFillBuffer(ex->cb, d.buffer, d.offset, n, p->value); + // A VRAM mirror's guest copy is filled too, at once: atomics through the device address table count in both + // copies (translator patch 0036), and a counter reset here must start from the same value in both. + VkBuffer twin = vk_mem_guest_twin(&d); + if (twin) vkCmdFillBuffer(ex->cb, twin, d.offset, n, p->value), ex->recorded++; vk_mem_gpu_wrote(&d, n); ex->recorded++; vk_mem_count(d.cls, VK_MUSE_GPU_WRITE, n); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 6234e53f..301eae4f 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -63,6 +63,8 @@ typedef struct PoolRec { VmaAllocation malloc_; uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync) uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory + uint32_t vd_wrote_sub, vd_use_sub; // verify mode: the last submission that bound it written / used its mirror + uint64_t vd_wrote_n; // verify mode: written bindings of it so far uint32_t mgen; // gen_lock: every page write stamped up to this generation is in the mirror // (or in a copy queued before its next use) FlushRange fl[FLUSH_RANGES + 1]; // flush_lock: nvnMemoryPoolFlushMappedRange pages not copied yet (flush_add) @@ -1213,7 +1215,7 @@ static void dirty_frame(uint64_t frame) { // another pool or the allocation fails. #define VERIFY_BYTES (4u << 20) #define VERIFY_SAMPLES 64 -#define VERIFY_LOG 16 +#define VERIFY_LOG 200 enum { MR_BUDGET, MR_UNTRACKED, MR_OVERLAP, MR_ALLOC, MR_N }; int vk_mirror_on; @@ -1502,6 +1504,7 @@ void vk_mem_use(VkMemSpan *sp) { if (r->mstate == MS_NONE) mirror_create(r); if (r->mstate != MS_ON) return; if (r->acc_sub != msub) mirror_sync(r); + r->vd_use_sub = msub; sp->buffer = r->mbuf; } @@ -1512,6 +1515,11 @@ void vk_mem_pick(VkMemSpan *sp) { sp->buffer = r->mbuf; } +VkBuffer vk_mem_guest_twin(const VkMemSpan *sp) { + const PoolRec *r = sp->pool; + return r && r->mstate == MS_ON && sp->buffer == r->mbuf ? r->buf : VK_NULL_HANDLE; +} + int vk_mem_mirror_probe(uint64_t addr) { VkMemSpan sp; if (!vk_mirror_on || !vk_mem_resolve(addr, 1, &sp) || !sp.pool) return 0; @@ -1755,6 +1763,7 @@ static void snap_free(void *arg) { // completion thread: the submission is don static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; + r->vd_wrote_sub = msub, r->vd_wrote_n++; // A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies. if (shader && !(sp->offset & 3) && !(size & 3) && size <= MERGE_MAX && merging() && snap_add(r, sp->offset, size)) return; @@ -1948,6 +1957,13 @@ static void verify_cb(void *arg) { // completion thread rt_log("mem: verify mismatch: pool %#llx+%#llx (%s) offset %#llx: bytes %#x..%#x of the page differ, " "mirror %02x guest %02x at the first (submission %u)", (unsigned long long)r->lo, (unsigned long long)r->size, f, (unsigned long long)(a - r->lo), k, e, m[k], g[k], j->sub); + const uint32_t *gs = gen_slot(a & ~((1ull << GEN_PAGE_SHIFT) - 1), 0); + char mh[100], gh[100]; + for (int b = 0; b < 16; b++) snprintf(mh + 3 * b, 4, "%02x ", m[b]), snprintf(gh + 3 * b, 4, "%02x ", g[b]); + rt_log("mem: verify mismatch detail: written binds %llu, last %u; used %u; written back %u; gpa record %u; " + "page cpu stamp %u (synced to %u, now %u); mirror %s| guest %s", + (unsigned long long)r->vd_wrote_n, r->vd_wrote_sub, r->vd_use_sub, wb, r->gpa_rec, gs ? *gs : 0, + s->mgen, gen_now, mh, gh); } } } diff --git a/tools/build_xlate.py b/tools/build_xlate.py index cf356d9c..20536448 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 34: - sys.exit(f"expected 34 patches, found {len(patches)}") + if len(patches) != 35: + sys.exit(f"expected 35 patches, found {len(patches)}") for patch in patches: # git apply works outside a repository when given a directory; --check first so a failure names the patch. for extra in (["--check"], []): diff --git a/tools/xlate/patches/0036-device-address-twin-atomics.patch b/tools/xlate/patches/0036-device-address-twin-atomics.patch new file mode 100644 index 00000000..20711630 --- /dev/null +++ b/tools/xlate/patches/0036-device-address-twin-atomics.patch @@ -0,0 +1,93 @@ +--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs ++++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs +@@ -1795,19 +1795,25 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv + AstOperation operation, + Func emitU) + { +- SpvInstruction elemPointer = GetStoragePointer(context, operation, out AggregateType varType); ++ SpvInstruction elemPointer = GetStoragePointer(context, operation, out AggregateType varType, out SpvInstruction twin); + + SpvInstruction value = context.Get(varType, operation.GetSource(operation.SourcesCount - 1)); + + SpvInstruction one = context.Constant(context.TypeU32(), 1); + SpvInstruction zero = context.Constant(context.TypeU32(), 0); + +- return new OperationResult(varType, emitU(context.GetType(varType), elemPointer, one, zero, value)); ++ SpvInstruction result = emitU(context.GetType(varType), elemPointer, one, zero, value); ++ if (twin != null) ++ { ++ emitU(context.GetType(varType), twin, one, zero, value); // NX2NAVI: guest memory's copy (GetDeviceAddressPointer) ++ } ++ ++ return new OperationResult(varType, result); + } + + private static OperationResult GenerateAtomicMemoryCas(CodeGenContext context, AstOperation operation) + { +- SpvInstruction elemPointer = GetStoragePointer(context, operation, out AggregateType varType); ++ SpvInstruction elemPointer = GetStoragePointer(context, operation, out AggregateType varType, out SpvInstruction twin); + + SpvInstruction value0 = context.Get(varType, operation.GetSource(operation.SourcesCount - 2)); + SpvInstruction value1 = context.Get(varType, operation.GetSource(operation.SourcesCount - 1)); +@@ -1815,7 +1821,14 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv + SpvInstruction one = context.Constant(context.TypeU32(), 1); + SpvInstruction zero = context.Constant(context.TypeU32(), 0); + +- return new OperationResult(varType, context.AtomicCompareExchange(context.GetType(varType), elemPointer, one, zero, zero, value1, value0)); ++ SpvInstruction result = context.AtomicCompareExchange(context.GetType(varType), elemPointer, one, zero, zero, value1, value0); ++ if (twin != null) ++ { ++ // NX2NAVI: the same exchange on guest memory's copy, which holds the same word while both are in step ++ context.AtomicCompareExchange(context.GetType(varType), twin, one, zero, zero, value1, value0); ++ } ++ ++ return new OperationResult(varType, result); + } + + private static OperationResult GenerateLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore) +@@ -1864,9 +1877,12 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv + // The pointer addresses the 32-bit word at the address's 4-byte-aligned word. Stores also go to the twin + // (GetDeviceAddressPointers): what the GPU writes into a mirror through the table is not recorded as a GPU write, + // so without it the CPU never saw it and a sync of a CPU-written page copied the stale guest page over it +- // (runtime/vk_gpa.c). Atomics do not: the twin's word may be stale (a counter the GPU reset with a buffer fill +- // went on counting from its old value in guest memory), and that page's next sync copied the wrong count into +- // the mirror, so culled instance lists drew garbage for a frame (flickering foliage). ++ // (runtime/vk_gpa.c). So do atomics (GenerateAtomicMemoryBinary, GenerateAtomicMemoryCas): the culling passes ++ // count instances with them into records the CPU rewrites other words of each frame, and with the counts in the ++ // mirror alone the sync of such a page copied the guest's zeros over them (foliage that flickered, went missing or ++ // stretched into columns). The twin's atomic leaves its result unused, so the driver can issue it without a round ++ // trip to guest memory. Buffer fills reach the twin at once (runtime/vk_exec_xfer.c clear_buf), so a counter the ++ // GPU resets with one counts on from the same value in both. + private static SpvInstruction GetDeviceAddressPointer(CodeGenContext context, int binding, SpvInstruction low, SpvInstruction high) + { + return GetDeviceAddressPointers(context, binding, low, high, out _); +@@ -1983,19 +1999,27 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv + } + + private static SpvInstruction GetStoragePointer(CodeGenContext context, AstOperation operation, out AggregateType varType) ++ { ++ return GetStoragePointer(context, operation, out varType, out _); ++ } ++ ++ // twin: for a device address table pointer, the twin GetDeviceAddressPointers gives; null for any other storage. ++ private static SpvInstruction GetStoragePointer(CodeGenContext context, AstOperation operation, out AggregateType varType, out SpvInstruction twin) + { + StorageKind storageKind = operation.StorageKind; ++ twin = null; + + // NX2NAVI: atomics on the device address table's pointers (GenerateDeviceAddressLoadOrStore covers loads and + // stores): sources are the table binding and the address's two words. + if (storageKind == StorageKind.GlobalMemory) + { + varType = AggregateType.U32; +- return GetDeviceAddressPointer( ++ return GetDeviceAddressPointers( + context, + ((AstOperand)operation.GetSource(0)).Value, + context.Get(AggregateType.U32, operation.GetSource(1)), +- context.Get(AggregateType.U32, operation.GetSource(2))); ++ context.Get(AggregateType.U32, operation.GetSource(2)), ++ out twin); + } + + StorageClass storageClass;