diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index 691901d7..99c06e49 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -8,9 +8,17 @@ // One storage buffer of 32-bit words, bound whole to every shader that declares it (res.json "table", cb_slot 0x40): // words 0-1 the device address of a zeroed 4 KiB buffer, which an address no page maps reads and writes instead // word 2 the word index of the records (REC_WORD) +// words 3-4 the device address of a 4 KiB scratch buffer, where a twin write goes when its page has no twin // words 16.. L1: 65536 entries, one per 16 MiB of the 40-bit GPU address space, the index of its L2 block (0: none) // word 65552 L2 blocks of 4096 entries, one per 4 KiB page: the record of the buffer that holds the page (0: none) -// REC_WORD records of two words: the buffer's device address minus the GPU address of its byte 0 (low word first) +// REC_WORD records of four words: the buffer's device address minus the GPU address of its byte 0, then the same +// for the page's twin (low words first) +// The twin: a pool with a VRAM mirror (vk_memory.c mirror_create) has the mirror in its record's first address and its +// imported buffer, guest memory, in the second; any other record has the same buffer in both. Shaders read through the +// first and write through both (translator patch 0034): the GPU's writes through the table are not recorded as GPU +// writes (vk_exec_draw.c ssbo_info), so with the mirror alone the CPU never saw them, and a sync of a page the CPU wrote +// copied the stale guest page over them. The game's compute passes write tree and foliage records this way, and a +// record overwritten so drew a tree stretched into a column reaching the sky (Hyrule Field, Kakariko, Death Mountain). // A record a page entry names is never written: a new mapping takes a new record, and a record goes back to the free // list only once the GPU finished every submission issued while a page could still name it (vk_completion_add). Page // and L1 entries are single 32-bit stores, so a shader that reads the table while it changes sees the old mapping or @@ -36,13 +44,14 @@ #define BLOCKS 1024u // L2 blocks: 16 GiB of mapped GPU address space; block 0 stays zero #define REC_WORD (L2_WORD + BLOCKS * L2_N) #define RECS 65536u // record 0 means none -#define WORDS (REC_WORD + RECS * 2u) +#define REC_WORDS 4u // words per record +#define WORDS (REC_WORD + RECS * REC_WORDS) #define PAGE_SHIFT 12 static struct { int state; // 0 untried, 1 on, -1 off - VkBuffer buf, dummy; - VmaAllocation alloc, dummy_alloc; + VkBuffer buf, dummy, scratch; + VmaAllocation alloc, dummy_alloc, scratch_alloc; volatile uint32_t *gpu; // the mapped table uint32_t *cpu; // its shadow uint32_t nblocks; // L2 blocks handed out (block 0 is the zero block) @@ -99,8 +108,13 @@ static int create(void) { return 0; } memset(dinfo.pMappedData, 0, 4096); - uint64_t z = vk_gpa_device_address(G.dummy); - put(0, (uint32_t)z), put(1, (uint32_t)(z >> 32)), put(2, REC_WORD); + VmaAllocationCreateInfo si = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (vmaCreateBuffer(c->vma, &di, &si, &G.scratch, &G.scratch_alloc, NULL) != VK_SUCCESS) { + rt_log("vk_gpa: the scratch buffer could not be allocated"); + return 0; + } + uint64_t z = vk_gpa_device_address(G.dummy), sc = vk_gpa_device_address(G.scratch); + put(0, (uint32_t)z), put(1, (uint32_t)(z >> 32)), put(2, REC_WORD), put(3, (uint32_t)sc), put(4, (uint32_t)(sc >> 32)); G.nblocks = 1, G.nrec = 1; rt_log("vk_gpa: device address table on (%.1f MiB, %u blocks of 16 MiB, %u records)", WORDS * 4 / 1048576.0, BLOCKS, RECS); return 1; @@ -132,14 +146,16 @@ static uint32_t page_word(uint64_t page, int alloc) { return L2_WORD + blk * L2_N + (uint32_t)(page & (L2_N - 1)); } -static uint32_t rec_alloc(uint64_t delta) { +static uint32_t rec_alloc(uint64_t delta, uint64_t twin) { uint32_t r; if (G.nfree) r = G.free_recs[--G.nfree]; else if (G.nrec < RECS) r = G.nrec++; else return 0; // The record before any page names it: the page store below is ordered after these by the release fence. - put(REC_WORD + r * 2, (uint32_t)delta); - put(REC_WORD + r * 2 + 1, (uint32_t)(delta >> 32)); + put(REC_WORD + r * REC_WORDS, (uint32_t)delta); + put(REC_WORD + r * REC_WORDS + 1, (uint32_t)(delta >> 32)); + put(REC_WORD + r * REC_WORDS + 2, (uint32_t)twin); + put(REC_WORD + r * REC_WORDS + 3, (uint32_t)(twin >> 32)); __atomic_thread_fence(__ATOMIC_RELEASE); return r; } @@ -162,7 +178,7 @@ uint32_t vk_gpa_map(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo) { uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; if (p1 <= p0) return 0; plat_mutex_lock(&gpa_lock); - uint32_t r = rec_alloc(dev - buf_lo); + uint32_t r = rec_alloc(dev - buf_lo, dev - buf_lo); if (!r) { G.refused++; plat_mutex_unlock(&gpa_lock); @@ -227,7 +243,11 @@ uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t dev = vk_gpa_device_address(buf); uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; plat_mutex_lock(&gpa_lock); - uint32_t r = dev ? rec_alloc(dev - buf_lo) : 0; + // the twin: the buffer the pages named before (the pool's imported buffer), so writes reach guest memory too + uint64_t twin = old_rec ? (uint64_t)G.cpu[REC_WORD + old_rec * REC_WORDS] | + (uint64_t)G.cpu[REC_WORD + old_rec * REC_WORDS + 1] << 32 + : dev - buf_lo; + uint32_t r = dev ? rec_alloc(dev - buf_lo, twin) : 0; if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } for (uint64_t p = p0; p < p1; p++) { uint32_t w = page_word(p, 1); @@ -253,6 +273,7 @@ void vk_gpa_shutdown(void) { if (G.state <= 0 || !c) return; vmaDestroyBuffer(c->vma, G.buf, G.alloc); vmaDestroyBuffer(c->vma, G.dummy, G.dummy_alloc); + vmaDestroyBuffer(c->vma, G.scratch, G.scratch_alloc); free(G.cpu), free(G.free_recs); memset(&G, 0, sizeof G); G.state = -1; diff --git a/runtime/vk_gpa.h b/runtime/vk_gpa.h index ad5965ae..fcffee6d 100644 --- a/runtime/vk_gpa.h +++ b/runtime/vk_gpa.h @@ -18,7 +18,8 @@ uint32_t vk_gpa_map(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo); void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint64_t page_addr)); // Unmaps every page of [lo, hi), whatever record it names (pages of a virtual pool whose physical memory goes away). void vk_gpa_clear(uint64_t lo, uint64_t hi); -// Moves the pages of [lo, hi) that name old_rec (or none) to buf (a pool's VRAM mirror); returns the new record. +// Moves the pages of [lo, hi) that name old_rec (or none) to buf (a pool's VRAM mirror), with old_rec's buffer as their +// twin, which shaders write too; returns the new record. uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); void vk_gpa_report(void); void vk_gpa_shutdown(void); diff --git a/tools/build_xlate.py b/tools/build_xlate.py index 931748b9..57fc9b27 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 32: - sys.exit(f"expected 32 patches, found {len(patches)}") + if len(patches) != 33: + sys.exit(f"expected 33 patches, found {len(patches)}") for patch in patches: # git apply works outside a repository when given a directory; --check first so a failure names the patch. for extra in (["--check"], []): diff --git a/tools/xlate/patches/0034-device-address-twin-writes.patch b/tools/xlate/patches/0034-device-address-twin-writes.patch new file mode 100644 index 00000000..cacd6d39 --- /dev/null +++ b/tools/xlate/patches/0034-device-address-twin-writes.patch @@ -0,0 +1,132 @@ +--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs ++++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs +@@ -1802,7 +1802,13 @@ + SpvInstruction one = context.Constant(context.TypeU32(), 1); + SpvInstruction zero = context.Constant(context.TypeU32(), 0); + +- return new OperationResult(varType, emitU(context.GetType(varType), elemPointer, one, zero, value)); ++ SpvInstruction result = emitU(context.GetType(varType), elemPointer, one, zero, value); ++ if (operation.StorageKind == StorageKind.GlobalMemory) // NX2NAVI: the twin (GetDeviceAddressTwinPointer) ++ { ++ emitU(context.GetType(varType), GlobalTwinPointer(context, operation), one, zero, value); ++ } ++ ++ return new OperationResult(varType, result); + } + + private static OperationResult GenerateAtomicMemoryCas(CodeGenContext context, AstOperation operation) +@@ -1815,7 +1821,22 @@ + SpvInstruction one = context.Constant(context.TypeU32(), 1); + SpvInstruction zero = context.Constant(context.TypeU32(), 0); + +- return new OperationResult(varType, context.AtomicCompareExchange(context.GetType(varType), elemPointer, one, zero, zero, value1, value0)); ++ SpvInstruction result = context.AtomicCompareExchange(context.GetType(varType), elemPointer, one, zero, zero, value1, value0); ++ if (operation.StorageKind == StorageKind.GlobalMemory) // NX2NAVI: the twin (GetDeviceAddressTwinPointer) ++ { ++ context.AtomicCompareExchange(context.GetType(varType), GlobalTwinPointer(context, operation), one, zero, zero, value1, value0); ++ } ++ ++ return new OperationResult(varType, result); ++ } ++ ++ private static SpvInstruction GlobalTwinPointer(CodeGenContext context, AstOperation operation) ++ { ++ return GetDeviceAddressTwinPointer( ++ context, ++ ((AstOperand)operation.GetSource(0)).Value, ++ context.Get(AggregateType.U32, operation.GetSource(1)), ++ context.Get(AggregateType.U32, operation.GetSource(2))); + } + + private static OperationResult GenerateLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore) +@@ -1856,11 +1877,29 @@ + // second-level block, 0 for none (block 0 stays zero) + // word 65552 second-level blocks of 4096 entries, one per 4 KiB page: the record of the buffer that holds + // the page, 0 for none +- // records two words each, the buffer's device address minus its GPU address (low word first); the host +- // never changes a record a page names, so a page entry read at any time gives a whole one +- // The pointer addresses the 32-bit word at the address's 4-byte-aligned word. ++ // words 3-4 the device address of a 4 KiB scratch buffer, where a twin write goes when the page has no twin ++ // records four words each: the device address of the buffer the page's accesses use minus its GPU address, ++ // then the same for the page's twin, the pool's imported buffer in guest memory when the first is ++ // its VRAM mirror (low words first); the host never changes a record a page names, so a page entry ++ // read at any time gives a whole one ++ // The pointer addresses the 32-bit word at the address's 4-byte-aligned word. Stores and atomics also go to the ++ // twin (GetDeviceAddressTwinPointer): what the GPU writes into a mirror through the table is not recorded as a ++ // GPU write, so without it the CPU never saw it and a sync of a CPU-written page copied the stale guest page ++ // over it (runtime/vk_gpa.c). + private static SpvInstruction GetDeviceAddressPointer(CodeGenContext context, int binding, SpvInstruction low, SpvInstruction high) + { ++ return GetDeviceAddressPointers(context, binding, low, high, out _); ++ } ++ ++ // The twin of an access's pointer: the guest copy of a mirrored page, else the scratch buffer. ++ private static SpvInstruction GetDeviceAddressTwinPointer(CodeGenContext context, int binding, SpvInstruction low, SpvInstruction high) ++ { ++ GetDeviceAddressPointers(context, binding, low, high, out SpvInstruction twin); ++ return twin; ++ } ++ ++ private static SpvInstruction GetDeviceAddressPointers(CodeGenContext context, int binding, SpvInstruction low, SpvInstruction high, out SpvInstruction twinPointer) ++ { + SpvInstruction u32 = context.TypeU32(); + SpvInstruction u64 = context.TypeInt(64, 0); + SpvInstruction tableVar = context.StorageBuffers[binding]; +@@ -1879,16 +1918,26 @@ + SpvInstruction page = context.BitwiseAnd(u32, context.ShiftRightLogical(u32, low, U(12)), U(0xfff)); + SpvInstruction record = Word(context.IAdd(u32, context.IAdd(u32, context.IMul(u32, block, U(4096)), U(65552)), page)); + SpvInstruction mapped = context.INotEqual(context.TypeBool(), record, U(0)); +- SpvInstruction recordWord = context.IAdd(u32, Word(U(2)), context.IMul(u32, record, U(2))); ++ SpvInstruction recordWord = context.IAdd(u32, Word(U(2)), context.IMul(u32, record, U(4))); + SpvInstruction deltaLow = Word(recordWord); + SpvInstruction deltaHigh = Word(context.IAdd(u32, recordWord, U(1))); ++ SpvInstruction twinLow = Word(context.IAdd(u32, recordWord, U(2))); ++ SpvInstruction twinHigh = Word(context.IAdd(u32, recordWord, U(3))); + + SpvInstruction wordLow = context.BitwiseAnd(u32, low, U(0xfffffffc)); + SpvInstruction device = context.IAdd(u64, Wide(wordLow, high), Wide(deltaLow, deltaHigh)); + SpvInstruction dummy = context.IAdd(u64, Wide(Word(U(0)), Word(U(1))), context.UConvert(u64, context.BitwiseAnd(u32, low, U(0xffc)))); + SpvInstruction address = context.Select(u64, mapped, device, dummy); + +- return context.ConvertUToPtr(context.TypePointer(StorageClass.PhysicalStorageBuffer, u32), address); ++ SpvInstruction boolType = context.TypeBool(); ++ SpvInstruction sameTwin = context.LogicalAnd(boolType, context.IEqual(boolType, twinLow, deltaLow), context.IEqual(boolType, twinHigh, deltaHigh)); ++ SpvInstruction hasTwin = context.LogicalAnd(boolType, mapped, context.LogicalNot(boolType, sameTwin)); ++ SpvInstruction twinDevice = context.IAdd(u64, Wide(wordLow, high), Wide(twinLow, twinHigh)); ++ SpvInstruction scratch = context.IAdd(u64, Wide(Word(U(3)), Word(U(4))), context.UConvert(u64, context.BitwiseAnd(u32, low, U(0xffc)))); ++ SpvInstruction ptrType = context.TypePointer(StorageClass.PhysicalStorageBuffer, u32); ++ twinPointer = context.ConvertUToPtr(ptrType, context.Select(u64, hasTwin, twinDevice, scratch)); ++ ++ return context.ConvertUToPtr(ptrType, address); + } + + private static SpvInstruction LoadAligned(CodeGenContext context, SpvInstruction type, SpvInstruction pointer) +@@ -1912,7 +1961,7 @@ + SpvInstruction u32 = context.TypeU32(); + SpvInstruction low = context.Get(AggregateType.U32, operation.GetSource(1)); + SpvInstruction high = context.Get(AggregateType.U32, operation.GetSource(2)); +- SpvInstruction pointer = GetDeviceAddressPointer(context, binding, low, high); ++ SpvInstruction pointer = GetDeviceAddressPointers(context, binding, low, high, out SpvInstruction twin); + StorageKind kind = operation.StorageKind; + int bits = kind is StorageKind.GlobalMemoryS8 or StorageKind.GlobalMemoryU8 ? 8 + : kind is StorageKind.GlobalMemoryS16 or StorageKind.GlobalMemoryU16 ? 16 : 32; +@@ -1921,7 +1970,9 @@ + { + if (isStore) + { +- StoreAligned(context, pointer, context.Get(AggregateType.U32, operation.GetSource(operation.SourcesCount - 1))); ++ SpvInstruction stored = context.Get(AggregateType.U32, operation.GetSource(operation.SourcesCount - 1)); ++ StoreAligned(context, pointer, stored); ++ StoreAligned(context, twin, stored); + return OperationResult.Invalid; + } + +@@ -1939,6 +1990,8 @@ + SpvInstruction semantics = context.Constant(u32, 0); + context.AtomicAnd(u32, pointer, scope, semantics, context.Not(u32, mask)); + context.AtomicOr(u32, pointer, scope, semantics, context.ShiftLeftLogical(u32, value, shift)); ++ context.AtomicAnd(u32, twin, scope, semantics, context.Not(u32, mask)); ++ context.AtomicOr(u32, twin, scope, semantics, context.ShiftLeftLogical(u32, value, shift)); + return OperationResult.Invalid; + } +