--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs
@@ -1820,6 +1820,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
private static OperationResult GenerateLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore)
{
+ if (IsGlobalStorageKind(operation.StorageKind))
+ {
+ return GenerateDeviceAddressLoadOrStore(context, operation, isStore);
+ }
+
SpvInstruction pointer = GetStoragePointer(context, operation, out AggregateType varType);
if (isStore)
@@ -1834,10 +1839,137 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
}
}
+ private static bool IsGlobalStorageKind(StorageKind storageKind)
+ {
+ return storageKind is StorageKind.GlobalMemory or
+ StorageKind.GlobalMemoryS8 or
+ StorageKind.GlobalMemoryS16 or
+ StorageKind.GlobalMemoryU8 or
+ StorageKind.GlobalMemoryU16;
+ }
+
+ // NX2NAVI: the device address table (Translation/Optimizations/GlobalToStorage.cs TryKeepDeviceAddressAccess), a
+ // storage buffer of 32-bit words the host keeps (runtime/vk_gpa.c):
+ // words 0-1 the device address of a zeroed 4 KiB buffer, for addresses no page maps
+ // word 2 the word index of the records
+ // words 16.. 65536 first-level entries, one per 16 MiB of the 40-bit GPU address space: the index of the
+ // second-level block, 0 for none (block 0 stays zero)
+ // word 65552 second-level blocks of 4096 entries, one per 4 KiB page: the record of the buffer that holds
+ // the page, 0 for none
+ // records two words each, the buffer's device address minus its GPU address (low word first); the host
+ // never changes a record a page names, so a page entry read at any time gives a whole one
+ // The pointer addresses the 32-bit word at the address's 4-byte-aligned word.
+ private static SpvInstruction GetDeviceAddressPointer(CodeGenContext context, int binding, SpvInstruction low, SpvInstruction high)
+ {
+ SpvInstruction u32 = context.TypeU32();
+ SpvInstruction u64 = context.TypeInt(64, 0);
+ SpvInstruction tableVar = context.StorageBuffers[binding];
+ SpvInstruction elemPtr = context.TypePointer(StorageClass.Uniform, u32);
+ SpvInstruction zero = context.Constant(context.TypeS32(), 0);
+
+ SpvInstruction Word(SpvInstruction index) => context.Load(u32, context.AccessChain(elemPtr, tableVar, zero, index));
+ SpvInstruction U(uint value) => context.Constant(u32, value);
+ SpvInstruction Wide(SpvInstruction lo, SpvInstruction hi) =>
+ context.BitwiseOr(u64, context.UConvert(u64, lo), context.ShiftLeftLogical(u64, context.UConvert(u64, hi), U(32)));
+
+ SpvInstruction inRange = context.ULessThan(context.TypeBool(), high, U(256));
+ SpvInstruction l1 = context.BitwiseOr(u32, context.ShiftLeftLogical(u32, high, U(8)), context.ShiftRightLogical(u32, low, U(24)));
+ l1 = context.Select(u32, inRange, l1, U(0));
+ SpvInstruction block = Word(context.IAdd(u32, l1, U(16)));
+ SpvInstruction page = context.BitwiseAnd(u32, context.ShiftRightLogical(u32, low, U(12)), U(0xfff));
+ SpvInstruction record = Word(context.IAdd(u32, context.IAdd(u32, context.IMul(u32, block, U(4096)), U(65552)), page));
+ SpvInstruction mapped = context.INotEqual(context.TypeBool(), record, U(0));
+ SpvInstruction recordWord = context.IAdd(u32, Word(U(2)), context.IMul(u32, record, U(2)));
+ SpvInstruction deltaLow = Word(recordWord);
+ SpvInstruction deltaHigh = Word(context.IAdd(u32, recordWord, U(1)));
+
+ SpvInstruction wordLow = context.BitwiseAnd(u32, low, U(0xfffffffc));
+ SpvInstruction device = context.IAdd(u64, Wide(wordLow, high), Wide(deltaLow, deltaHigh));
+ SpvInstruction dummy = context.IAdd(u64, Wide(Word(U(0)), Word(U(1))), context.UConvert(u64, context.BitwiseAnd(u32, low, U(0xffc))));
+ SpvInstruction address = context.Select(u64, mapped, device, dummy);
+
+ return context.ConvertUToPtr(context.TypePointer(StorageClass.PhysicalStorageBuffer, u32), address);
+ }
+
+ private static SpvInstruction LoadAligned(CodeGenContext context, SpvInstruction type, SpvInstruction pointer)
+ {
+ SpvInstruction load = context.Load(type, pointer, MemoryAccessMask.Aligned);
+ load.AddOperand((Spv.Generator.LiteralInteger)4);
+ return load;
+ }
+
+ private static void StoreAligned(CodeGenContext context, SpvInstruction pointer, SpvInstruction value)
+ {
+ SpvInstruction store = context.Store(pointer, value, MemoryAccessMask.Aligned);
+ store.AddOperand((Spv.Generator.LiteralInteger)4);
+ }
+
+ // Sources: the table binding, the address's low and high words, the value for a store. 8- and 16-bit accesses
+ // read the word and extract the bytes; their stores clear and set them with two atomics.
+ private static OperationResult GenerateDeviceAddressLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore)
+ {
+ int binding = ((AstOperand)operation.GetSource(0)).Value;
+ SpvInstruction u32 = context.TypeU32();
+ SpvInstruction low = context.Get(AggregateType.U32, operation.GetSource(1));
+ SpvInstruction high = context.Get(AggregateType.U32, operation.GetSource(2));
+ SpvInstruction pointer = GetDeviceAddressPointer(context, binding, low, high);
+ StorageKind kind = operation.StorageKind;
+ int bits = kind is StorageKind.GlobalMemoryS8 or StorageKind.GlobalMemoryU8 ? 8
+ : kind is StorageKind.GlobalMemoryS16 or StorageKind.GlobalMemoryU16 ? 16 : 32;
+
+ if (bits == 32)
+ {
+ if (isStore)
+ {
+ StoreAligned(context, pointer, context.Get(AggregateType.U32, operation.GetSource(operation.SourcesCount - 1)));
+ return OperationResult.Invalid;
+ }
+
+ return new OperationResult(AggregateType.U32, LoadAligned(context, u32, pointer));
+ }
+
+ SpvInstruction shift = context.ShiftLeftLogical(u32, context.BitwiseAnd(u32, low, context.Constant(u32, bits == 8 ? 3u : 2u)), context.Constant(u32, 3));
+ SpvInstruction fieldMask = context.Constant(u32, bits == 8 ? 0xffu : 0xffffu);
+
+ if (isStore)
+ {
+ SpvInstruction value = context.BitwiseAnd(u32, context.Get(AggregateType.U32, operation.GetSource(operation.SourcesCount - 1)), fieldMask);
+ SpvInstruction mask = context.ShiftLeftLogical(u32, fieldMask, shift);
+ SpvInstruction scope = context.Constant(u32, 1); // device
+ SpvInstruction semantics = context.Constant(u32, 0);
+ context.AtomicAnd(u32, pointer, scope, semantics, context.Not(u32, mask));
+ context.AtomicOr(u32, pointer, scope, semantics, context.ShiftLeftLogical(u32, value, shift));
+ return OperationResult.Invalid;
+ }
+
+ SpvInstruction word = LoadAligned(context, u32, pointer);
+
+ if (kind is StorageKind.GlobalMemoryS8 or StorageKind.GlobalMemoryS16)
+ {
+ SpvInstruction s32 = context.TypeS32();
+ SpvInstruction field = context.BitFieldSExtract(s32, context.Bitcast(s32, word), shift, context.Constant(u32, bits));
+ return new OperationResult(AggregateType.S32, field);
+ }
+
+ return new OperationResult(AggregateType.U32, context.BitwiseAnd(u32, context.ShiftRightLogical(u32, word, shift), fieldMask));
+ }
+
private static SpvInstruction GetStoragePointer(CodeGenContext context, AstOperation operation, out AggregateType varType)
{
StorageKind storageKind = operation.StorageKind;
+ // NX2NAVI: atomics on the device address table's pointers (GenerateDeviceAddressLoadOrStore covers loads and
+ // stores): sources are the table binding and the address's two words.
+ if (storageKind == StorageKind.GlobalMemory)
+ {
+ varType = AggregateType.U32;
+ return GetDeviceAddressPointer(
+ context,
+ ((AstOperand)operation.GetSource(0)).Value,
+ context.Get(AggregateType.U32, operation.GetSource(1)),
+ context.Get(AggregateType.U32, operation.GetSource(2)));
+ }
+
StorageClass storageClass;
SpvInstruction baseObj;
int srcIndex = 0;
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
@@ -45,7 +45,18 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
context.AddCapability(Capability.Shader);
- context.SetMemoryModel(AddressingModel.Logical, MemoryModel.GLSL450);
+ if (context.Properties.UsesDeviceAddresses)
+ {
+ // NX2NAVI: global memory through the device address table (Instructions.cs GetDeviceAddressPointer)
+ context.AddCapability(Capability.PhysicalStorageBufferAddresses);
+ context.AddCapability(Capability.Int64);
+ context.AddExtension("SPV_KHR_physical_storage_buffer");
+ context.SetMemoryModel(AddressingModel.PhysicalStorageBuffer64, MemoryModel.GLSL450);
+ }
+ else
+ {
+ context.SetMemoryModel(AddressingModel.Logical, MemoryModel.GLSL450);
+ }
context.AddCapability(Capability.GroupNonUniformBallot);
context.AddCapability(Capability.GroupNonUniformShuffle);
--- a/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs
+++ b/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs
@@ -347,6 +347,15 @@ namespace Ryujinx.Graphics.Shader
{
return true;
}
+
+ ///
+ /// NX2NAVI: queries whether the host binds a device address table (GPU address pages to buffer device addresses)
+ /// for global memory accesses whose storage buffer the translator cannot find.
+ ///
+ bool QueryHostSupportsBufferDeviceAddress()
+ {
+ return false;
+ }
bool QueryHostSupportsShaderNonUniformIndexing()
{
return false;
--- a/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs
+++ b/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs
@@ -19,6 +19,10 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
public IReadOnlyDictionary LocalMemories => _localMemories;
public IReadOnlyDictionary SharedMemories => _sharedMemories;
+ // NX2NAVI: global memory operations go through the device address table (ResourceManager
+ // TryGetDeviceAddressTableBinding); the SPIR-V module uses the PhysicalStorageBuffer64 addressing model.
+ public bool UsesDeviceAddresses { get; set; }
+
public ShaderProperties()
{
_constantBuffers = new Dictionary();
--- a/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs
+++ b/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs
@@ -273,6 +273,11 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
// The returned value being null means that the global memory replacement failed,
// so we just make loads read 0 and stores do nothing.
+ if (TryKeepDeviceAddressAccess(resourceManager, gpuAccessor, node, operation))
+ {
+ continue;
+ }
+
gpuAccessor.Log($"Failed to reserve storage buffer for global memory operation \"{operation.Inst}\".");
if (operation.Dest != null)
@@ -323,6 +328,51 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
}
}
+ // NX2NAVI: a global access no storage buffer serves (its address is a pointer the program read from memory at an
+ // index, a phi of such pointers and constant buffer ones, or past the slots) stays a global access, with the device
+ // address table's binding as a new first source: the SPIR-V back end looks the address's page up in the table
+ // and accesses the buffer device address it gives (CodeGen/Spirv/Instructions.cs GetDeviceAddressPointer). An
+ // access the table cannot serve (a float atomic, which needs the loop of TryGenerateStorageOp) keeps the old
+ // fallback. The operation is replaced in place, so the pass goes on from the same node.
+ private static bool TryKeepDeviceAddressAccess(
+ ResourceManager resourceManager,
+ IGpuAccessor gpuAccessor,
+ LinkedListNode node,
+ Operation operation)
+ {
+ if (!gpuAccessor.QueryHostSupportsBufferDeviceAddress() ||
+ (operation.Inst & Instruction.FP32) != 0 ||
+ (operation.Inst & Instruction.Mask) == Instruction.AtomicCompareAndSwap && operation.StorageKind != StorageKind.GlobalMemory ||
+ operation.SourcesCount < 2 ||
+ (operation.StorageKind != StorageKind.GlobalMemory && operation.Inst.IsAtomic()) ||
+ !resourceManager.TryGetDeviceAddressTableBinding(out int binding))
+ {
+ return false;
+ }
+
+ Operand[] sources = new Operand[operation.SourcesCount + 1];
+ sources[0] = Const(binding);
+
+ for (int index = 0; index < operation.SourcesCount; index++)
+ {
+ sources[index + 1] = operation.GetSource(index);
+ }
+
+ Operand dest = operation.Dest;
+
+ for (int index = 0; index < operation.SourcesCount; index++)
+ {
+ operation.SetSource(index, null);
+ }
+
+ operation.Dest = null;
+ node.Value = new Operation(operation.Inst, operation.StorageKind, dest, sources);
+
+ gpuAccessor.Log($"Global memory operation \"{operation.Inst}\" goes through the device address table.");
+
+ return true;
+ }
+
private static bool IsGlobalMemory(StorageKind storageKind)
{
return storageKind is StorageKind.GlobalMemory or
@@ -660,6 +710,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
Queue phis = new();
HashSet visited = [];
List targetCbs = [];
+ bool unresolved = false; // NX2NAVI: a phi source with no constant buffer base (a pointer from memory)
Operand globalAddress = operation.GetSource(0);
@@ -714,9 +765,22 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
{
phis.Enqueue(phi2);
}
+ else if (phiSource.Type != OperandType.Undefined && phiSource.AsgOp is not PhiNode)
+ {
+ unresolved = true;
+ }
}
}
+ // NX2NAVI: with the device address table, an address that may come from a pointer the program read from memory
+ // goes through the table (the caller keeps it global): a range check against the constant buffer candidates
+ // would send it to one of them, or read 0.
+ if (gpuAccessor.QueryHostSupportsBufferDeviceAddress() && (targetCbs.Count == 0 || unresolved))
+ {
+ functionId = 0;
+ return false;
+ }
+
if (targetCbs.Count == 0)
{
AddTableCandidates(resourceManager, operation.GetSource(0), targetCbs);
--- a/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs
+++ b/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs
@@ -176,6 +176,22 @@ namespace Ryujinx.Graphics.Shader.Translation
return setAndBinding.Binding;
}
+ // NX2NAVI: the storage buffer slot of the device address table, with a constant buffer slot no program has
+ // (NVN banks are 0-17, GlobalToStorage.PointerSlotFlag is 0x20); tools/xlate/Program.cs writes it as "table".
+ public const int DeviceAddressTableCbSlot = 0x40;
+
+ public bool TryGetDeviceAddressTableBinding(out int binding)
+ {
+ if (!TryGetStorageBufferBinding(DeviceAddressTableCbSlot, 0, false, out binding))
+ {
+ return false;
+ }
+
+ Properties.UsesDeviceAddresses = true;
+
+ return true;
+ }
+
public bool TryGetStorageBufferBinding(int sbCbSlot, int sbCbOffset, bool write, out int binding)
{
if (!TryGetSbSlot((byte)sbCbSlot, (ushort)sbCbOffset, out int slot))