From 936dd8094a8e18a73ba3e141d53ff029c8615007 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Sun, 8 Nov 2026 00:33:48 -0500 Subject: [PATCH] Translate 40 more shaders, and build the upscaler's compute shaders five times faster in the driver. The SM86 lowering now takes MEMBAR (as Maxwell MEMBAR.GL), drops ERRBAR, takes F2I of an immediate and MATCH.ALL.U64, and finds a switch's jump table bound past the BSSY the compiler puts between the IMNMX and the shift: three vertex shaders, one of them drawn in Kokiri Forest, were skipped for it. A task program and the mesh programs it feeds whose payload size field reads 0 (it counts 256-byte units) get one unit. Of the 42,177 programs, 31 still fail, all modes whose meaning is not certain. Translator patch 0035 inlines SHFL.IDX over the whole warp (mask 0x1f) on a 32-lane host as the subgroup shuffle, instead of a call to the general shuffle helper. The tensor lowering shuffles thousands of times per program and the driver inlined the helper at each call: a mid-size upscaler shader built in 185 s before and in 34 s now. Co-Authored-By: Claude Opus 5.5 --- tools/build_xlate.py | 4 +- tools/xlate/AmpereLowering.cs | 32 +++++++++++--- tools/xlate/Program.cs | 4 ++ .../0035-whole-warp-shuffle-inline.patch | 43 +++++++++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 tools/xlate/patches/0035-whole-warp-shuffle-inline.patch diff --git a/tools/build_xlate.py b/tools/build_xlate.py index 57fc9b27..cf356d9c 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 33: - sys.exit(f"expected 33 patches, found {len(patches)}") + if len(patches) != 34: + sys.exit(f"expected 34 patches, found {len(patches)}") for patch in patches: # git apply works outside a repository when given a directory; --check first so a failure names the patch. for extra in (["--check"], []): diff --git a/tools/xlate/AmpereLowering.cs b/tools/xlate/AmpereLowering.cs index 4909d50c..491ba5bc 100644 --- a/tools/xlate/AmpereLowering.cs +++ b/tools/xlate/AmpereLowering.cs @@ -390,6 +390,7 @@ static class AmpereLowering case 0x941: // BSYNC case 0x942: // BREAK case 0x948: // WARPSYNC + case 0x9ab: // ERRBAR: waits for earlier memory errors to be reported, which the host GPU does not do here return; case 0x202: case 0x802: case 0xa02: case 0xc02: Mov_(op); return; @@ -410,7 +411,8 @@ static class AmpereLowering case 0x388: case 0x984: case 0x988: SharedAccess(op); return; case 0x83b: Ldsm(); return; case 0xb1d: Bar(); return; - case 0x305: case 0xb05: case 0xd05: F2i(op); return; + case 0x992: Membar(); return; + case 0x305: case 0x905: case 0xb05: case 0xd05: F2i(op); return; case 0xb82: Ldc(); return; case 0x381: case 0x981: Ldg(op); return; case 0x386: case 0x986: Stg(op); return; @@ -1136,7 +1138,9 @@ static class AmpereLowering uint dstSize = _i.Bits(75, 2), srcSize = _i.Bits(84, 2); if ((w2 & ~known) != 0 || dstSize > 2 || srcSize is not 1 and not 2 || (srcSize == 1 && _i.Bits(60, 2) != 0)) throw Fail("F2I format"); - var (b, abs, neg) = Raw(1, SB); + int b; bool abs, neg; + if (op == 0x905) { Mov32i(SB, _i.W1); (b, abs, neg) = (SB, false, false); } // F2I of a 32-bit immediate + else (b, abs, neg) = Raw(1, SB); uint fmt = dstSize | (_i.Bit(72) ? 4u : 0u); // ISrcDstFmt: U8 0, U16 1, U32 2, +4 signed EmitG(0x5cb0000000000000UL | ((ulong)b << 20) | (uint)Gpr((uint)Dst) | ((ulong)(fmt & 3) << 8) | ((ulong)(fmt >> 2) << 12) | ((ulong)srcSize << 10) | ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) @@ -1386,20 +1390,29 @@ static class AmpereLowering // before an aggregated shared atomic: @!Pd per-lane ATOMS, else REDUX and one): Pd = every active lane holds Ra's // value, Rd = the active-lane mask then, else 0. Lowered as VOTE.ANY (the active lanes), FLO (the highest of // them), SHFL.IDX (that lane's value), ISETP.EQ into Pd and VOTE.ALL over Pd. Only the unguarded form with a - // predicate destination is taken. + // predicate destination is taken. Bit 73 is .U64: Ra:Ra+1 is compared, the high word ANDed into Pd. void Match() { uint pd = _i.Bits(81, 3); - if (_i.W1 != 0 || (_i.W2 & ~(7u << 17)) != 0 || _i.Bits(12, 4) != 7 || pd == PT) + bool wide = _i.Bit(73); + if (_i.W1 != 0 || (_i.W2 & ~(7u << 17 | 1u << 9)) != 0 || _i.Bits(12, 4) != 7 || pd == PT) throw Fail("MATCH mode"); int d = Gpr((uint)Dst), src = Gpr(_i.Bits(24, 8)); - const int ballot = 240, lead = 241, value = 242; + if (wide && (src == RZ || (src & 1) != 0)) throw Fail("MATCH.U64 register pair"); + const int ballot = 240, lead = 241, value = 242, valueHi = 243; Emit(Unguarded(0x50d8000000000000UL | (1UL << 48) | (7UL << 45) | ((ulong)PT << 39) | ballot)); // VOTE.ANY Emit(Unguarded(0x5c30000000000000UL | ((ulong)ballot << 20) | lead)); // FLO.U32 Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)lead << 20) | ((ulong)src << 8) | value)); // SHFL.IDX Emit(Unguarded(0x5b60000000000000UL | (2UL << 49) | ((ulong)PT << 39) | ((ulong)value << 20) | ((ulong)src << 8) | ((ulong)pd << 3) | PT)); // ISETP.EQ.U32 + if (wide) + { + Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)lead << 20) + | ((ulong)(src + 1) << 8) | valueHi)); // SHFL.IDX + Emit(Unguarded(0x5b60000000000000UL | (2UL << 49) | ((ulong)pd << 39) | ((ulong)valueHi << 20) + | ((ulong)(src + 1) << 8) | ((ulong)pd << 3) | PT)); // ISETP.EQ.U32.AND + } Emit(Unguarded(0x50d8000000000000UL | ((ulong)pd << 45) | ((ulong)pd << 39) | RZ)); // VOTE.ALL if (d != RZ) Sel(d, ballot, RZ, pd, false); } @@ -1819,6 +1832,11 @@ static class AmpereLowering Emit(Unguarded(0xf0a8000000000000UL | (1UL << 43) | (1UL << 44) | (7UL << 39))); } + // MEMBAR: Maxwell MEMBAR.GL whatever the scope. A barrier at least as wide as the one asked for orders the same + // accesses; the decoder reads MEMBAR's level from bits 8-9 (CTA 0, GL 1, SYS 2), and the SPIR-V it makes is a + // device-scope memory barrier for GL and SYS alike. + void Membar() => Emit(Unguarded(0xef98000000000000UL | (1UL << 8))); + // ISETP.EX: the high-word half of a 64-bit compare. The low words' result is the predicate of bits 68-71; EQ is // (high equal AND low), NE (high different OR low), an ordering (high strictly ordered OR (high equal AND low)), // signed with bit 73. The result is combined with Pc (87-90) by the operation in 74-75 into P (81-83) and Q @@ -2281,7 +2299,9 @@ static class AmpereLowering { var p = _code[k]; uint op = p.Bits(0, 12), b = op & 0x1ff; - if (b is >= 0x140 and <= 0x15f) return -1; // control flow: not straight-line + // control flow: not straight-line. BSSY, BSYNC and WARPSYNC branch nowhere and write no register: the + // compiler puts a BSSY for the switch's reconvergence between the IMNMX bound and the SHL. + if (b is >= 0x140 and <= 0x15f && op is not 0x945 and not 0x941 and not 0x948) return -1; // uniform datapath: ULDC, UMOV and the U* ALU ops (base 0x080-0x0bf; UISETP/UPLOP3 write predicates) bool isUr = op is 0xab9 or 0xabb or 0x882 or 0xc82 || (b is >= 0x080 and < 0x0c0 && b != 0x08c && b != 0x09c); if (p.Bits(16, 8) == reg && isUr == ur) return k; diff --git a/tools/xlate/Program.cs b/tools/xlate/Program.cs index 8bca05a2..e632c478 100644 --- a/tools/xlate/Program.cs +++ b/tools/xlate/Program.cs @@ -181,6 +181,10 @@ static class SwshXlate MaxPrimitives = (int)mj["maxp"], Topology = (int)mj["topo"], PayloadBytes = (int)mj["payload"], HasTask = mj["task"] is JsonValue tv && tv.TryGetValue(out bool tb) && tb, }; + // The size field counts 256-byte units, and some task programs and the mesh programs they feed + // carry 0 there while the payload is used (the task writes the mesh count at +4, the mesh reads + // ISBERD region 7 at 0x10 and 0x14): a payload under 256 bytes rounds down. One unit covers it. + if ((mesh.Stage == 6 || mesh.HasTask) && mesh.PayloadBytes == 0) mesh.PayloadBytes = 256; code = AmpereLowering.Lower(code, size, false, out int meshLocal, null, mesh); if (meshLocal != 0) job["local_mem"] = meshLocal; job["local"] = new JsonArray(mesh.Threads, 1, 1); diff --git a/tools/xlate/patches/0035-whole-warp-shuffle-inline.patch b/tools/xlate/patches/0035-whole-warp-shuffle-inline.patch new file mode 100644 index 00000000..bae80b6d --- /dev/null +++ b/tools/xlate/patches/0035-whole-warp-shuffle-inline.patch @@ -0,0 +1,43 @@ +--- a/src/Ryujinx.Graphics.Shader/Translation/Transforms/ShufflePass.cs ++++ b/src/Ryujinx.Graphics.Shader/Translation/Transforms/ShufflePass.cs +@@ -30,14 +30,38 @@ + return node; + } + +- int functionId = context.Hfm.GetOrCreateShuffleFunctionId(functionName, context.GpuAccessor.QueryHostSubgroupSize()); +- + Operand result = operation.GetDest(0); + Operand valid = operation.GetDest(1); + Operand value = operation.GetSource(0); + Operand index = operation.GetSource(1); + Operand mask = operation.GetSource(2); + ++ // NX2NAVI: SHFL.IDX over the whole warp (mask 0x1f: clamp 31, no segments) on a 32-lane host is the subgroup ++ // shuffle itself, for an index up to 31, else the lane keeps its value. Inline, not a call of the general ++ // helper (about 55 instructions once inlined): the tensor lowering (tools/xlate/AmpereLowering.cs Hmma) ++ // shuffles thousands of times per program, and the driver took minutes over the inlined helpers. ++ if (functionName == HelperFunctionName.Shuffle && mask.Type == OperandType.Constant && mask.Value == 0x1f && ++ context.GpuAccessor.QueryHostSubgroupSize() == 32) ++ { ++ Operand inRange = Local(); ++ Operand shuffled = Local(); ++ node.List.AddBefore(node, new Operation(Instruction.CompareLessOrEqualU32, inRange, index, Const(31))); ++ node.List.AddBefore(node, new Operation(Instruction.Shuffle, shuffled, value, index)); ++ if (valid != null) ++ { ++ node.List.AddBefore(node, new Operation(Instruction.Copy, valid, inRange)); ++ } ++ ++ LinkedListNode selectNode = node.List.AddBefore(node, new Operation(Instruction.ConditionalSelect, result, inRange, shuffled, value)); ++ ++ operation.Dest = null; ++ Utils.DeleteNode(node, operation); ++ ++ return selectNode; ++ } ++ ++ int functionId = context.Hfm.GetOrCreateShuffleFunctionId(functionName, context.GpuAccessor.QueryHostSubgroupSize()); ++ + operation.Dest = null; + + Operand[] callArgs = [Const(functionId), value, index, mask, valid];