diff --git a/tests/maxwell_warp_emu.py b/tests/maxwell_warp_emu.py index 0d9602d0..242c0031 100644 --- a/tests/maxwell_warp_emu.py +++ b/tests/maxwell_warp_emu.py @@ -703,6 +703,16 @@ def op_f2f_r(w, i, m): # InstEmitConversion F2fR / EmitF2F w.W(i.rd, pack_half(x, zero), m, halves=(x, zero)) +def op_mufu(w, i, m): # InstEmitMultifunction Mufu: FPAbsNeg of Ra, the function in F32, FPSaturate + fn = {2: np.exp2, 3: np.log2, 4: lambda v: F32(1) / v, 5: lambda v: F32(1) / np.sqrt(v), 8: np.sqrt}.get(i.bits(20, 4)) + if fn is None: + raise Unsupported(f"MUFU operation {i.bits(20, 4)} {i}") + x = abs_neg(as_f32(w.R(i.ra)), i.bit(46), i.bit(48)) + with np.errstate(divide="ignore", invalid="ignore", over="ignore"): + r = fn(x).astype(F32) + w.W(i.rd, from_f32(saturate(r) if i.bit(50) else r), m) + + def _ffma(w, i, m, a, b, c, neg_b, neg_c, sat): # EmitFfma (rounding mode and FMZ ignored) r = fma_f32(as_f32(a), abs_neg(as_f32(b), False, neg_b), abs_neg(as_f32(c), False, neg_c)) w.W(i.rd, from_f32(saturate(r) if sat else r), m) @@ -926,6 +936,7 @@ TABLE = [ ("1110111101000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LDL", op_ldl), ("1110111101010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "STL", op_stl), ("0101110010101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "F2F", op_f2f_r), + ("0101000010000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MUFU", op_mufu), ("010110011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA", op_ffma_r), ("0011001x1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA.I", op_ffma_i), ("000011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA32I", op_ffma32i), diff --git a/tests/tensor_lowering_check.py b/tests/tensor_lowering_check.py index a429aff8..ec1f21ce 100644 --- a/tests/tensor_lowering_check.py +++ b/tests/tensor_lowering_check.py @@ -38,6 +38,9 @@ SHFL is guarded. Derived cases, beyond the hardware vectors, for forms the game uses: reference models of HFMA2(.RELU) and LDSM, each first checked against every hardware case of its instruction, give the expected values of HFMA2.RELU with an immediate B (opcode 0x831, as in the game's HFMA2.RELU Rd, Ra, 1, 1, Rc) and LDSM .M88.4 / .MT88.4 with an offset. +MUFU.F16 (LG2 and EX2 of either half, with abs and neg, on the HFMA2 case's inputs) is checked against a model of +the lowering (F32 function of the widened half, rounded back to a half): no hardware vectors, as PTX has no lg2.f16; +the instruction forms were confirmed with nvdisasm. """ from __future__ import annotations @@ -90,6 +93,8 @@ def operands(sm86) -> tuple[list[int], list[int]]: return regs(rd, 1), regs(ra, 1) if op in (0x231, 0x831): # HFMA2: B a register (0x231) or an immediate (0x831) return regs(rd, 1), regs(ra, 1) + (regs(rb, 1) if op == 0x231 else []) + regs(rc, 1) + if op == 0x308: # MUFU (register form): the operand is B + return regs(rd, 1), regs(rb, 1) raise ValueError(f"no operand table for SM86 opcode 0x{op:03x}") @@ -381,9 +386,31 @@ def derived_cases(cases: list[dict]) -> list[dict]: regs_out = [{f"R{rd + m}": f"{model[t][m]:08x}" for m in range(4)} for t in range(LANES)] out.append(dict(src, name=f"derived_ldsm_{'mt88' if transpose else 'm88'}_x4_off40", sm86=[f"{x:08x}" for x in sm86], regs_out=regs_out)) + if relu is not None: # MUFU.F16 on the halves of the HFMA2 case's A register, as the upscaler's colour decode + w0 = words_of(relu["sm86"])[0] + rd, rs = (w0 >> 16) & 0xFF, (w0 >> 24) & 0xFF + regs = emu.load_regs(relu["regs_in"]) + for name, operation, high, absolute, negate in (("lg2_abs_h1", 3, True, True, False), ("lg2_abs_h0", 3, False, True, False), + ("ex2_h1", 2, True, False, False), ("ex2_neg_h0", 2, False, False, True)): + sm86 = [(rd << 16) | (7 << 12) | 0x308, rs | (high << 28) | (absolute << 30) | (negate << 31), + (operation << 10) | (1 << 9), 0x000FE200] + regs_out = [{f"R{rd}": f"{mufu_f16_model(regs[t][('R', rs)], operation, high, absolute, negate):08x}"} + for t in range(LANES)] + out.append({"name": f"derived_mufu_f16_{name}", "sm86": [f"{x:08x}" for x in sm86], + "regs_in": relu["regs_in"], "regs_out": regs_out}) return out +def mufu_f16_model(reg: int, operation: int, high: bool, absolute: bool, negate: bool) -> int: + """MUFU.F16 as AmpereLowering.cs Mufu lowers it: the half widened to F32 with abs and neg, the function in F32 + (EX2 2, LG2 3), rounded to a half in the low 16 bits, the high 16 bits clear.""" + x = np.array([(reg >> 16) & 0xFFFF if high else reg & 0xFFFF], dtype=np.uint16).view(np.float16).astype(np.float32) + x = np.negative(np.abs(x) if absolute else x) if negate else (np.abs(x) if absolute else x) + with np.errstate(divide="ignore", invalid="ignore", over="ignore"): + y = (np.exp2 if operation == 2 else np.log2)(x).astype(np.float32) + return int(y.astype(np.float16).view(np.uint16)[0]) + + # ---- driver ------------------------------------------------------------------------------------------------------- def main(argv=None) -> int: diff --git a/tools/build_xlate.py b/tools/build_xlate.py index aaac0b69..88ee3a50 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 35: - sys.exit(f"expected 35 patches, found {len(patches)}") + if len(patches) != 36: + sys.exit(f"expected 36 patches, found {len(patches)}") # build/xlate is inside this checkout, and git inside a repository reads a "diff --git" patch's paths from the # repository's top: those hunks fall outside the directory handed to --directory, git skips every file of the patch # and still exits 0. 0021-prmt-byte-permute.patch is the only one written in that form, so it used to be skipped in diff --git a/tools/xlate/AmpereLowering.cs b/tools/xlate/AmpereLowering.cs index 83d2edb1..bdbfb588 100644 --- a/tools/xlate/AmpereLowering.cs +++ b/tools/xlate/AmpereLowering.cs @@ -196,6 +196,8 @@ static class AmpereLowering uint k16 = i.Bits(75, 1), cd = i.Bit(76) ? 3u : 1u; see(i.Bits(16, 8) + cd); see(i.Bits(24, 8) + 1 + 2 * k16); see(i.Bits(32, 8) + k16); see(i.Bits(64, 8) + cd); } + if (op == 0x3a9) see(i.Bits(64, 8)); // ATOMG.CAS: the new value + if (op == 0x593 || (op == 0xb66 && i.Bit(78))) see(i.Bits(32, 8)); // SUATOM data, TLD.MS sample index if (b < 0x080 || b is >= 0x160 and <= 0x16f) see(i.Bits(64, 8)); // C of ALU forms, TEX second destinations if (i.Bits(9, 3) == 1) see(i.Bits(32, 8)); } @@ -446,6 +448,8 @@ static class AmpereLowering case 0x234: case 0x434: Hsetp2(); return; case 0x803: case 0x804: PredicateRegister(op == 0x803); return; case 0x59f: Sured(); return; + case 0x593: Suatom(); return; + case 0x3a9: AtomgCas(); return; case 0x3c4: Redux(); return; case 0x81a: Sgxt(); return; case 0x806: case 0x886: Vote(op == 0x886); return; @@ -612,12 +616,24 @@ static class AmpereLowering // MUFU: operation in bits 74-79, numbered as Maxwell's (0 COS, 1 SIN, 2 EX2, 3 LG2, 4 RCP, 5 RSQ, 8 SQRT). // SM70+ COS/SIN take the angle in turns (the compiler multiplies by 1/2pi first); the translator's take radians. + // MUFU.F16 (bit 73, register form): the half in the low or, with bit 60, the high 16 bits of Rb, abs 62 and + // neg 63; the result is a half in the low 16 bits. The upscaler's colour decode raises halves to a power this + // way (LG2.F16, HMUL2, EX2.F16, the two lanes packed by PRMT). Lowered as F2F to F32, the F32 function and F2F + // back to F16 (round to nearest), which writes the half with the high 16 bits clear. void Mufu(uint op) { uint operation = _i.Bits(74, 6); - if (operation > 8 || operation is 6 or 7 || _i.Bit(73)) throw Fail($"MUFU operation {operation}"); + bool half = _i.Bit(73); + if (operation > 8 || operation is 6 or 7 || (half && (op != 0x308 || _i.Bit(61)))) + throw Fail($"MUFU operation {operation}{(half ? ".F16" : "")}"); int src; bool abs, neg; if (op == 0x908) { Mov32i(SB, _i.W1); (src, abs, neg) = (SB, false, false); } + else if (half) + { + Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)Gpr(_i.Bits(32, 8)) << 20) + | (_i.Bit(60) ? 1UL << 41 : 0) | (_i.Bit(63) ? 1UL << 45 : 0) | (_i.Bit(62) ? 1UL << 49 : 0) | SB)); // F2F.F32.F16 + (src, abs, neg) = (SB, false, false); + } else (src, abs, neg) = Raw(1, SB); if (operation <= 1) { @@ -625,8 +641,10 @@ static class AmpereLowering Fmul32i(SB, src, 0x40c90fdb); // 2pi src = SB; abs = neg = false; } - EmitG(0x5080000000000000UL | ((ulong)operation << 20) | ((ulong)src << 8) | (uint)Gpr((uint)Dst) - | (abs ? 1UL << 46 : 0) | (neg ? 1UL << 48 : 0)); + ulong mufu = 0x5080000000000000UL | ((ulong)operation << 20) | ((ulong)src << 8) | (abs ? 1UL << 46 : 0) | (neg ? 1UL << 48 : 0); + if (!half) { EmitG(mufu | (uint)Gpr((uint)Dst)); return; } + Emit(Unguarded(mufu | SB)); + EmitG(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)SB << 20) | (uint)Gpr((uint)Dst)); // F2F.F16.F32 } // FRND.F32: rounding (bits 78-79: nearest/floor/ceil/trunc) as Maxwell F2F.F32.F32 with integer rounding. @@ -956,14 +974,23 @@ static class AmpereLowering // ISBERD Rd, [Ra + imm]: a stage buffer read (fields as ISBEWR's, bit 79 clear). Address 0 of region 15 (a mesh // program without a task stage, a task program) or of region 7 (a mesh program after a task stage) is the - // workgroup's index; region 7 elsewhere is the task payload. + // workgroup's index; region 7 elsewhere is the task payload. With bit 79 set (the output side, as ISBEWR's) a + // task program reads back the payload it writes (AglShader's task program stores a word and reloads it). void Isberd() { if (_mesh == null) throw Fail("ISBERD outside a task or mesh program"); uint region = _i.Bits(75, 4), offset = _i.Bits(40, 24); - if (_i.Bit(79) || (_i.W2 & ~0x7800u) != 0 || _i.Bits(32, 8) != 0 || _i.Bits(64, 8) != 0 || offset >= 0x10000) throw Fail("ISBERD mode"); + bool output = _i.Bit(79); + if ((_i.W2 & ~0xf800u) != 0 || (output && (region != 7 || _mesh.Stage != 6)) || _i.Bits(32, 8) != 0 || _i.Bits(64, 8) != 0 + || offset >= 0x10000) + throw Fail("ISBERD mode"); int a = Gpr(_i.Bits(24, 8)), d = Gpr((uint)Dst); if (d == RZ) return; + if (output) + { + EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | ((ulong)(uint)(_mesh.Pay + (int)offset) << 20) | (4UL << 48)); + return; + } if ((region == 15 || region == 7) && a == RZ && offset == 0) { Put(0xf0c8000000000000UL | (0x25UL << 20) | (uint)d, true); // S2R SR_CTAID.X @@ -1176,8 +1203,10 @@ static class AmpereLowering uint ur = store ? _i.Bits(64, 6) : _i.Bits(32, 6); bool descriptor = store ? (_i.W2 & ~(0x3fu | 0xe00u)) == 0x0c101100 : (_i.W2 & ~(0x3fu | 0xe00u)) == 0x0c1e1100 || _i.W2 == 0x0c1ebd00; + // Bit 78 is .STRONG.GPU instead of .STRONG.SM (two agl_common compute programs load their arguments and store + // their results so): the translator's accesses have no scope, which gives the same values outside a race. uint plain = store ? 0x0810a100u : 0x081e8100u; - if (!descriptor && (_i.W2 & ~(0x04000000u | 0x3fu | 0xe00u | (store ? 0u : 0x2000u))) != plain) throw Fail("global access uniform mode"); + if (!descriptor && (_i.W2 & ~(0x04000000u | 0x3fu | 0xe00u | 0x4000u | (store ? 0u : 0x2000u))) != plain) throw Fail("global access uniform mode"); if (!store && _i.Bits(38, 2) != 0) throw Fail("global access uniform mode"); if (descriptor) { @@ -1222,18 +1251,21 @@ static class AmpereLowering } // TLD (texel fetch, integer coordinates): fields as TEX; LOD mode 87-89 is 1 (.LZ) or 3 (.LL, the level after - // the coordinates), offsets bit 76. Maxwell TLD takes its operands as Maxwell TEX does. + // the coordinates), offsets bit 76. Maxwell TLD takes its operands as Maxwell TEX does. Bit 78 (.MS) reads one + // sample of a multisampled 2D texture, its index in the operand after the coordinates (agl_common's 2x resolve: + // TLD.LZ.MS with samples 0 and 1, averaged); it is lowered for .LZ without offsets only, Maxwell's MS bit 50. void Tld() { uint dim = _i.Bits(61, 3), lod = _i.Bits(87, 3), mask = _i.Bits(72, 4); - bool offsets = _i.Bit(76); - if (dim == 6 || _i.Bits(77, 3) != 0 || _i.Bits(81, 3) != 7 || _i.Bit(59) || lod is not 1 and not 3 || mask == 0) + bool offsets = _i.Bit(76), ms = _i.Bit(78); + if (dim == 6 || _i.Bit(77) || _i.Bit(79) || _i.Bits(81, 3) != 7 || _i.Bit(59) || lod is not 1 and not 3 || mask == 0 + || (ms && (dim != 1 || lod != 1 || offsets))) throw Fail("TLD mode"); uint handle = Handle(false), bank = _i.Bits(54, 5); if (handle > 0x1fff) throw Fail("TLD descriptor offset"); if (!TexBank(bank)) throw Fail($"TLD descriptor bank {bank}"); int coords = TexCoords[dim]; - int count = coords + (lod == 3 ? 1 : 0) + (offsets ? 1 : 0); + int count = coords + (lod == 3 ? 1 : 0) + (offsets ? 1 : 0) + (ms ? 1 : 0); int inA = count <= 4 ? (count + 1) / 2 : 4; int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))); @@ -1241,9 +1273,10 @@ static class AmpereLowering int next = coords, meta = TexMeta; if (lod == 3) Mov(meta++, Value(next++), false); if (offsets) Mov(meta++, Value(next++), false); + if (ms) Mov(meta++, Value(next++), false); EmitG(0xdc00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20) | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | (offsets ? 1UL << 35 : 0) | ((ulong)handle << 36) - | ((ulong)_i.Bits(90, 1) << 49) | (7UL << 51) | (lod == 3 ? 1UL << 55 : 0)); + | ((ulong)_i.Bits(90, 1) << 49) | (ms ? 1UL << 50 : 0) | (7UL << 51) | (lod == 3 ? 1UL << 55 : 0)); TexResults(mask, false); } @@ -1349,6 +1382,33 @@ static class AmpereLowering | ((ulong)dim << 33) | ((ulong)handle << 36)); } + // SUATOM.D.op U32 (bound descriptor): Rd = the image word's old value, fields as SURED's plus the destination and + // the result predicate 81-83 (PT only). agl_technique_pfx's append counters add to word 0 of a 1D image this + // way. Maxwell's SUATOM.D is 11101010 in the top bits with bit 54 clear (InstTable Suatom): operation 29-32, + // dimension 33-35, descriptor 36-48, size 51-53 (U32 0). + void Suatom() + { + uint dim = _i.Bits(61, 3), handle = Handle(true), bank = _i.Bits(54, 5); + if ((_i.W2 & ~(0xfu << 23)) != 0x001ee000 || _i.Bits(87, 4) > 8 || dim > 5 || _i.Bits(59, 2) != 0 || handle > 0x1fff + || !TexBank(bank)) + throw Fail("SUATOM mode"); + EmitG(0xea00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20) + | ((ulong)_i.Bits(87, 4) << 29) | ((ulong)dim << 33) | ((ulong)handle << 36)); + } + + // ATOMG.E.CAS U32: Rd = [Ra.64 + offset 40-63], replaced by Rc (64-71) where it equals Rb (32-39). The foliage + // compute programs claim hash slots with it. Maxwell ATOM.E.CAS takes the compare value in Rb and the new value + // in Rb + 1 (xlate/patches/0037-global-compare-and-swap.patch, which also makes the translator emit it), so both + // go to SB and SB + 1 first. Only a zero offset is lowered. + void AtomgCas() + { + if ((_i.W2 & ~0xffu) != 0x001ee100 || _i.Bits(40, 24) != 0 || (_i.Bits(24, 8) & 1) != 0 || _i.Bits(24, 8) == RZ) + throw Fail("ATOMG.CAS mode"); + Mov(SB, Gpr(_i.Bits(32, 8)), false); + Mov(SB + 1, Gpr(_i.Bits(64, 8)), false); + EmitG(0xeef0000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)SB << 20) | (1UL << 48)); + } + // REDUX URd, Ra: Ra reduced over the warp's active lanes into a uniform register. Mode (bits 78-80): SUM 3 // (modulo 2^32), MIN 4, MAX 5 (unsigned, signed with bit 73), OR 1, XOR 2. Lowered as VOTE.ANY (the active-lane // mask) and one SHFL.IDX per lane; an inactive lane contributes the operation's identity. @@ -1420,13 +1480,19 @@ static class AmpereLowering // ATOMS.POPC.INC RZ, [Ra + URb + offset] (op 0xf8c, mode word 0x0d000000; the increment of Ocarina's aggregated // shared counters after MATCH and REDUX): memory goes up by the number of active lanes. With no result register // that is one ATOMS.ADD of 1 per active lane, which is what this emits; a form that wants the per-lane result (its - // rank among the active lanes) is not taken. + // rank among the active lanes) is not taken. Bits 78-79 scale Ra as LDS / STS's stride does (SharedAccess): the + // luminance histograms of agl_common use 3, three counters in each 16-byte bin at offsets 0, 4 and 8. void AtomsPopcInc() { - uint offset = _i.Bits(40, 24), ur = _i.Bits(64, 6); - if ((_i.W2 & ~0x3fu) != 0x0d000000u || _i.Bits(32, 8) != 0 || Dst != RZ || (offset & 3) != 0 || offset >= (1u << 22)) + uint offset = _i.Bits(40, 24), ur = _i.Bits(64, 6), stride = _i.Bits(78, 2); + if ((_i.W2 & ~(0x3fu | 3u << 14)) != 0x0d000000u || _i.Bits(32, 8) != 0 || Dst != RZ || (offset & 3) != 0 || offset >= (1u << 22)) throw Fail("ATOMS.POPC.INC mode"); int addr = Gpr(_i.Bits(24, 8)); + if (stride != 0 && addr != RZ) + { + Emit(Unguarded(0x3848000000000000UL | ((ulong)(stride + 1) << 20) | ((ulong)addr << 8) | SA)); // SHL imm + addr = SA; + } if (ur != 0x3f) { Iadd3(SA, addr, Ur(ur), RZ, false); diff --git a/tools/xlate/patches/0037-global-compare-and-swap.patch b/tools/xlate/patches/0037-global-compare-and-swap.patch new file mode 100644 index 00000000..75e27917 --- /dev/null +++ b/tools/xlate/patches/0037-global-compare-and-swap.patch @@ -0,0 +1,43 @@ +--- a/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs ++++ b/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs +@@ -5,13 +5,6 @@ + { + static partial class InstEmit + { +- public static void AtomCas(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction AtomCas is not implemented."); +- } +- + public static void AtomsCas(EmitterContext context) + { + context.GetOp(); +--- a/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs ++++ b/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs +@@ -24,6 +24,24 @@ + context.Copy(GetDest(op.Dest), res); + } + ++ // NX2NAVI: ATOM.E.CAS Rd, [Ra], Rb, Rb + 1 (32-bit): the word at the address is replaced by Rb + 1 where it ++ // equals Rb, and Rd gets its old value. tools/xlate/AmpereLowering.cs AtomgCas lowers SM86 ATOMG.CAS to it. ++ // BcRz 1 and 2 read the compare and the new value as zero. ++ public static void AtomCas(EmitterContext context) ++ { ++ InstAtomCas op = context.GetOp(); ++ ++ (Operand addrLow, Operand addrHigh) = Get40BitsAddress(context, new Register(op.SrcA, RegisterType.Gpr), op.E, 0); ++ ++ bool rz = op.SrcB == RegisterConsts.RegisterZeroIndex; ++ Operand compare = rz || op.BcRz == 1 ? Const(0) : GetSrcReg(context, op.SrcB); ++ Operand value = rz || op.BcRz == 2 ? Const(0) : GetSrcReg(context, op.SrcB + 1); ++ ++ Operand res = context.AtomicCompareAndSwap(StorageKind.GlobalMemory, addrLow, addrHigh, compare, value); ++ ++ context.Copy(GetDest(op.Dest), res); ++ } ++ + public static void Atoms(EmitterContext context) + { + if (context.TranslatorContext.Definitions.Stage != ShaderStage.Compute)