// Switch 2 (NVN2) shader programs are SM86 (Ampere) code; the shader translator decodes Maxwell (SM5x) only. This file // rewrites each SM86 instruction as one or more Maxwell instructions with the same meaning, so its Maxwell decoder, // IR and SPIR-V backend run unchanged. Instructions this file does not know make the whole program fail with the opcode // and the reason, so coverage grows by census (python nvn2_shaders.py over a NX2NAVI_SHADER_DUMP directory). // // Encoding facts: the SM70+ instruction field layouts; the Maxwell side is the translator's Decoders/InstTable.cs and // InstDecoders.cs. Where SM86 and the translator's Maxwell model differ: MUFU.SIN/COS take turns, BSSY/BSYNC are // reconvergence hints, bank 24 is never bound. // // Layout: graphics programs keep the first 0x50 bytes of the 0x80-byte Turing+ header (the same fields as Maxwell's // header), then the instructions; Maxwell puts a scheduling word first in every 32 bytes counted from the first // instruction (the translator's Decoder skips them), written as 0. Compute programs start at address 0. // // Registers: guest registers keep their numbers. Uniform registers live just above the highest register the program // uses (UrBase, even so pairs stay aligned; URZ is RZ) and must end below R239. R239-R254 are this file's scratch // registers; R237 and R238 hold the uniform predicate state (see "uniform predicates" below). A program that leaves // no room for them (task and mesh programs use up to R252) is lowered again in spill mode: guest registers from // SpillFrom up live in local memory after the program's own, and an instruction that names one works on a block of // four temporaries (R188-R203, one block per aligned group of four guest registers, so pairs and quads stay // contiguous) loaded before it and stored after it; uniform registers then take blocks of four from R204-R235 (eight // groups: an Ocarina mesh program used more than the four R220-R235 held). A program with more than eight groups of // uniform registers is lowered once more in the wide spill layout: guest registers from R156 spilled, temporaries // R156-R171, uniform registers R172-R235 (sixteen groups, UR0-UR62). Programs that fit keep the first layout. using System; using System.Collections.Generic; static class AmpereLowering { const int RZ = 255, PT = 7, ScratchBase = 239; // Uniform predicates: UPk is bit k of RUP. An instruction that reads or writes uniform predicates runs with UPk // swapped into Pk: the guest's Pk saved to USAVE (P2R), UPk loaded (R2P), the instruction lowered as if its // predicate fields named P registers, written UPs merged back into RUP (P2R), the guest's P restored (R2P). const int RUP = 238, USAVE = 237, UrLimit = 237; // Scratch: operand A/B/C materialisation, a spare, texture coordinates, texture meta values, texture results. const int SA = 252, SB = 253, SC = 254, SX = 239, TexCoord = 240, TexMeta = 244, TexOut = 248; public class LoweringException : Exception { public LoweringException(string message) : base(message) { } } // The registers do not fit the plain layout: Lower retries in spill mode. sealed class RegisterPressureException : LoweringException { public RegisterPressureException(string message) : base(message) { } } // The uniform registers do not fit the spill layout's pool: Lower retries in the wide spill layout. sealed class UrPoolException : LoweringException { public UrPoolException(string message) : base(message) { } } // Spill layouts: guest registers from SpillFrom up in local memory, the temporaries from there, the uniform pool // after them up to UrPoolEnd. readonly record struct SpillLayout(int SpillFrom, int UrPool) { public int TempBase => SpillFrom; public int SpillBytes => (256 - SpillFrom) * 4; } const int UrPoolEnd = 236; static readonly SpillLayout Spill = new(188, 204), SpillWide = new(156, 172); // Task (NVN2 stage 6) and mesh (stage 7) programs. Both are lowered as compute programs: the stage buffer they // write with ISBEWR and read with ISBERD becomes shared memory, laid out by this record, and MeshSpirv.cs turns // the compute SPIR-V into a TaskEXT or MeshEXT entry point that hands the buffer to Vulkan. // Mesh: vertex outputs at Attr: 32-vertex groups, a 32-word line per output component, word // ((v >> 5) * Lines + line) * 32 + (v & 31); the primitive count at Map + 0 (ISBEWR region 1) and the // primitive's vertex indices as bytes from Map + 4 (region 0); the task payload at Pay (ISBERD region 7). // Task: the payload at Pay: the mesh workgroup count at Pay + 4, the data from Pay + 0x10 (region 7). // ISBERD of address 0 in region 15 (no task) or 7 reads the workgroup's index. Control section fields // (runtime/gfx_shader.c build_request): +0x58 low 16 bits the workgroup size, bits 24-31 the payload size in 256 // bytes; +0x5c byte 3 the primitive type (1 points, 2 lines, 3 triangles), byte 2 set when a task stage feeds the // mesh stage; +0x60 low 16 bits the vertex limit, high 16 bits the primitive limit. public sealed class MeshInfo { public int Stage, Threads, MaxVertices, MaxPrimitives, Topology, PayloadBytes; public bool HasTask; // Output lines in order: Maxwell attribute address of each component (0x60.. system values, 0x80 + 16g + 4c // generic g component c). public readonly List Lines = new(); public int Attr, Map, Pay, SharedBytes; public int Groups => (MaxVertices + 31) / 32; public int IndicesPerPrimitive => Topology switch { 1 => 1, 2 => 2, _ => 3 }; // Lines from the output map of the program header (Turing+ VTG header, the same bytes as Maxwell's: system // values B at byte 0x35, attributes 0x60-0x7c one bit each; generic vectors from byte 0x36, a nibble each, // component x in bit 0). System values A (0x34) and C (0x48) are not expected in mesh programs. public void Layout(byte[] sph) { if (Stage == 7) { if (sph[0x34] != 0 || sph[0x48] != 0 || sph[0x49] != 0) throw new LoweringException("mesh output map: system values A or C"); for (int b = 0; b < 8; b++) if ((sph[0x35] >> b & 1) != 0) Lines.Add(0x60 + 4 * b); for (int g = 0; g < 32; g++) for (int c = 0; c < 4; c++) if ((sph[0x36 + g / 2] >> (4 * (g & 1) + c) & 1) != 0) Lines.Add(0x80 + 16 * g + 4 * c); Attr = 0; Map = (Attr + Groups * Lines.Count * 128 + 15) & ~15; Pay = (Map + 4 + IndicesPerPrimitive * MaxPrimitives + 15) & ~15; SharedBytes = Pay + (HasTask ? PayloadBytes : 0); } else { Pay = 0; SharedBytes = PayloadBytes; } if (SharedBytes == 0) SharedBytes = 16; } } struct Insn { public ulong Lo, Hi; public uint W0 => (uint)Lo; public uint W1 => (uint)(Lo >> 32); public uint W2 => (uint)Hi; public uint Bits(int offset, int count) { ulong v = offset >= 64 ? Hi >> (offset - 64) : offset + count <= 64 ? Lo >> offset : (Lo >> offset) | (Hi << (64 - offset)); return (uint)(v & ((1UL << count) - 1)); } public bool Bit(int offset) => Bits(offset, 1) != 0; } sealed class Lowerer { readonly Insn[] _code; readonly List _out = new(); readonly List<(int output, int targetSource)> _branches = new(); readonly int[] _firstOutput; Insn _i; int _index; ulong _guard; // the SM86 instruction's predicate, in Maxwell position (bits 16-19) readonly byte[] _header; // the program header, null for compute readonly int UrBase; // R(UrBase + n) holds URn; guest registers are below it readonly bool _compute; readonly int[] _table; // NVN2 compute binding table: start byte, texture entries, image entries (or null) readonly MeshInfo _mesh; // task or mesh program (lowered as compute), else null public byte[] C1; // the program's c1 constants (NVN2: after the code; BRX jump tables), or null readonly int _spillAt = -1; // spill mode: byte offset of guest register SpillFrom in local memory; -1 otherwise readonly SpillLayout _sl; // spill mode: the layout int SpillFrom => _sl.SpillFrom; readonly Dictionary _temps = new(); // spill mode: guest register group (r & ~3) -> its temporaries, this instruction readonly Dictionary _urBlocks = new(); // spill mode: uniform register group (n & ~3) -> its registers public Lowerer(Insn[] code, byte[] header, bool compute, int[] table, MeshInfo mesh, int spillAt = -1, SpillLayout layout = default) { _code = code; _header = header; _compute = compute; _table = table; _mesh = mesh; _firstOutput = new int[code.Length + 1]; _spillAt = spillAt; _sl = layout.SpillFrom != 0 ? layout : Spill; int highest = HighestGpr(code); if (spillAt < 0 && highest >= UrLimit) throw new RegisterPressureException($"guest registers up to R{highest}"); UrBase = spillAt >= 0 ? SpillFrom : (highest + 2) & ~1; LastUrBase = UrBase; } // Spill mode: the temporary that holds guest register r for this instruction, loading its group on first use. int Spilled(int r) { int g = r & ~3; if (!_temps.TryGetValue(g, out int t)) { if (_temps.Count == 4) throw Fail("more than four spilled register groups in one instruction"); t = _sl.TempBase + 4 * _temps.Count; _temps[g] = t; for (int k = 0; k < 4; k++) // LDL.32 t+k, [RZ + slot] Emit(Unguarded(0xef40000000000000UL | (uint)(t + k) | ((ulong)RZ << 8) | ((ulong)(uint)(_spillAt + 4 * (g - SpillFrom + k)) << 20) | (4UL << 48))); } return t + (r & 3); } // Spill mode: the temporaries of this instruction back to local memory. void SpillStore() { foreach (var (g, t) in _temps) for (int k = 0; k < 4; k++) // STL.32 [RZ + slot], t+k Emit(Unguarded(0xef50000000000000UL | (uint)(t + k) | ((ulong)RZ << 8) | ((ulong)(uint)(_spillAt + 4 * (g - SpillFrom + k)) << 20) | (4UL << 48))); _temps.Clear(); } // The highest general register the program names (destination, A, B in the register form, C), counting the // register fields of uniform-datapath instructions too: an overestimate only moves the uniform registers up. static int HighestGpr(Insn[] code) { int max = -1; foreach (var i in code) { void see(uint r) { if (r < RZ && r > max) max = (int)r; } uint op = i.Bits(0, 12), b = op & 0x1ff; if (b is >= 0x140 and <= 0x15f) continue; // branches, calls, exits, barriers: no register fields see(i.Bits(16, 8)); see(i.Bits(24, 8)); // Register ranges the base fields do not show: LDSM writes Rd..Rd+count-1; TXD reads Ra..Ra+2 and Rb..Rb+3 // and writes the second destination pair; HMMA reads A (K/4 registers), B (K/8) and C and writes D, where // C and D are a pair (.F16) or four registers (.F32, bit 76). if (op == 0x83b && i.Bits(16, 8) != RZ) see(i.Bits(16, 8) + (1u << (int)i.Bits(72, 2)) - 1); if (op == 0xb6c) { see(i.Bits(16, 8) + 1); see(i.Bits(24, 8) + 2); see(i.Bits(32, 8) + 3); see(i.Bits(64, 8) + 1); } if (op == 0x23c) { uint k16 = i.Bits(75, 1), cd = i.Bit(76) ? 3u : 1u; see(i.Bits(16, 8) + cd); see(i.Bits(24, 8) + 1 + 2 * k16); see(i.Bits(32, 8) + k16); see(i.Bits(64, 8) + cd); } if (op == 0x3a9) see(i.Bits(64, 8)); // ATOMG.CAS: the new value if (op == 0x593 || (op == 0xb66 && i.Bit(78))) see(i.Bits(32, 8)); // SUATOM data, TLD.MS sample index if (b < 0x080 || b is >= 0x160 and <= 0x16f) see(i.Bits(64, 8)); // C of ALU forms, TEX second destinations if (i.Bits(9, 3) == 1) see(i.Bits(32, 8)); } return max; } LoweringException Fail(string reason) => new($"sm86 instruction {_index} (0x{_index * 16:x}) op {_i.Bits(0, 12):x3} [{_i.W0:x8} {_i.W1:x8} {_i.W2:x8} {(uint)(_i.Hi >> 32):x8}]: {reason}"); void Emit(ulong word) => _out.Add(word); // Guarded by the instruction's own predicate. void EmitG(ulong word) => _out.Add(word | _guard); static ulong Unguarded(ulong word) => word | ((ulong)PT << 16); // ---- Maxwell building blocks ------------------------------------------------------------------------------- // guarded: under the instruction's predicate (a guest-visible write); otherwise unconditional (scratch). void Put(ulong word, bool guarded) => _out.Add(guarded ? word | _guard : Unguarded(word)); void Mov(int dst, int src, bool guarded = true) => Put(0x5c98000000000000UL | (15UL << 39) | ((ulong)src << 20) | (uint)dst, guarded); void Mov32i(int dst, uint imm, bool guarded = false) => Put(0x0100000000000000UL | (15UL << 12) | ((ulong)imm << 20) | (uint)dst, guarded); void MovC(int dst, int bank, int byteOffset, bool guarded = false) => Put(0x4c98000000000000UL | (15UL << 39) | ((ulong)(byteOffset / 4) << 20) | ((ulong)bank << 34) | (uint)dst, guarded); void Lop32i(int dst, int a, uint imm, int op) => Emit(Unguarded(0x0400000000000000UL | ((ulong)op << 53) | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)dst)); void Fmul32i(int dst, int a, uint imm) => Emit(Unguarded(0x1e00000000000000UL | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)dst)); void Iadd3(int dst, int a, int b, int c, bool guarded, bool negB = false) => Put(0x5cc0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst | (negB ? 1UL << 50 : 0), guarded); // Rd = p ? a : b (p: index | negate << 3) void Sel(int dst, int a, int b, uint p, bool guarded) => Put(0x5ca0000000000000UL | ((ulong)(p & 7) << 39) | ((ulong)(p >> 3) << 42) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst, guarded); // Pd = a < b, unsigned (the carry out of a + b is sum < a) void IsetpLtU(int pd, int a, int b) => EmitG(0x5b60000000000000UL | (1UL << 49) | ((ulong)PT << 39) | ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)pd << 3) | PT); // ---- SM86 operand decoding --------------------------------------------------------------------------------- uint Form => _i.Bits(9, 3); int Dst => (int)_i.Bits(16, 8); int RegA => (int)_i.Bits(24, 8); int Ur(uint reg) { if (reg == 63) return RZ; if (reg > 62) throw Fail($"UR{reg}"); if (_spillAt >= 0) { int q = (int)reg & ~3; if (!_urBlocks.TryGetValue(q, out int b)) { b = _sl.UrPool + 4 * _urBlocks.Count; if (b + 4 > UrPoolEnd) throw new UrPoolException(Fail("more uniform register groups than spill mode has room for").Message); _urBlocks[q] = b; } return b + ((int)reg & 3); } if (UrBase + (int)reg >= UrLimit) throw new RegisterPressureException($"UR{reg} has no free register (guest registers up to R{UrBase - 1})"); return UrBase + (int)reg; } // While a uniform-datapath instruction is lowered through its vector twin (UniformAlu), register fields name // uniform registers. bool _uniform; int Gpr(uint reg) { if (_uniform) return reg == RZ ? RZ : Ur(reg); if (reg != RZ && _spillAt >= 0 && reg >= SpillFrom) return Spilled((int)reg); if (reg != RZ && reg >= UrBase) throw Fail($"R{reg} above the registers the program was scanned for"); return (int)reg; } // Constant bank 24 holds NVN2 driver data the shaders read (fullscreen passes: texel size, viewport rectangle); // Maxwell has 18 banks, so it is bank 17 here and the runtime binds it there. static int Bank(uint bank) => bank == 24 ? 17 : (int)bank; // Constant-buffer operand: byte offset 38-53, bank 54-58. (int bank, int offset) Cbuf() { int bank = Bank(_i.Bits(54, 5)), offset = (int)_i.Bits(38, 16); if (bank >= 18 || (offset & 3) != 0 || (_i.Bit(91) && !_uniform)) throw Fail($"constant buffer operand c[{_i.Bits(54, 5)}][0x{offset:x}]"); return (bank, offset); } // Source 0 (A), 1 (B) or 2 (C) with its float modifiers. Forms (bits 9-11): 1 B and C registers, 2 C immediate, // 3 C constant, 4 B immediate, 5 B constant, 6 B uniform, 7 C uniform. A wide (immediate/constant/uniform) operand // uses bits 32-63; the register operand that would sit there then moves to bits 64-71. (int reg, bool abs, bool neg) Raw(int which, int scratch) { if (which == 0) return (Gpr(_i.Bits(24, 8)), _i.Bit(73), _i.Bit(72)); uint form = Form; if (form == 0) throw Fail("source form 0"); bool wide = which == 1 ? form is 4 or 5 or 6 : form is 2 or 3 or 7; if (wide) { if (form is 4 or 2) { Mov32i(scratch, _i.W1); return (scratch, false, false); } if (form is 5 or 3) { var (bank, off) = Cbuf(); MovC(scratch, bank, off); return (scratch, _i.Bit(62), _i.Bit(63)); } return (Ur(_i.Bits(32, 8)), _i.Bit(62), _i.Bit(63)); } bool upper = which == 2 || form is 2 or 3 or 7; return upper ? (Gpr(_i.Bits(64, 8)), _i.Bit(74), _i.Bit(75)) : (Gpr(_i.Bits(32, 8)), _i.Bit(62), _i.Bit(63)); } // The second operand of a two-source instruction sits in the B slot, or in the C slot for forms 2, 3 and 7. int SecondSlot => Form is 2 or 3 or 7 ? 2 : 1; // Applies float abs/neg in a scratch register (bit operations, so -0.0 and NaN payloads stay exact). int Mat(int reg, bool abs, bool neg, int scratch) { if (!abs && !neg) return reg; if (abs) { Lop32i(scratch, reg, 0x7fffffff, 0); reg = scratch; } if (neg) { Lop32i(scratch, reg, 0x80000000, 2); reg = scratch; } return reg; } int Plain(int which, int scratch) { var (r, _, _) = Raw(which, scratch); return r; } int FloatMat(int which, int scratch) { var (r, a, n) = Raw(which, scratch); return Mat(r, a, n, scratch); } // A uniform guard is swapped in by Run like any other uniform predicate operand. void RequireUniformGuardTrue() { } // Uniform-datapath instructions: their guard and predicate fields name uniform predicates. static bool IsUniformOp(uint op) => ((op & 0x1ff) is >= 0x080 and <= 0x0bf && op != 0x886) || op == 0x9c3; // VOTEU: Vote // Uniform predicates the instruction reads and writes, as masks of UP indices (UPT excluded). (uint inUp, uint outUp) UniformPredicates() { uint op = _i.Bits(0, 12); if (!IsUniformOp(op)) return (0, 0); uint inUp = 0, outUp = 0; void use(ref uint mask, uint p) { if ((p & 7) != 7) mask |= 1u << (int)(p & 7); } use(ref inUp, _i.Bits(12, 3)); switch (op & 0x1ff) { case 0x08c: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); use(ref inUp, _i.Bits(87, 3)); break; // UISETP case 0x09c: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); // UPLOP3 use(ref inUp, _i.Bits(87, 3)); use(ref inUp, _i.Bits(77, 3)); use(ref inUp, _i.Bits(68, 3)); break; case 0x090: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); // UIADD3 if (_i.Bit(74)) { use(ref inUp, _i.Bits(87, 3)); use(ref inUp, _i.Bits(77, 3)); } break; case 0x092: use(ref outUp, _i.Bits(81, 3)); use(ref inUp, _i.Bits(87, 3)); break; // ULOP3 case 0x087: use(ref inUp, _i.Bits(87, 3)); break; // USEL case 0x091: case 0x0a4: case 0x0a5: use(ref outUp, _i.Bits(81, 3)); if (_i.Bit(74)) use(ref inUp, _i.Bits(87, 3)); break; } return (inUp, outUp); } // ---- driver ---------------------------------------------------------------------------------------------- public List Run(out List<(int output, int targetSource)> branches, out int[] firstOutput) { Mov32i(RUP, 0); // uniform predicates start false for (_index = 0; _index < _code.Length; _index++) { _i = _code[_index]; _firstOutput[_index] = _out.Count; if (_i.Bits(12, 4) == 15) continue; // @!PT: never runs (compiler padding) _guard = (ulong)_i.Bits(12, 4) << 16; var (inUp, outUp) = UniformPredicates(); uint m = inUp | outUp; if (m != 0) { Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RZ << 8) | USAVE)); // P2R: save guest P Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)RUP << 8))); // R2P: load UPs } LowerOne(); if (_temps.Count != 0) SpillStore(); if (m != 0) { if (outUp != 0) Emit(Unguarded(0x38e8000000000000UL | ((ulong)outUp << 20) | ((ulong)RUP << 8) | RUP)); // P2R merge Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)USAVE << 8))); // R2P: restore } } _firstOutput[_code.Length] = _out.Count; branches = _branches; firstOutput = _firstOutput; return _out; } void LowerOne() { uint op = _i.Bits(0, 12); switch (op) { // Scheduling, dependency and reconvergence hints: the translator runs each thread on its own. case 0x918: // NOP case 0x91a: // DEPBAR case 0x945: // BSSY case 0x941: // BSYNC case 0x942: // BREAK case 0x948: // WARPSYNC case 0x9ab: // ERRBAR: waits for earlier memory errors to be reported, which the host GPU does not do here return; case 0x202: case 0x802: case 0xa02: case 0xc02: Mov_(op); return; case 0x882: case 0xc82: Umov(op); return; case 0xab9: Uldc(); return; case 0x290: case 0x890: case 0xa90: // UIADD3 case 0x2a4: case 0x4a4: case 0x8a4: case 0xaa4: case 0x2a5: case 0x8a5: // UIMAD, UIMAD.WIDE case 0x292: case 0x892: case 0xa92: // ULOP3 case 0x299: case 0x499: case 0x899: // USHF case 0x291: case 0x891: // ULEA case 0x287: case 0x887: // USEL case 0x28c: case 0x88c: case 0xa8c: // UISETP case 0x89c: // UPLOP3 case 0x296: case 0x896: // UPRMT UniformAlu(op); return; case 0x81c: Plop3(); return; case 0xb63: Tld4(); return; case 0x388: case 0x984: case 0x988: SharedAccess(op); return; case 0x83b: Ldsm(); return; case 0xb1d: Bar(); return; case 0x992: Membar(); return; case 0x305: case 0x905: case 0xb05: case 0xd05: F2i(op); return; case 0xb82: Ldc(); return; case 0x381: case 0x981: Ldg(op); return; case 0x386: case 0x986: Stg(op); return; case 0xb66: Tld(); return; case 0xb6c: Txd(); return; case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xc21: case 0xe21: Fadd(); return; case 0x220: case 0x420: case 0x620: case 0x820: case 0xa20: case 0xc20: Fmul(); return; case 0x223: case 0x423: case 0x623: case 0x823: case 0xa23: case 0xc23: case 0xe23: Ffma(); return; case 0x209: case 0x409: case 0x609: case 0x809: case 0xa09: case 0xc09: Fmnmx(); return; case 0x308: case 0x908: case 0xb08: case 0xd08: Mufu(op); return; case 0x307: case 0xb07: Frnd(op); return; case 0x20b: case 0x40b: case 0x60b: case 0x80b: case 0xa0b: case 0xc0b: Setp(true); return; case 0x20c: case 0x40c: case 0x60c: case 0x80c: case 0xa0c: case 0xc0c: Setp(false); return; case 0x20a: case 0x40a: case 0x60a: case 0x80a: case 0xa0a: case 0xc0a: Fset(); return; case 0x207: case 0x807: case 0xa07: case 0xc07: Sel_(false); return; case 0x208: case 0x808: case 0xa08: case 0xc08: Sel_(true); return; case 0x245: case 0x845: case 0xa45: case 0xc45: I2fp(); return; case 0x210: case 0x410: case 0x610: case 0x810: case 0xa10: case 0xc10: Iadd3_(); return; case 0x224: case 0x424: case 0x624: case 0x824: case 0xa24: case 0xc24: case 0xe24: Imad(); return; case 0x212: case 0x812: case 0xa12: case 0xc12: Lop3(); return; case 0x217: case 0x817: case 0xa17: case 0xc17: Imnmx(); return; case 0x219: case 0x419: case 0x619: case 0x819: case 0xa19: case 0xc19: Shf(); return; case 0x23e: case 0x43e: case 0x83e: case 0xa3e: case 0xc3e: F2fp(); return; case 0x216: case 0x416: case 0x616: case 0x816: case 0xa16: case 0xc16: Prmt(); return; case 0x309: case 0xd09: Popc(); return; case 0x300: case 0xd00: Flo(); return; case 0x301: Brev(); return; case 0x240: case 0x840: Hmnmx2(); return; case 0x234: case 0x434: Hsetp2(); return; case 0x803: case 0x804: PredicateRegister(op == 0x803); return; case 0x59f: Sured(); return; case 0x593: Suatom(); return; case 0x3a9: AtomgCas(); return; case 0x3c4: Redux(); return; case 0x81a: Sgxt(); return; case 0x806: case 0x886: Vote(op == 0x886); return; case 0x3a1: Match(); return; case 0xf8c: AtomsPopcInc(); return; case 0x3a8: case 0x9a8: Atomg(op == 0x9a8); return; case 0x38c: Atoms(); return; case 0x98e: Red(); return; case 0x389: case 0x589: case 0x989: case 0xf89: Shfl(op); return; case 0x822: Fswzadd(); return; case 0x230: case 0x430: case 0x630: Half(0x5d10000000000000UL); return; // HADD2 case 0x232: case 0x832: case 0xa32: Half(0x5d08000000000000UL); return; // HMUL2 case 0x231: case 0x431: case 0x631: case 0x831: case 0xa31: Half(0x5d00000000000000UL); return; // HFMA2 case 0xb9b: SurfaceAccess(true); return; case 0xb97: SurfaceAccess(false); return; case 0x211: case 0x811: case 0xa11: case 0xc11: Lea(); return; case 0x321: case 0x322: AttributeAccess(op == 0x322); return; case 0x326: Ipa(); return; case 0x805: Cs2r(); return; case 0x919: case 0x9c3: S2r(op == 0x9c3); return; case 0x927: Isbewr(); return; case 0x949: Brx(false); return; case 0x958: Brx(true); return; case 0x923: Isberd(); return; case 0x3c2: R2ur(); return; case 0x225: case 0x227: case 0x625: case 0x825: case 0x827: case 0xa27: ImadWide(op); return; case 0x213: case 0xa13: case 0xc13: Iabs(); return; case 0xb6f: Txq(); return; case 0xb60: case 0xf60: case 0x361: Tex(op); return; case 0xb69: case 0xf69: case 0x36a: Tmml(op); return; case 0xf66: TldBindless(); return; case 0x370: TxqBindless(); return; case 0x95b: Kill(); return; case 0x94d: Exit(); return; case 0x947: Bra(); return; case 0x944: Call(); return; case 0x950: Ret(); return; case 0x2bd: Uflo(); return; case 0x2be: Ubrev(); return; case 0x2bf: Upopc(); return; case 0x883: Up2ur(); return; case 0x243: F2ipByte(); return; case 0x23c: Hmma(); return; case 0x23a: Movm(); return; case 0x306: case 0xb06: case 0xd06: I2f(op); return; case 0x387: case 0x987: case 0x983: LocalAccess(op != 0x983); return; case 0x526: IpaIndexed(); return; case 0xabb: UldcIndexed(); return; } throw Fail("opcode not lowered yet"); } // ---- moves and uniform datapath ---------------------------------------------------------------------------- void Mov_(uint op) { if (_i.Bits(72, 4) != 15) throw Fail("MOV lane mask"); int d = Gpr((uint)Dst); switch (op) { case 0x802: Mov32i(d, _i.W1, true); return; case 0x202: Mov(d, Gpr(_i.Bits(32, 8))); return; case 0xc02: Mov(d, Ur(_i.Bits(32, 8))); return; default: { var (bank, off) = Cbuf(); MovC(d, bank, off, true); return; } } } void Umov(uint op) { RequireUniformGuardTrue(); int d = Ur((uint)Dst); if (d == RZ) return; if (op == 0x882) Mov32i(d, _i.W1, true); else EmitG(0x5c98000000000000UL | (15UL << 39) | ((ulong)Ur(_i.Bits(32, 8)) << 20) | (uint)d); } // ULDC: width (bits 73-75) 4 = 32 bit, 5 = 64 bit into a uniform pair. void Uldc() { RequireUniformGuardTrue(); uint width = _i.Bits(73, 3); int d = Ur((uint)Dst); if (width < 4) // U8, S8, U16, S16: Maxwell LDC with the same width code, immediate address (any alignment) { // (this form, 0xab9, has no register index: bits 24-29 are not read, as for the 32- and 64-bit loads // below; compilers leave 0 there as well as 63, and the 0 cases failed. 0xabb is the indexed form.) int cb = Bank(_i.Bits(54, 5)), at = (int)_i.Bits(38, 16); if (cb >= 18 || at >= 0x8000) throw Fail($"ULDC width {width} c[{_i.Bits(54, 5)}][0x{at:x}]"); if (d != RZ) EmitG(0xef90000000000000UL | ((ulong)RZ << 8) | ((ulong)at << 20) | ((ulong)cb << 36) | ((ulong)width << 48) | (uint)d); return; } var (bank, off) = Cbuf(); if (width == 4) { if (d != RZ) MovC(d, bank, off, true); return; } if (width == 5 && (Dst & 1) == 0) { if (d != RZ) { MovC(d, bank, off, true); MovC(d + 1, bank, off + 4, true); } return; } throw Fail($"ULDC width {width}"); } // Uniform-datapath arithmetic (UIADD3, UIMAD, UIMAD.WIDE, ULOP3, USHF, ULEA, USEL) has the encoding of its vector // twin (base opcode - 0x80) with uniform registers in the register fields and bit 91 set; it is lowered as the // twin with the fields read as uniform registers; its predicate fields name uniform predicates, which Run has // swapped into the P registers of the same numbers. void UniformAlu(uint op) { uint twin = (op & 0x1ff) - 0x80; Insn saved = _i; _i.Lo = (_i.Lo & ~0x1ffUL) | twin; _i.Hi &= ~(1UL << 27); // bit 91 _uniform = true; try { LowerOne(); } finally { _uniform = false; _i = saved; } } // ---- floating point ---------------------------------------------------------------------------------------- void Fadd() { if (_i.Bits(81, 10) != 0) throw Fail("FADD mode"); var (a, aAbs, aNeg) = Raw(0, SA); var (b, bAbs, bNeg) = Raw(SecondSlot, SB); EmitG(0x5c58000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) | (aNeg ? 1UL << 48 : 0) | (aAbs ? 1UL << 46 : 0) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0) | ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) | ((ulong)_i.Bits(77, 1) << 50)); } // FMUL: scale (bits 84-86) 1-3 divide by 8/4/2, 4 none, 5-7 multiply by 2/4/8; Maxwell's scale field is 0 none, // 1-3 divide by 2/4/8, 4-6 multiply by 8/4/2. static readonly uint[] FmulScale = { 0, 3, 2, 1, 0, 6, 5, 4 }; void Fmul() { if (_i.Bit(76) || _i.Bits(81, 3) != 0 || _i.Bits(87, 4) != 0 || _i.Bits(84, 3) == 0) throw Fail("FMUL mode"); var (a, aAbs, aNeg) = Raw(0, SA); var (b, bAbs, bNeg) = Raw(1, SB); a = Mat(a, aAbs, false, SA); b = Mat(b, bAbs, false, SB); EmitG(0x5c68000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) | (aNeg != bNeg ? 1UL << 48 : 0) | ((ulong)FmulScale[_i.Bits(84, 3)] << 41) | ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) | ((ulong)_i.Bits(77, 1) << 50)); } void Ffma() { if (_i.Bit(76) || _i.Bits(81, 10) != 0) throw Fail("FFMA mode"); var (a, aAbs, aNeg) = Raw(0, SA); var (b, bAbs, bNeg) = Raw(1, SB); var (c, cAbs, cNeg) = Raw(2, SC); a = Mat(a, aAbs, false, SA); b = Mat(b, bAbs, false, SB); c = Mat(c, cAbs, false, SC); EmitG(0x5980000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) | (aNeg != bNeg ? 1UL << 48 : 0) | (cNeg ? 1UL << 49 : 0) | ((ulong)_i.Bits(77, 1) << 50) | ((ulong)_i.Bits(78, 2) << 51) | ((ulong)_i.Bits(80, 1) << 53)); } // FMNMX: Rd = p ? min : max (p in bits 87-90). .NAN (bit 81) is not lowered. void Fmnmx() { if (_i.Bits(74, 6) != 0 || _i.Bits(81, 6) != 0) throw Fail("FMNMX mode"); var (a, aAbs, aNeg) = Raw(0, SA); var (b, bAbs, bNeg) = Raw(SecondSlot, SB); EmitG(0x5c60000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) | (aNeg ? 1UL << 48 : 0) | (aAbs ? 1UL << 46 : 0) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0) | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(80, 1) << 44)); } // MUFU: operation in bits 74-79, numbered as Maxwell's (0 COS, 1 SIN, 2 EX2, 3 LG2, 4 RCP, 5 RSQ, 8 SQRT). // SM70+ COS/SIN take the angle in turns (the compiler multiplies by 1/2pi first); the translator's take radians. // MUFU.F16 (bit 73, register form): the half in the low or, with bit 60, the high 16 bits of Rb, abs 62 and // neg 63; the result is a half in the low 16 bits. The upscaler's colour decode raises halves to a power this // way (LG2.F16, HMUL2, EX2.F16, the two lanes packed by PRMT). Lowered as F2F to F32, the F32 function and F2F // back to F16 (round to nearest), which writes the half with the high 16 bits clear. void Mufu(uint op) { uint operation = _i.Bits(74, 6); bool half = _i.Bit(73); if (operation > 8 || operation is 6 or 7 || (half && (op != 0x308 || _i.Bit(61)))) throw Fail($"MUFU operation {operation}{(half ? ".F16" : "")}"); int src; bool abs, neg; if (op == 0x908) { Mov32i(SB, _i.W1); (src, abs, neg) = (SB, false, false); } else if (half) { Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)Gpr(_i.Bits(32, 8)) << 20) | (_i.Bit(60) ? 1UL << 41 : 0) | (_i.Bit(63) ? 1UL << 45 : 0) | (_i.Bit(62) ? 1UL << 49 : 0) | SB)); // F2F.F32.F16 (src, abs, neg) = (SB, false, false); } else (src, abs, neg) = Raw(1, SB); if (operation <= 1) { src = Mat(src, abs, neg, SB); Fmul32i(SB, src, 0x40c90fdb); // 2pi src = SB; abs = neg = false; } ulong mufu = 0x5080000000000000UL | ((ulong)operation << 20) | ((ulong)src << 8) | (abs ? 1UL << 46 : 0) | (neg ? 1UL << 48 : 0); if (!half) { EmitG(mufu | (uint)Gpr((uint)Dst)); return; } Emit(Unguarded(mufu | SB)); EmitG(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)SB << 20) | (uint)Gpr((uint)Dst)); // F2F.F16.F32 } // FRND.F32: rounding (bits 78-79: nearest/floor/ceil/trunc) as Maxwell F2F.F32.F32 with integer rounding. void Frnd(uint op) { if (_i.Bits(75, 2) != 2 || _i.Bits(84, 2) != 2 || _i.Bits(81, 3) != 0 || _i.Bits(86, 6) != 0) throw Fail("FRND format"); var (b, bAbs, bNeg) = Raw(1, SB); EmitG(0x5ca8000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)b << 20) | (uint)Gpr((uint)Dst) | ((ulong)(8 + _i.Bits(78, 2)) << 39) | ((ulong)_i.Bits(80, 1) << 44) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0)); } // FSETP / ISETP: P (bits 81-83), Q (84-86), combined with Pc (87-90) by the operation in 74-75. void Setp(bool floating) { if (!floating && _i.Bit(72)) { SetpExtended(); return; } uint combine = _i.Bits(74, 2); if (combine > 2) throw Fail("predicate combination"); var (a, aAbs, aNeg) = Raw(0, SA); var (b, bAbs, bNeg) = Raw(1, SB); ulong w = (floating ? 0x5bb0000000000000UL : 0x5b60000000000000UL) | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3) | ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)combine << 45); if (floating) w |= ((ulong)_i.Bits(80, 1) << 47) | ((ulong)_i.Bits(76, 4) << 48) | (aNeg ? 1UL << 43 : 0) | (aAbs ? 1UL << 7 : 0) | (bNeg ? 1UL << 6 : 0) | (bAbs ? 1UL << 44 : 0); else w |= ((ulong)_i.Bits(73, 1) << 48) | ((ulong)_i.Bits(76, 3) << 49); EmitG(w); } // FSET.BF: Rd = 1.0 or 0.0. void Fset() { if (_i.Bits(64, 8) != 0 || _i.Bits(74, 2) == 3 || _i.Bits(81, 6) != 0) throw Fail("FSET output mode"); var (a, aAbs, aNeg) = Raw(0, SA); var (b, bAbs, bNeg) = Raw(1, SB); EmitG(0x5800000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(74, 2) << 45) | ((ulong)_i.Bits(76, 4) << 48) | (1UL << 52) | ((ulong)_i.Bits(80, 1) << 55) | (aNeg ? 1UL << 43 : 0) | (bNeg ? 1UL << 53 : 0) | (aAbs ? 1UL << 54 : 0) | (bAbs ? 1UL << 44 : 0)); } // SEL / FSEL: Rd = p ? a : b (p in bits 87-90). FSEL applies the float modifiers first. void Sel_(bool floating) { int a = floating ? FloatMat(0, SA) : Plain(0, SA); int b = floating ? FloatMat(1, SB) : Plain(1, SB); Sel(Gpr((uint)Dst), a, b, _i.Bits(87, 4), true); } // I2FP.F32.{U32,S32}: bit 74 signed, rounding in 78-79. void I2fp() { uint w2 = _i.W2 & ~(0x400u | 0xc000u | 0x08000000u); if (w2 != 0x00201000) throw Fail("I2FP format"); int b = Plain(1, SB); EmitG(0x5cb8000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)_i.Bits(74, 1) << 13) | ((ulong)b << 20) | ((ulong)_i.Bits(78, 2) << 39) | (uint)Gpr((uint)Dst)); } // ---- integer ----------------------------------------------------------------------------------------------- // IADD3 Rd, [Pu,] a, b, c. Negations are arithmetic (Maxwell IADD3's too). One carry out (bits 81-83) with c = RZ, // or .X (bit 74) adding the carry-in predicate of bits 87-90. void Iadd3_() { uint op = _i.Bits(0, 12); bool negA = _i.Bit(72), negB = op != 0x810 && _i.Bit(63) && Form is not 2 and not 3, negC = _i.Bit(75), extended = _i.Bit(74); uint carry1 = _i.Bits(81, 3), carry2 = _i.Bits(84, 3); int a = Plain(0, SA), b = Plain(1, SB), c = Plain(2, SC); int d = Gpr((uint)Dst); if (Form is 2 or 3 && (_i.Bit(63) || negC)) throw Fail("IADD3 negation with a wide third source"); if (!extended && carry1 == 7 && carry2 == 7) { EmitG(0x5cc0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)d | (negA ? 1UL << 51 : 0) | (negB ? 1UL << 50 : 0) | (negC ? 1UL << 49 : 0)); return; } // The 34-bit sum: low word in R244, carry count in R245 (IADD.CC / IADD.X). A negated source is ~x, plus 1 // unless .X, whose carry-ins (bits 87-90 and 77-80) are added instead. Pu = bit 32, Pv = bit 33. const int lo = 244, hi = 245, one = 246; int inv(int r, bool neg, int t) { if (!neg) return r; Lop32i(t, r, 0xffffffff, 2); return t; } a = inv(a, negA, 241); b = inv(b, negB, 242); c = inv(c, negC, 243); void addc(int r) { Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)r << 20) | ((ulong)lo << 8) | lo)); // IADD.CC Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)RZ << 20) | ((ulong)hi << 8) | hi)); // IADD.X } Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)b << 20) | ((ulong)a << 8) | lo)); Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)RZ << 20) | ((ulong)RZ << 8) | hi)); addc(c); if (extended) foreach (uint p in new[] { _i.Bits(87, 4), _i.Bits(77, 4) }) { if (p == 15) continue; Mov32i(one, 1); if (p != 7) Sel(one, one, RZ, p, false); addc(one); } else if (negA || negB || negC) { Mov32i(one, (uint)((negA ? 1 : 0) + (negB ? 1 : 0) + (negC ? 1 : 0))); addc(one); } if (d != RZ) Mov(d, lo); uint guard = _i.Bits(12, 3); foreach (int k in guard == carry1 ? new[] { 1, 0 } : new[] { 0, 1 }) // an output that is the guard goes last { uint p = k == 0 ? carry1 : carry2; if (p == 7) continue; Lop32i(one, hi, k == 0 ? 1u : 2u, 0); EmitG(0x5b60000000000000UL | (5UL << 49) | ((ulong)PT << 39) | ((ulong)RZ << 20) | ((ulong)one << 8) | ((ulong)p << 3) | PT); } } // IMAD Rd, a, b, c (low word); bit 73 signed, bit 75 negates c (~c with .X), .X (bit 74) adds the carry-in // predicate of bits 87-90. A carry out (bits 81-83) is not lowered; .HI/.WIDE are ImadWide. void Imad() { uint known = 0xffu | 1u << 9 | 1u << 10 | 1u << 11 | 7u << 17 | 0xfu << 23 | (Form is 6 or 7 ? 1u << 27 : 0u); bool x = _i.Bit(74); if ((_i.W2 & ~known) != 0 || _i.Bits(81, 3) != 7 || (!x && _i.Bits(87, 4) != 15)) throw Fail("IMAD mode"); int a = Plain(0, SA), b = Plain(1, SB), c = Plain(2, SC); if (_i.Bit(75) && c != RZ) { if (x) Lop32i(SC, c, 0xffffffff, 2); // .X: ~c, the +1 comes as the carry else Iadd3(SC, RZ, c, RZ, false, negB: true); c = SC; } ulong signed = _i.Bit(73) ? (1UL << 48) | (1UL << 53) : 0; int d = Gpr((uint)Dst); ulong imad(int dst) => 0x5a00000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst | signed; if (!x) { EmitG(imad(d)); return; } Mov32i(SX, 1); Sel(SX, SX, RZ, _i.Bits(87, 4), false); Emit(Unguarded(imad(SA))); Iadd3(d, SA, SX, RZ, true); } // IMAD.HI Rd / IMAD.WIDE Rd:Rd+1 = a * b + (c+1:c), bit 73 signed. b: a register (bits 32-39), an immediate // (0x8xx) or a constant (0xa27); 0x625 takes b from bits 64-71 and adds a constant pair. The 64-bit sum uses // Maxwell IMUL.LO/.HI and IADD.CC/.X into R247:R248. A carry out (bits 81-83) is not lowered. void ImadWide(uint op) { bool wide = op is 0x225 or 0x625 or 0x825, signedMul = _i.Bit(73); if ((_i.W2 & ~0x000e02ffu) != 0x07800000 || _i.Bits(81, 3) != 7) throw Fail("IMAD.HI/WIDE mode"); int a = Gpr(_i.Bits(24, 8)), b, cLo, cHi; if (op is 0x825 or 0x827) { Mov32i(SB, _i.W1); b = SB; } else if (op == 0xa27) { var (bank, off) = Cbuf(); MovC(SB, bank, off); b = SB; } else b = Gpr(_i.Bits(op == 0x625 ? 64 : 32, 8)); if (op == 0x625) { var (bank, off) = Cbuf(); MovC(SC, bank, off); MovC(SX, bank, off + 4); cLo = SC; cHi = SX; } else { int c = Gpr(_i.Bits(64, 8)); if (c != RZ && (c & 1) != 0) throw Fail("IMAD.WIDE odd addend pair"); cLo = c; cHi = c == RZ ? RZ : c + 1; } int d = Gpr((uint)Dst); if (wide && (d == RZ || (d & 1) != 0)) throw Fail("IMAD.WIDE destination pair"); ulong sign = signedMul ? (1UL << 40) | (1UL << 41) : 0; Emit(Unguarded(0x5c38000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | 247 | sign)); // IMUL lo Emit(Unguarded(0x5c38000000000000UL | (1UL << 39) | ((ulong)b << 20) | ((ulong)a << 8) | 248 | sign)); // IMUL.HI Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)cLo << 20) | (247UL << 8) | 247)); // IADD.CC Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)cHi << 20) | (248UL << 8) | 248)); // IADD.X if (wide) { Mov(d, 247); Mov(d + 1, 248); } else Mov(d, 248); } // IABS Rd, b: max(b, -b), signed. void Iabs() { uint op = _i.Bits(0, 12); if ((_i.W2 & ~(op == 0xc13 ? 0x08000000u : 0u)) != 0) throw Fail("IABS mode"); int b = Plain(1, SB); Iadd3(SX, RZ, b, RZ, false, negB: true); EmitG(0x5c20000000000000UL | ((ulong)SX << 20) | ((ulong)b << 8) | (uint)Gpr((uint)Dst) | (15UL << 39) | (1UL << 48)); } // LOP3.LUT: table in bits 72-79, optional P = result != 0 (bits 81-83). void Lop3() { if (_i.Bit(80) || _i.Bits(87, 4) != 15) throw Fail("LOP3 predicate mode"); int a = Plain(0, SA), b = Plain(1, SB), c = Gpr(_i.Bits(64, 8)); uint p = _i.Bits(81, 3); EmitG(0x5be0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) | ((ulong)_i.Bits(72, 8) << 28) | ((ulong)(p == 7 ? 0 : 3) << 36) | ((ulong)p << 48)); } // IMNMX: Rd = p ? min : max (p in bits 87-90), bit 73 signed. void Imnmx() { int a = Plain(0, SA), b = Plain(1, SB); EmitG(0x5c20000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(73, 1) << 48)); } // SHF Rd, low (A), shift (B), high (C): bits 73-74 type (0 S64, 1 U64, 2 S32, 3 U32), 75 .W (wrap), 76 right, // 80 .HI (the high word of the 64-bit result). void Shf() { if (_i.Bits(81, 10) != 0) throw Fail("SHF mode"); uint type = _i.Bits(73, 2); bool wrap = _i.Bit(75), right = _i.Bit(76), hi = _i.Bit(80), wide = type <= 1, signedType = type == 0 || type == 2; int low = Plain(0, SA), s = Plain(1, SB), high = Plain(2, SC); int d = Gpr((uint)Dst); ulong maxShift = type == 0 ? 3UL : type == 1 ? 2UL : 0UL; if (!right && hi) // high word of (high:low) << s EmitG(0x5bf8000000000000UL | ((ulong)high << 39) | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 50 : 0) | (maxShift << 37)); else if (!right) // low word: low << s { if (wrap && wide) throw Fail("SHF.L.W 64-bit low word"); EmitG(0x5c48000000000000UL | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 39 : 0)); } else if (!hi) // low word of (high:low) >> s EmitG(0x5cf8000000000000UL | ((ulong)high << 39) | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 50 : 0) | (maxShift << 37)); else // high word: high >> s { if (wrap && wide) throw Fail("SHF.R.W 64-bit high word"); EmitG(0x5c28000000000000UL | ((ulong)s << 20) | ((ulong)high << 8) | (uint)d | (wrap ? 1UL << 39 : 0) | (signedType ? 1UL << 48 : 0)); } } // LEA Rd, [Pu,] a, b, s: (a << s) + b, shift in bits 75-79, bit 63 negates b. LEA.HI (bit 80): b + high word of // (c:a) << s (bit 73 sign-extends a when c is RZ), .X (bit 74) adds the carry predicate of bits 87-90. void Lea() { uint op = _i.Bits(0, 12); int a = Plain(0, SA), b = Plain(1, SB), d = Gpr((uint)Dst); uint shift = _i.Bits(75, 5), carry = _i.Bits(81, 3); bool negB = op != 0x811 && _i.Bit(63); if (_i.Bit(72)) throw Fail("LEA negated a"); if (!_i.Bit(80)) { if (_i.Bit(74)) throw Fail("LEA.X without .HI"); ulong iscadd(int dst, int bb) => 0x5c18000000000000UL | ((ulong)shift << 39) | ((ulong)bb << 20) | ((ulong)a << 8) | (uint)dst | (negB ? 1UL << 48 : 0); if (carry == 7) { EmitG(iscadd(d, b)); return; } if (negB) throw Fail("LEA carry out with negated b"); // The sum goes to scratch unconditionally; only the destination and the carry are guarded. A guarded // copy of b made a constant-buffer base (c[0][0x110] in Ocarina's edge-AA tile list, prog 98) a phi // of it and the scratch register's old value, and GlobalToStorage could not find the buffer. Mov(SX, b, false); Emit(Unguarded(iscadd(SC, SX))); if (d != RZ) Mov(d, SC); IsetpLtU((int)carry, SC, SX); // carry = sum < b return; } if (carry != 7) throw Fail("LEA.HI carry out"); int high = Gpr(_i.Bits(64, 8)); // With c = RZ the high word is a right shift of a by 32 - s (arithmetic when bit 73 sign-extends a); // otherwise SHF.L.HI of (c:a) by s. Mov32i(SX, high == RZ ? 32 - shift : shift); if (high == RZ) EmitG(0x5c28000000000000UL | ((ulong)SX << 20) | ((ulong)a << 8) | SC | (_i.Bit(73) ? 1UL << 48 : 0)); else EmitG(0x5bf8000000000000UL | ((ulong)high << 39) | ((ulong)SX << 20) | ((ulong)a << 8) | SC); if (_i.Bit(74)) { Mov32i(SX, 1); Sel(SX, SX, RZ, _i.Bits(87, 4), false); Iadd3(SC, SC, SX, RZ, true); } Iadd3(d, SC, b, RZ, true, negB); } // ---- attributes, special registers ------------------------------------------------------------------------ // ALD / AST: address bits 40-49, components 74-75 (+1), patch 76, output 79; vertex register 32 (ALD) / 64 (AST). void AttributeAccess(bool store) { bool patch = _i.Bit(76), output = _i.Bit(79); uint address = _i.Bits(40, 10), components = _i.Bits(74, 2) + 1; if (_i.Bit(77) || patch || (output && store) || (address & 3) != 0 || address + components * 4 > 0x400 || _i.Bits(24, 8) != RZ) throw Fail("indexed/patch attribute access"); // a[0x5c] is an Ampere per-primitive hint (shading rate); nothing reads it back. if (store && address == 0x5c && components == 1) return; int value = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst); if (value == RZ) { if (!store) return; for (int k = 0; k < components; k++) Mov32i(TexOut + k, 0); value = TexOut; } int vertex = Gpr(store ? _i.Bits(64, 8) : _i.Bits(32, 8)); EmitG((store ? 0xeff0000000000000UL : 0xefd8000000000000UL) | (uint)value | ((ulong)RZ << 8) | ((ulong)address << 20) | (output ? 1UL << 32 : 0) | ((ulong)vertex << 39) | ((ulong)(components - 1) << 47)); } // IPA: attribute (bits 64-71, in words), sample mode 76-77 (default/centroid), bit 78 flat. Maxwell IPA.PASS: // like Maxwell's, SM86 IPA of a perspective input returns the value times 1/w, and the NVN2 compiler multiplies // by w itself (IPA a[0x7c], MUFU.RCP, FMUL). Treating it as perspective-correct and adding IPA.MULTIPLY by w // scaled every texture coordinate by w a second time: Ocarina's floors and walls smeared along the view. void Ipa() { if (_i.Bits(32, 8) != RZ || _i.Bits(76, 2) > 1 || _i.Bit(79) || _i.Bits(81, 3) != 7) throw Fail("IPA mode"); EmitG(0xe000000000000000UL | ((ulong)RZ << 8) | ((ulong)RZ << 20) | ((ulong)_i.Bits(64, 8) << 30) | ((ulong)_i.Bits(76, 2) << 52) | ((ulong)_i.Bits(78, 1) << 55) | ((ulong)PT << 47) | (uint)Gpr((uint)Dst)); } // CS2R Rd, SRZ: a zeroed register pair. void Cs2r() { if (_i.Bits(72, 8) != RZ || !_i.Bit(80)) throw Fail("CS2R special register"); int d = Gpr((uint)Dst); if (d == RZ) return; Mov32i(d, 0, true); Mov32i(d + 1, 0, true); } // ISBEWR [Ra + imm], Rb: a stage buffer write of a task or mesh program (MeshInfo): Ra 24-31, Rb 32-39, the byte // offset 40-63, the region 75-78, bit 79 set (a write). Region 15 the vertex outputs, 0 the primitive indices // (bytes), 1 the primitive count, 7 the task payload. A shared memory store here. void Isbewr() { if (_mesh == null) throw Fail("ISBEWR outside a task or mesh program"); uint region = _i.Bits(75, 4), offset = _i.Bits(40, 24); if (!_i.Bit(79) || (_i.W2 & ~0xf800u) != 0 || _i.Bits(64, 8) != 0 || offset >= 0x10000) throw Fail("ISBEWR mode"); int a = Gpr(_i.Bits(24, 8)), data = Gpr(_i.Bits(32, 8)); int at; uint width = 4; if (_mesh.Stage == 7 && region == 15) at = _mesh.Attr + (int)offset; else if (_mesh.Stage == 7 && region == 0) { at = _mesh.Map + (int)offset; width = 0; } else if (_mesh.Stage == 7 && region == 1) { if (a != RZ) throw Fail("ISBEWR primitive count at a register address"); at = _mesh.Map; } else if (_mesh.Stage == 6 && region == 7) at = _mesh.Pay + (int)offset; else throw Fail($"ISBEWR region {region} in a stage {_mesh.Stage} program"); EmitG(0xef58000000000000UL | (uint)data | ((ulong)a << 8) | ((ulong)(uint)at << 20) | ((ulong)width << 48)); } // ISBERD Rd, [Ra + imm]: a stage buffer read (fields as ISBEWR's, bit 79 clear). Address 0 of region 15 (a mesh // program without a task stage, a task program) or of region 7 (a mesh program after a task stage) is the // workgroup's index; region 7 elsewhere is the task payload. With bit 79 set (the output side, as ISBEWR's) a // task program reads back the payload it writes (AglShader's task program stores a word and reloads it). void Isberd() { if (_mesh == null) throw Fail("ISBERD outside a task or mesh program"); uint region = _i.Bits(75, 4), offset = _i.Bits(40, 24); bool output = _i.Bit(79); if ((_i.W2 & ~0xf800u) != 0 || (output && (region != 7 || _mesh.Stage != 6)) || _i.Bits(32, 8) != 0 || _i.Bits(64, 8) != 0 || offset >= 0x10000) throw Fail("ISBERD mode"); int a = Gpr(_i.Bits(24, 8)), d = Gpr((uint)Dst); if (d == RZ) return; if (output) { EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | ((ulong)(uint)(_mesh.Pay + (int)offset) << 20) | (4UL << 48)); return; } if ((region == 15 || region == 7) && a == RZ && offset == 0) { Put(0xf0c8000000000000UL | (0x25UL << 20) | (uint)d, true); // S2R SR_CTAID.X return; } if (region != 7 || _mesh.Stage != 7 || !_mesh.HasTask) throw Fail($"ISBERD region {region} at a nonzero address"); EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | ((ulong)(uint)(_mesh.Pay + (int)offset) << 20) | (4UL << 48)); } // S2R / S2UR: special register in bits 72-79, numbered as Maxwell's (lane id 0, thread and workgroup ids // 33-39, lane masks 56-60, ...). SR 132 (variable-rate shading: fragment width and height in bytes 1 and 2) // reads as 1x1. void S2r(bool uniform) { uint special = _i.Bits(72, 8); if (_i.W1 != 0 || _i.W2 != special << 8) throw Fail("special register mode"); if (uniform) RequireUniformGuardTrue(); int d = uniform ? Ur((uint)Dst) : Gpr((uint)Dst); if (d == RZ) return; if (special == 132) { Mov32i(d, 0x00010100, true); return; } Put(0xf0c8000000000000UL | ((ulong)special << 20) | (uint)d, true); } // R2UR: a uniform register takes the value of the first active lane. Uniform registers are per-thread here, so // this is a move (the compiler emits R2UR for values that are uniform, or loops until they are). void R2ur() { if (_i.W1 != 0 || _i.W2 != 0x000e0000) throw Fail("R2UR mode"); int d = Ur((uint)Dst); if (d != RZ) Mov(d, Gpr(_i.Bits(24, 8))); } // ---- texture --------------------------------------------------------------------------------------------- // TEX (bound): Rd 16-23, Ra 24-31, Rb 32-39, descriptor word offset 40-53, its bank 54-58, dimension 61-63 // (1D, 2D, 3D, CUBE, ARRAY_1D, ARRAY_2D, -, ARRAY_CUBE), second destination 64-71, mask 72-75, offsets 76, // depth compare 78, FP16 results 79, LOD mode 87-89 (numbered as Maxwell's), bit 90 no-dependency. // Operands: array index, coordinates, then LOD/bias, offsets, depth reference; up to four values are split as // halves over Ra and Rb (1+1, 2+1, 2+2), five or more as four in Ra and the rest in Rb. Maxwell TEX takes the // coordinates in Ra.. and the rest in Rb.., and writes the masked components to consecutive registers. static readonly int[] TexCoords = { 1, 2, 3, 3, 2, 3, 0, 4 }; static readonly int[] TexMaxwellDim = { 0, 2, 4, 6, 1, 3, 0, 7 }; // TEX. 0xb60 a bound descriptor; the bindless forms take the 32-bit handle (texture index in bits 0-19, sampler // index above) from a uniform register in bits 40-45 (0xf60) or, with bit 59 (.B) set, as one more operand // right after the coordinates (0x361; with bit 60 clear Ra holds all coordinates and Rb the handle and the // rest). Bindless becomes Maxwell TEX.B, the handle first among the meta registers (offsets bit 36, LOD mode 37): // the translator then indexes its texture-pool arrays with it (BindlessElimination GenerateBindlessAccess, // the handle being loaded from memory). void Tex(uint op) { bool bindless = op != 0xb60; uint dim = _i.Bits(61, 3), lod = _i.Bits(87, 3), mask = _i.Bits(72, 4); bool offsets = _i.Bit(76), dc = _i.Bit(78); if (dim == 6 || _i.Bit(77) || _i.Bits(81, 3) != 7 || _i.Bit(59) != (op == 0x361) || (lod > 3 && lod != 6 && lod != 7) || mask == 0) throw Fail("TEX mode"); uint handle = 0; int handleReg = RZ; if (op == 0xb60) { handle = Handle(false); uint bank = _i.Bits(54, 5); if (handle > 0x1fff) throw Fail("TEX descriptor offset"); if (!TexBank(bank)) throw Fail($"TEX descriptor bank {bank}"); } else if (op == 0xf60) { if (_i.Bits(46, 13) != 0) throw Fail("bindless TEX mode"); handleReg = Ur(_i.Bits(40, 6)); } else if (_i.Bits(40, 19) != 0) throw Fail("TEX.B mode"); bool hasLod = lod is 2 or 3 or 6 or 7; int coords = TexCoords[dim], handleSlot = op == 0x361 ? 1 : 0; int count = coords + handleSlot + (hasLod ? 1 : 0) + (offsets ? 1 : 0) + (dc ? 1 : 0); int inA = op == 0x361 && !_i.Bit(60) ? coords : count <= 4 ? (count + 1) / 2 : op == 0x361 ? coords : 4; int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))); for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false); int next = coords + handleSlot, meta = TexMeta; if (bindless) Mov(meta++, op == 0x361 ? Value(coords) : handleReg, false); if (hasLod) Mov(meta++, Value(next++), false); if (offsets) Mov(meta++, Value(next++), false); if (dc) Mov(meta++, Value(next++), false); if (bindless) EmitG(0xde80000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20) | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | (offsets ? 1UL << 36 : 0) | ((ulong)lod << 37) | ((ulong)_i.Bits(90, 1) << 49) | (dc ? 1UL << 50 : 0) | (7UL << 51)); else EmitG(0xc000000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20) | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49) | (dc ? 1UL << 50 : 0) | (7UL << 51) | (offsets ? 1UL << 54 : 0) | ((ulong)lod << 55)); TexResults(mask, _i.Bit(79)); } // TMML (LOD query; results 8.8 fixed point as Maxwell's), 1D/2D/3D/CUBE. 0xb69 a bound descriptor; 0xf69 the // handle in a uniform register (bits 40-45, bit 91 set); 0x36a (TMML.B, bit 59) the handle in Rb and every // coordinate in Ra. Coordinates come from Ra, or from Ra and Rb as TEX's when Rb is a register. Maxwell TMML // (0xdf58, descriptor in 36-48) / TMML.B (0xdf60, handle in Rb), dimension in Maxwell's TexDim numbering. void Tmml(uint op) { bool bindless = op != 0xb69; uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4); int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); int coords = dim < 4 ? new[] { 1, 2, 3, 3 }[dim] : 0; if (coords == 0 || _i.Bits(76, 4) != 0 || _i.Bits(80, 10) != 0 || mask == 0 || _i.Bit(59) != (op == 0x36a)) throw Fail("TMML mode"); uint handle = 0; int handleReg = RZ; if (op == 0xb69) { handle = Handle(false); if (handle > 0x1fff || !TexBank(_i.Bits(54, 5))) throw Fail("TMML descriptor"); } else if (op == 0xf69) { if (_i.Bits(46, 13) != 0 || !_i.Bit(91)) throw Fail("bindless TMML mode"); handleReg = Ur(_i.Bits(40, 6)); } else { if (_i.Bits(40, 19) != 0 || _i.Bit(91)) throw Fail("TMML.B mode"); handleReg = Gpr((uint)rb); } int inA = rb == RZ || op == 0x36a ? coords : (coords + 1) / 2; for (int k = 0; k < coords; k++) Mov(TexCoord + k, k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))), false); if (bindless) Mov(TexMeta, handleReg, false); EmitG((bindless ? 0xdf60000000000000UL : 0xdf58000000000000UL) | TexOut | ((ulong)TexCoord << 8) | ((ulong)(bindless ? TexMeta : RZ) << 20) | ((ulong)(dim * 2) << 28) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49)); TexResults(mask, false); } // TLD.LZ with the handle in a uniform register (0xf66: bits 40-45, bit 91 and bit 60 set): Maxwell TLD.B, the // coordinates x | y, layer, x | y or x, y | z over Ra and Rb (1D, 2D, ARRAY_2D, 3D) and the handle in Rb. void TldBindless() { uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4); bool array = dim == 5, volume = dim == 2; if (_i.Bits(46, 13) != 0 || !_i.Bit(91) || !_i.Bit(60) || _i.Bit(59) || _i.Bits(76, 5) != 0 || _i.Bits(81, 3) != 7 || _i.Bits(87, 3) != 1 || (dim != 0 && dim != 1 && !array && !volume) || mask == 0) throw Fail("bindless TLD mode"); int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); int handleReg = Ur(_i.Bits(40, 6)); Mov(TexCoord, ra == RZ ? RZ : Gpr((uint)ra), false); if (array || volume) Mov(TexCoord + 1, ra == RZ ? RZ : Gpr((uint)(ra + 1)), false); if (dim != 0) Mov(array || volume ? TexCoord + 2 : TexCoord + 1, rb == RZ ? RZ : Gpr((uint)rb), false); Mov(TexMeta, handleReg, false); EmitG(0xdd00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20) | ((ulong)(array ? 3u : volume ? 4u : dim == 0 ? 0u : 2u) << 28) | ((ulong)mask << 31) | ((ulong)_i.Bits(90, 1) << 49) | (7UL << 51)); TexResults(mask, false); } // TXQ.B (0x370): the handle in Ra and the level in Ra+1, as Maxwell TXQ.B (0xdf50) takes them; query 1 // (dimensions), results as TXQ's. void TxqBindless() { uint mask = _i.Bits(72, 4); if (_i.Bits(59, 5) != 1 || mask == 0 || _i.Bits(76, 14) != 0 || _i.Bit(91) || _i.Bits(32, 27) != 0) throw Fail("TXQ.B mode"); int ra = (int)_i.Bits(24, 8); Mov(TexCoord, ra == RZ ? RZ : Gpr((uint)ra), false); Mov(TexCoord + 1, ra == RZ ? RZ : Gpr((uint)(ra + 1)), false); EmitG(0xdf50000000000000UL | TexOut | ((ulong)TexCoord << 8) | (1UL << 22) | ((ulong)mask << 31) | ((ulong)_i.Bits(90, 1) << 49)); TexResults(mask, false); } // F2I: destination signed (bit 72) and size (75-76: 8/16/32 bits), rounding 78-79 (nearest, floor, ceil, // trunc), FTZ 80, source size 84-85 (F16, F32). Maxwell F2I has the same rounding order. void F2i(uint op) { const uint known = 1u << 8 | 3u << 11 | 1u << 13 | 3u << 14 | 1u << 16 | 3u << 20; uint w2 = _i.W2 & ~(op == 0xd05 ? 0x08000000u : 0u); uint dstSize = _i.Bits(75, 2), srcSize = _i.Bits(84, 2); if ((w2 & ~known) != 0 || dstSize > 2 || srcSize is not 1 and not 2 || (srcSize == 1 && _i.Bits(60, 2) != 0)) throw Fail("F2I format"); int b; bool abs, neg; if (op == 0x905) { Mov32i(SB, _i.W1); (b, abs, neg) = (SB, false, false); } // F2I of a 32-bit immediate else (b, abs, neg) = Raw(1, SB); uint fmt = dstSize | (_i.Bit(72) ? 4u : 0u); // ISrcDstFmt: U8 0, U16 1, U32 2, +4 signed EmitG(0x5cb0000000000000UL | ((ulong)b << 20) | (uint)Gpr((uint)Dst) | ((ulong)(fmt & 3) << 8) | ((ulong)(fmt >> 2) << 12) | ((ulong)srcSize << 10) | ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) | (neg ? 1UL << 45 : 0) | (abs ? 1UL << 49 : 0)); } // LDC Rd, c[bank][Ra + offset]: offset 38-53, bank 54-58, size 73-75 (as Maxwell's: U8 S8 U16 S16 32 64), // mode 78-79 (0, or 2 .IS: Ra addresses all banks). void Ldc() { uint width = _i.Bits(73, 3), mode = _i.Bits(78, 2); int bank = Bank(_i.Bits(54, 5)); if (width > 5 || mode is not 0 and not 2 || (_i.W2 & ~0xce00u) != 0 || (_i.W1 & 0xf800003fu) != 0 || bank >= 18) throw Fail("LDC mode"); int d = Gpr((uint)Dst); if (width == 5 && (d & 1) != 0) throw Fail("LDC.64 odd destination"); EmitG(0xef90000000000000UL | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.Bits(38, 16) << 20) | ((ulong)bank << 36) | ((ulong)mode << 44) | ((ulong)width << 48) | (uint)d); } // Global memory address of LDG/STG: a 64-bit register pair (Ra, .E), or a uniform pair (bits 32-37 for loads, // 64-69 for stores) plus Ra (64-bit when bit 90 is set, else zero-extended), or with the "desc[UR][Ra.64]" // forms Ra alone (the uniform pair is a cache-policy descriptor). Sums go to R244:R245. int GlobalAddress(bool uniformForm, bool store) { int ra = (int)_i.Bits(24, 8); if (!uniformForm) { if (store ? (_i.W2 & ~0xe00u) != 0x0010a100 : (_i.W2 & ~0x2e00u) != 0x001e8100) throw Fail("global access mode"); if (ra != RZ && ((ra & 1) != 0)) throw Fail("global address pair"); return ra == RZ ? RZ : Gpr((uint)ra); } uint ur = store ? _i.Bits(64, 6) : _i.Bits(32, 6); bool descriptor = store ? (_i.W2 & ~(0x3fu | 0xe00u)) == 0x0c101100 : (_i.W2 & ~(0x3fu | 0xe00u)) == 0x0c1e1100 || _i.W2 == 0x0c1ebd00; // Bit 78 is .STRONG.GPU instead of .STRONG.SM (two agl_common compute programs load their arguments and store // their results so): the translator's accesses have no scope, which gives the same values outside a race. uint plain = store ? 0x0810a100u : 0x081e8100u; if (!descriptor && (_i.W2 & ~(0x04000000u | 0x3fu | 0xe00u | 0x4000u | (store ? 0u : 0x2000u))) != plain) throw Fail("global access uniform mode"); if (!store && _i.Bits(38, 2) != 0) throw Fail("global access uniform mode"); if (descriptor) { if ((ra & 1) != 0 || ra == RZ) throw Fail("global address pair"); return Gpr((uint)ra); } int u = Ur(ur); if (u != RZ && (ur & 1) != 0) throw Fail("global uniform address pair"); if (ra == RZ) return u; bool wide = _i.Bit(90); if (wide && (ra & 1) != 0) throw Fail("global address pair"); int r = Gpr((uint)ra); Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)r << 20) | ((ulong)u << 8) | 244)); // IADD.CC Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)(wide ? r + 1 : RZ) << 20) | ((ulong)(u == RZ ? RZ : u + 1) << 8) | 245)); // IADD.X return 244; } // LDG Rd, [address + offset]: offset 40-63 (signed), size 73-75 (U8 S8 U16 S16 32 64 128, as Maxwell's). void Ldg(uint op) { uint size = _i.Bits(73, 3); int d = Gpr((uint)Dst); if (size > 6 || (size == 5 && (d & 1) != 0) || (size == 6 && (d & 3) != 0)) throw Fail("LDG size"); if (op == 0x381 && _i.Bits(32, 8) != 0) throw Fail("LDG mode"); int addr = GlobalAddress(op == 0x981, false); EmitG(0xeed0000000000000UL | (uint)d | ((ulong)addr << 8) | ((ulong)_i.Bits(40, 24) << 20) | (1UL << 45) | ((ulong)size << 48)); } // STG [address + offset], Rdata: data 32-39, offset 40-63, size 73-75; .E (bit 72) required. void Stg(uint op) { uint size = _i.Bits(73, 3); int data = Gpr(_i.Bits(32, 8)); if (size > 6 || !_i.Bit(72) || (data != RZ && ((size == 5 && (data & 1) != 0) || (size == 6 && (data & 3) != 0)))) throw Fail("STG size"); if (data == RZ && size > 4) { for (int k = 0; k < (1 << (int)(size - 4)); k++) Mov32i(TexOut + k, 0); data = TexOut; } int addr = GlobalAddress(op == 0x986, true); EmitG(0xeed8000000000000UL | (uint)data | ((ulong)addr << 8) | ((ulong)_i.Bits(40, 24) << 20) | (1UL << 45) | ((ulong)size << 48)); } // TLD (texel fetch, integer coordinates): fields as TEX; LOD mode 87-89 is 1 (.LZ) or 3 (.LL, the level after // the coordinates), offsets bit 76. Maxwell TLD takes its operands as Maxwell TEX does. Bit 78 (.MS) reads one // sample of a multisampled 2D texture, its index in the operand after the coordinates (agl_common's 2x resolve: // TLD.LZ.MS with samples 0 and 1, averaged); it is lowered for .LZ without offsets only, Maxwell's MS bit 50. void Tld() { uint dim = _i.Bits(61, 3), lod = _i.Bits(87, 3), mask = _i.Bits(72, 4); bool offsets = _i.Bit(76), ms = _i.Bit(78); if (dim == 6 || _i.Bit(77) || _i.Bit(79) || _i.Bits(81, 3) != 7 || _i.Bit(59) || lod is not 1 and not 3 || mask == 0 || (ms && (dim != 1 || lod != 1 || offsets))) throw Fail("TLD mode"); uint handle = Handle(false), bank = _i.Bits(54, 5); if (handle > 0x1fff) throw Fail("TLD descriptor offset"); if (!TexBank(bank)) throw Fail($"TLD descriptor bank {bank}"); int coords = TexCoords[dim]; int count = coords + (lod == 3 ? 1 : 0) + (offsets ? 1 : 0) + (ms ? 1 : 0); int inA = count <= 4 ? (count + 1) / 2 : 4; int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))); for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false); int next = coords, meta = TexMeta; if (lod == 3) Mov(meta++, Value(next++), false); if (offsets) Mov(meta++, Value(next++), false); if (ms) Mov(meta++, Value(next++), false); EmitG(0xdc00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20) | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | (offsets ? 1UL << 35 : 0) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49) | (ms ? 1UL << 50 : 0) | (7UL << 51) | (lod == 3 ? 1UL << 55 : 0)); TexResults(mask, false); } // TXQ (dimensions): level in Ra, descriptor offset 40-53 in bank 54-58 (2), mask 72-75, second destination // 64-71 for the third and fourth values. Maxwell TXQ with query 1 (dimension) writes them to consecutive // registers. void Txq() { uint mask = _i.Bits(72, 4), handle = Handle(false), bank = _i.Bits(54, 5); if (_i.Bits(59, 5) != 0 || mask == 0 || _i.Bits(76, 14) != 0 || _i.Bit(91) || handle > 0x1fff) throw Fail("TXQ mode"); if (!TexBank(bank)) throw Fail($"TXQ descriptor bank {bank}"); int ra = Gpr(_i.Bits(24, 8)); EmitG(0xdf48000000000000UL | TexOut | ((ulong)ra << 8) | (1UL << 22) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49)); TexResults(mask, false); } // Texture results: the components of the mask, from TexOut on, to Rd, Rd+1 and then the second destination // (bits 64-71) when there is one. FP16 results pack (c0, c1) into the first register and (c2, c3) into the // second, the first of each pair in the low half (F2F.F16 writes its half there; ISCADD merges). void TexResults(uint mask, bool f16) { int d = Dst, d2 = (int)_i.Bits(64, 8); int n = System.Numerics.BitOperations.PopCount(mask); if (!f16) { for (int k = 0; k < n; k++) { int target = d2 == RZ || k < 2 ? (d == RZ ? RZ : d + k) : d2 + (k - 2); if (target != RZ) Mov(Gpr((uint)target), TexOut + k); } return; } void pack(int target, int lo, int hi) { if (target == RZ) return; Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)lo << 20) | 246)); if (hi == RZ) Mov32i(247, 0); else Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)hi << 20) | 247)); EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)target)); } pack(d, TexOut, n > 1 ? TexOut + 1 : RZ); if (n > 2) pack(d2 != RZ ? d2 : d == RZ ? RZ : d + 1, TexOut + 2, n > 3 ? TexOut + 3 : RZ); } // PRMT Rd, a, selector (B slot), c: byte i of the result is byte (s & 7) of c:a, its sign repeated when s & 8; // the mode (bits 72-75) other than the default index mode is not lowered. Maxwell PRMT has the same meaning. void Prmt() { if (_i.Bits(72, 4) != 0 || _i.Bits(76, 15) != 0) throw Fail("PRMT mode"); int a = Plain(0, SA), sel = Plain(1, SB), c = Plain(2, SC); EmitG(0x5bc0000000000000UL | ((ulong)c << 39) | ((ulong)sel << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)); } // FLO Rd, b: the index of the highest set bit (bit 73 signed, bit 74 .SH: the shift amount, bit 63 of ~b). void Flo() { if ((_i.W2 & ~(0x600u | 0x08000000u)) != 0x000e0000) throw Fail("FLO mode"); var (b, _, inv) = Raw(1, SB); EmitG(0x5c30000000000000UL | ((ulong)b << 20) | (inv ? 1UL << 40 : 0) | ((ulong)_i.Bits(74, 1) << 41) | ((ulong)_i.Bits(73, 1) << 48) | (uint)Gpr((uint)Dst)); } // HSETP2 P, Q, a, b: compare 76-79, FTZ 80, P/Q 81-86, combined with Pc (87-90) by bits 69-70, H_AND 71; // A: reg 24, neg 72, abs 73, swizzle 74-75; b: a register (swizzle 60-61, abs 62, neg 63) or an immediate half // pair. Maxwell HSETP2 has the same fields at its own positions. void Hsetp2() { uint op = _i.Bits(0, 12); ulong source; if (op == 0x234) { if ((_i.W1 & 0x0fffff00u) != 0 || _i.Bits(64, 5) != 0) throw Fail("HSETP2 mode"); source = ((ulong)Gpr(_i.Bits(32, 8)) << 20) | ((ulong)_i.Bits(60, 2) << 28) | ((ulong)_i.Bits(62, 1) << 30) | ((ulong)_i.Bits(63, 1) << 31); } else { Mov32i(SB, _i.W1); source = (ulong)SB << 20; } EmitG(0x5d20000000000000UL | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3) | ((ulong)_i.Bits(80, 1) << 6) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | source | ((ulong)_i.Bits(76, 4) << 35) | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(72, 1) << 43) | ((ulong)_i.Bits(73, 1) << 44) | ((ulong)_i.Bits(69, 2) << 45) | ((ulong)_i.Bits(74, 2) << 47) | ((ulong)_i.Bits(71, 1) << 49)); } // P2R Rd, PR, Ra, mask / R2P PR, Ra, mask: immediate mask (bits 32-38), byte select 76-77, Ra the merge source // (P2R) or the value (R2P), as Maxwell's P2R/R2P with an immediate. void PredicateRegister(bool toRegister) { if (_i.W1 > 0x7f || (_i.W2 & ~0x3000u) != 0 || (!toRegister && _i.Bits(16, 8) != 0)) throw Fail("P2R/R2P mode"); ulong w = (toRegister ? 0x38e8000000000000UL | (uint)Gpr((uint)Dst) : 0x38f0000000000000UL) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.W1 << 20) | ((ulong)_i.Bits(76, 2) << 41); EmitG(w); } // SURED.D.op U32 (bound descriptor): data 32-39, coordinates 24-31, operation 87-89 (ADD MIN MAX INC DEC AND OR // XOR, as Maxwell's), dimension 61-63 (1D buffer or 2D), descriptor offset 40-53 in bank 54-58. Maxwell's SURED // is 1110101101011 in the top bits (0xeb58, InstTable Sured); 0xeb48 matched no instruction, so the // translator skipped every SURED: Ocarina's edge AA (the blend-weight pass ORs its weights into an image) never // wrote anything. void Sured() { uint dim = _i.Bits(61, 3), handle = Handle(true), bank = _i.Bits(54, 5); if ((_i.W2 & ~(7u << 23)) != 0x0010e000 || dim is not 1 and not 3 || _i.Bits(59, 2) != 0 || handle > 0x1fff || !TexBank(bank)) throw Fail("SURED mode"); EmitG(0xeb58000000000000UL | (uint)Gpr(_i.Bits(32, 8)) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.Bits(87, 3) << 24) | ((ulong)dim << 33) | ((ulong)handle << 36)); } // SUATOM.D.op U32 (bound descriptor): Rd = the image word's old value, fields as SURED's plus the destination and // the result predicate 81-83 (PT only). agl_technique_pfx's append counters add to word 0 of a 1D image this // way. Maxwell's SUATOM.D is 11101010 in the top bits with bit 54 clear (InstTable Suatom): operation 29-32, // dimension 33-35, descriptor 36-48, size 51-53 (U32 0). void Suatom() { uint dim = _i.Bits(61, 3), handle = Handle(true), bank = _i.Bits(54, 5); if ((_i.W2 & ~(0xfu << 23)) != 0x001ee000 || _i.Bits(87, 4) > 8 || dim > 5 || _i.Bits(59, 2) != 0 || handle > 0x1fff || !TexBank(bank)) throw Fail("SUATOM mode"); EmitG(0xea00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20) | ((ulong)_i.Bits(87, 4) << 29) | ((ulong)dim << 33) | ((ulong)handle << 36)); } // ATOMG.E.CAS U32: Rd = [Ra.64 + offset 40-63], replaced by Rc (64-71) where it equals Rb (32-39). The foliage // compute programs claim hash slots with it. Maxwell ATOM.E.CAS takes the compare value in Rb and the new value // in Rb + 1 (xlate/patches/0037-global-compare-and-swap.patch, which also makes the translator emit it), so both // go to SB and SB + 1 first. Only a zero offset is lowered. void AtomgCas() { if ((_i.W2 & ~0xffu) != 0x001ee100 || _i.Bits(40, 24) != 0 || (_i.Bits(24, 8) & 1) != 0 || _i.Bits(24, 8) == RZ) throw Fail("ATOMG.CAS mode"); Mov(SB, Gpr(_i.Bits(32, 8)), false); Mov(SB + 1, Gpr(_i.Bits(64, 8)), false); EmitG(0xeef0000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)SB << 20) | (1UL << 48)); } // REDUX URd, Ra: Ra reduced over the warp's active lanes into a uniform register. Mode (bits 78-80): SUM 3 // (modulo 2^32), MIN 4, MAX 5 (unsigned, signed with bit 73), OR 1, XOR 2. Lowered as VOTE.ANY (the active-lane // mask) and one SHFL.IDX per lane; an inactive lane contributes the operation's identity. void Redux() { uint mode = (_i.W2 & ~0x200u) >> 14; bool signedOrder = _i.Bit(73); if (_i.W1 != 0 || (_i.W2 & ~0x200u & ~(7u << 14)) != 0 || mode is not (1 or 2 or 3 or 4 or 5) || (signedOrder && mode is not (4 or 5)) || _i.Bits(12, 4) != 7) throw Fail("REDUX mode"); int t = Ur((uint)Dst), src = Gpr(_i.Bits(24, 8)); if (t == RZ) return; const int ballot = 240, lanev = 241, keep = 242; bool min = mode == 4; Emit(Unguarded(0x50d8000000000000UL | (1UL << 48) | (7UL << 45) | ((ulong)PT << 39) | ballot)); // VOTE.ANY ballot Mov32i(t, min ? 0xffffffffu : 0u); ulong lop3(int d, int a, int b, uint lut) => Unguarded(0x5be0000000000000UL | ((ulong)RZ << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)d | ((ulong)lut << 28) | (7UL << 48)); for (uint lane = 0; lane < 32; lane++) { Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | (1UL << 28) | ((ulong)lane << 20) | ((ulong)src << 8) | lanev)); if (signedOrder) Lop32i(lanev, lanev, 0x80000000, 2); // signed order as unsigned Emit(Unguarded(0x3828000000000000UL | ((ulong)lane << 20) | ((ulong)ballot << 8) | keep)); // SHR imm Lop32i(keep, keep, 1, 0); Iadd3(keep, RZ, keep, RZ, false, negB: true); // 0 or ~0 Emit(lop3(lanev, lanev, keep, min ? 0xF3u : 0xC0u)); // x | ~m, x & m switch (mode) { case 4: Emit(Unguarded(0x5c20000000000000UL | ((ulong)lanev << 20) | ((ulong)t << 8) | (uint)t | ((ulong)PT << 39))); break; // min case 5: Emit(Unguarded(0x5c20000000000000UL | ((ulong)lanev << 20) | ((ulong)t << 8) | (uint)t | (15UL << 39))); break; // max case 1: Emit(lop3(t, t, lanev, 0xFC)); break; case 2: Emit(lop3(t, t, lanev, 0x3C)); break; default: Iadd3(t, t, lanev, RZ, false); break; } } if (signedOrder) Lop32i(t, t, 0x80000000, 2); } // MATCH.ALL Pd, Rd, Ra (op 0x3a1 with no mode bits; Ocarina's NVN2 compute programs test subgroupAllEqual with it // before an aggregated shared atomic: @!Pd per-lane ATOMS, else REDUX and one): Pd = every active lane holds Ra's // value, Rd = the active-lane mask then, else 0. Lowered as VOTE.ANY (the active lanes), FLO (the highest of // them), SHFL.IDX (that lane's value), ISETP.EQ into Pd and VOTE.ALL over Pd. Only the unguarded form with a // predicate destination is taken. Bit 73 is .U64: Ra:Ra+1 is compared, the high word ANDed into Pd. void Match() { uint pd = _i.Bits(81, 3); bool wide = _i.Bit(73); if (_i.W1 != 0 || (_i.W2 & ~(7u << 17 | 1u << 9)) != 0 || _i.Bits(12, 4) != 7 || pd == PT) throw Fail("MATCH mode"); int d = Gpr((uint)Dst), src = Gpr(_i.Bits(24, 8)); if (wide && (src == RZ || (src & 1) != 0)) throw Fail("MATCH.U64 register pair"); const int ballot = 240, lead = 241, value = 242, valueHi = 243; Emit(Unguarded(0x50d8000000000000UL | (1UL << 48) | (7UL << 45) | ((ulong)PT << 39) | ballot)); // VOTE.ANY Emit(Unguarded(0x5c30000000000000UL | ((ulong)ballot << 20) | lead)); // FLO.U32 Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)lead << 20) | ((ulong)src << 8) | value)); // SHFL.IDX Emit(Unguarded(0x5b60000000000000UL | (2UL << 49) | ((ulong)PT << 39) | ((ulong)value << 20) | ((ulong)src << 8) | ((ulong)pd << 3) | PT)); // ISETP.EQ.U32 if (wide) { Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)lead << 20) | ((ulong)(src + 1) << 8) | valueHi)); // SHFL.IDX Emit(Unguarded(0x5b60000000000000UL | (2UL << 49) | ((ulong)pd << 39) | ((ulong)valueHi << 20) | ((ulong)(src + 1) << 8) | ((ulong)pd << 3) | PT)); // ISETP.EQ.U32.AND } Emit(Unguarded(0x50d8000000000000UL | ((ulong)pd << 45) | ((ulong)pd << 39) | RZ)); // VOTE.ALL if (d != RZ) Sel(d, ballot, RZ, pd, false); } // ATOMS.POPC.INC RZ, [Ra + URb + offset] (op 0xf8c, mode word 0x0d000000; the increment of Ocarina's aggregated // shared counters after MATCH and REDUX): memory goes up by the number of active lanes. With no result register // that is one ATOMS.ADD of 1 per active lane, which is what this emits; a form that wants the per-lane result (its // rank among the active lanes) is not taken. Bits 78-79 scale Ra as LDS / STS's stride does (SharedAccess): the // luminance histograms of agl_common use 3, three counters in each 16-byte bin at offsets 0, 4 and 8. void AtomsPopcInc() { uint offset = _i.Bits(40, 24), ur = _i.Bits(64, 6), stride = _i.Bits(78, 2); if ((_i.W2 & ~(0x3fu | 3u << 14)) != 0x0d000000u || _i.Bits(32, 8) != 0 || Dst != RZ || (offset & 3) != 0 || offset >= (1u << 22)) throw Fail("ATOMS.POPC.INC mode"); int addr = Gpr(_i.Bits(24, 8)); if (stride != 0 && addr != RZ) { Emit(Unguarded(0x3848000000000000UL | ((ulong)(stride + 1) << 20) | ((ulong)addr << 8) | SA)); // SHL imm addr = SA; } if (ur != 0x3f) { Iadd3(SA, addr, Ur(ur), RZ, false); addr = SA; } Mov32i(SB, 1); EmitG(0xec00000000000000UL | RZ | ((ulong)addr << 8) | ((ulong)SB << 20) | ((ulong)(offset >> 2) << 30)); // ATOMS.ADD } // VOTE Rd, Pu, Pin: ballot into Rd (URd for VOTEU), mode 72-73 (ALL, ANY, EQ) over the vector predicate of bits // 87-90, result predicate 81-83 (a uniform predicate for VOTEU). Maxwell VOTE has the same fields. VOTEU's // output goes through Pk (k its UP index) with the guest's Pk saved around it; no uniform predicate is read. void Vote(bool uniform) { if (_i.W1 != 0 || _i.Bits(72, 2) > 2 || (_i.W2 & ~0x078e0300u) != 0 || (uniform && (_i.Bits(24, 8) != 0 || _i.Bits(12, 4) != 7))) throw Fail("VOTE mode"); uint pout = _i.Bits(81, 3); int d = uniform ? Ur((uint)Dst) : Gpr((uint)Dst); ulong vote = 0x50d8000000000000UL | (uint)d | ((ulong)_i.Bits(87, 4) << 39) | ((ulong)pout << 45) | ((ulong)_i.Bits(72, 2) << 48); if (!uniform || pout == 7) { EmitG(vote); return; } uint m = 1u << (int)pout; Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RZ << 8) | USAVE)); EmitG(vote); Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RUP << 8) | RUP)); Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)USAVE << 8))); } // ATOMG.E U32: Rd = atomic op (87-90, numbered as Maxwell ATOM's: ADD MIN MAX INC DEC AND OR XOR EXCH) of // [address + offset 40-63] with Rb (32-39); the address a register pair (24-31) or a uniform pair (64-69). void Atomg(bool uniformAddress) { uint operation = _i.Bits(87, 4); int offset = (int)(_i.Bits(40, 24) << 8) >> 8; uint w2 = _i.W2 & ~(0xfu << 23); int addr; if (uniformAddress) { if (_i.Bits(24, 8) != RZ || (w2 & ~0x3fu) != 0x081ee140 || (_i.Bits(64, 6) & 1) != 0) throw Fail("ATOMG mode"); addr = Ur(_i.Bits(64, 6)); } else { if (w2 != 0x001ee100 || (_i.Bits(24, 8) & 1) != 0) throw Fail("ATOMG mode"); addr = Gpr(_i.Bits(24, 8)); } if (operation > 8 || offset < -0x80000 || offset > 0x7ffff) throw Fail("ATOMG operation/offset"); EmitG(0xed00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)addr << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20) | ((ulong)((uint)offset & 0xfffff) << 28) | (1UL << 48) | ((ulong)operation << 52)); } // RED.E (global reduction, no result): op 87-89 (ADD MIN MAX INC DEC AND OR XOR), type 73-75 (U32, S32, F32 add), // data 32-39, address a register pair (24-31) or, with Ra = RZ, a uniform pair (64-69), offset 40-63. void Red() { uint type = _i.Bits(73, 3), operation = _i.Bits(87, 3); bool ur = _i.Bits(24, 8) == RZ; uint variable = 0x03800e00u | (ur ? 0x3fu : 0u), mode = ur ? 0x0c10e100u : 0x0010e100u; int offset = (int)(_i.Bits(40, 24) << 8) >> 8; if ((_i.W2 & ~variable) != mode || type is not (0 or 1 or 3) || (type == 3 && operation != 0)) throw Fail("RED mode"); uint a = ur ? _i.Bits(64, 6) : _i.Bits(24, 8); if ((a & 1) != 0 || offset < -0x80000 || offset > 0x7ffff) throw Fail("RED address"); int addr = ur ? Ur(a) : Gpr(a); EmitG(0xebf8000000000000UL | (uint)Gpr(_i.Bits(32, 8)) | ((ulong)addr << 8) | ((ulong)type << 20) | ((ulong)operation << 23) | ((ulong)((uint)offset & 0xfffff) << 28) | (1UL << 48)); } // SHFL Rd, [Pout,] a, lane, mask: lane an immediate (53-57; 0x989, 0xf89) or a register (32-39), the // clamp/segment mask an immediate (40-52; 0x589, 0xf89) or a register (64-71), mode 58-59 (IDX UP DOWN BFLY), // lane-valid predicate 81-83. Maxwell SHFL has the same choices. void Shfl(uint op) { bool laneImm = op is 0x989 or 0xf89, maskImm = op is 0x589 or 0xf89; if ((_i.W2 & ~(0x000e0000u | (maskImm ? 0u : 0xffu))) != 0 || _i.Bits(60, 4) != 0 || (laneImm && _i.Bits(32, 8) != 0) || (!laneImm && _i.Bits(53, 5) != 0)) throw Fail("SHFL mode"); ulong lane = laneImm ? ((ulong)_i.Bits(53, 5) << 20) | (1UL << 28) : (ulong)Gpr(_i.Bits(32, 8)) << 20; ulong mask = maskImm ? ((ulong)_i.Bits(40, 13) << 34) | (1UL << 29) : (ulong)Gpr(_i.Bits(64, 8)) << 39; EmitG(0xef10000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | lane | mask | ((ulong)_i.Bits(58, 2) << 30) | ((ulong)_i.Bits(81, 3) << 48)); } // FSWZADD Rd, a, b (64-71), per-lane operation codes (bits 32-39): 0 a+b, 1 a-b, 2 b-a, 3 b. Maxwell numbers 1 // and 2 the other way round. void Fswzadd() { uint swz = _i.W1; if (swz > 0xff || _i.W2 > 255) throw Fail("FSWZADD mode"); uint maxwell = ((swz & 0x55) << 1) | ((swz & 0xaa) >> 1); EmitG(0x50f8000000000000UL | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(64, 8)) << 20) | ((ulong)maxwell << 28) | (uint)Gpr((uint)Dst)); } // ATOMS (shared) U32/S32 (bit 73): Rd = op (87-90, as Maxwell ATOMS's) of [Ra + offset 40-63] with Rb. void Atoms() { uint offset = _i.Bits(40, 24), operation = _i.Bits(87, 4); if ((_i.W2 & ~0x07800200u) != 0 || operation > 8 || (offset & 3) != 0) throw Fail("ATOMS mode"); EmitG(0xec00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20) | ((ulong)_i.Bits(73, 1) << 28) | ((ulong)(offset >> 2) << 30) | ((ulong)operation << 52)); } // SGXT Rd, a, n (immediate): the low n bits of a, sign-extended with bit 73 (else zero-extended): Maxwell BFE // from bit 0, n bits. void Sgxt() { if ((_i.W2 & ~0x200u) != 0 || _i.W1 > 32) throw Fail("SGXT mode"); EmitG(0x3800000000000000UL | ((ulong)(_i.W1 << 8) << 20) | ((ulong)_i.Bits(73, 1) << 48) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | (uint)Gpr((uint)Dst)); } // BREV Rd, b (register): Maxwell SHR.BREV by 0. void Brev() { if (_i.W2 != 0 || _i.W1 > 255) throw Fail("BREV mode"); EmitG(0x3828000000000000UL | ((ulong)Gpr(_i.Bits(32, 8)) << 8) | (1UL << 39) | (1UL << 40) | (uint)Gpr((uint)Dst)); } // HMNMX2: per half lane, both operands widened to F32 (F2F, exact), FMNMX with the predicate of bits 87-90 // (true: min), narrowed back and packed (lane 1 in the high half). void Hmnmx2() { uint op = _i.Bits(0, 12); if (_i.Bits(76, 11) != 0 || _i.Bit(91)) throw Fail("HMNMX2 mode"); var a = HalfSrc(0, SA); if (a.swz == 1) throw Fail("HMNMX2 F32 operand"); (int reg, uint swz, bool abs, bool neg) b = op == 0x840 ? (SB, 0u, false, false) : HalfSrc(1, SB); if (op == 0x840) Mov32i(SB, _i.W1); if (b.swz == 1) throw Fail("HMNMX2 F32 operand"); void widen(int target, (int reg, uint swz, bool abs, bool neg) h, int lane) { bool high = h.swz == 3 || (h.swz == 0 && lane == 1); Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)h.reg << 20) | (high ? 1UL << 41 : 0) | (h.neg ? 1UL << 45 : 0) | (h.abs ? 1UL << 49 : 0) | (uint)target)); } for (int lane = 0; lane < 2; lane++) { int t = lane == 0 ? 246 : 247; widen(244, a, lane); widen(245, b, lane); Emit(Unguarded(0x5c60000000000000UL | (245UL << 20) | (244UL << 8) | (uint)t | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42))); Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)t << 20) | (uint)t)); } EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)Dst)); } // A packed-half operand: register, swizzle (0 H1_H0, 1 F32, 2 H0_H0, 3 H1_H1, as Maxwell's), abs, neg. // A: reg 24, swizzle 74-75, abs 73, neg 72. B in bits 32-39 (swizzle 60-61, abs 62, neg 63) or, for forms 2/3/7, // 64-71 (swizzle 81-82, abs 83, neg 84). C in 64-71 (81-82, 83, 84). Immediates are a half pair as stored; // constants take swizzle/abs/neg from 60-63. (int reg, uint swz, bool abs, bool neg) HalfSrc(int which, int scratch) { if (which == 0) return (Gpr(_i.Bits(24, 8)), _i.Bits(74, 2), _i.Bit(73), _i.Bit(72)); uint form = Form; bool wide = which == 1 ? form is 4 or 5 : form is 2 or 3; if (wide) { if (form is 4 or 2) { Mov32i(scratch, _i.W1); return (scratch, 0, false, false); } var (bank, off) = Cbuf(); MovC(scratch, bank, off); return (scratch, _i.Bits(60, 2), _i.Bit(62), _i.Bit(63)); } if (form is 6 or 7) throw Fail("half operand in a uniform register"); bool upper = which == 2 || form is 2 or 3; return upper ? (Gpr(_i.Bits(64, 8)), _i.Bits(81, 2), _i.Bit(83), _i.Bit(84)) : (Gpr(_i.Bits(32, 8)), _i.Bits(60, 2), _i.Bit(62), _i.Bit(63)); } // HADD2 / HMUL2 / HFMA2 with packed-half results: saturate 77, FTZ 80, bit 76 the FMZ precision of HMUL2/HFMA2. // HFMA2.RELU (79) clamps each lane at zero where the predicate in bits 87-90 holds; only PT is lowered. Measured // on an NVIDIA GPU (tests/tensor_lowering_check.py): a negative result or -0 becomes +0, +inf stays, -inf // becomes 0, a NaN result stays NaN (0x7fff). The sum goes to TexOut and integer operations build the mask of // the half lanes to clear. Maxwell PRMT would do that in one instruction and the translator implements it // (xlate/patches/0021-prmt-byte-permute.patch, Instructions/InstEmitPermute.cs PrmtR); this keeps the integer // form, which is what was measured. // F32 results (78), the H0_NH1 source (86) and absolute values where Maxwell has none are not lowered. HADD2's // second operand is B, or C for forms 2/3. void Half(ulong maxwell) { uint op = _i.Bits(0, 12) & 0x1ff; if (op == 0x030 && _i.Bit(78)) { // HADD2.F32 Rd, -RZ, b: the compiler's half-to-float conversion of one lane of b var a0 = HalfSrc(0, SA); var bh = HalfSrc(Form is 2 or 3 ? 2 : 1, SB); if (a0.reg != RZ || bh.swz is 0 or 1 || _i.Bit(77) || _i.Bits(81, 10) != 0) throw Fail("HADD2.F32 mode"); EmitG(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)bh.reg << 20) | (bh.swz == 3 ? 1UL << 41 : 0) | (bh.neg ? 1UL << 45 : 0) | (bh.abs ? 1UL << 49 : 0) | (uint)Gpr((uint)Dst)); return; } bool relu = op == 0x031 && _i.Bit(79); if (relu && (_i.Bit(77) || _i.Bits(85, 2) != 0 || _i.Bits(87, 4) != 7)) throw Fail("HFMA2.RELU mode"); if (_i.Bit(78) || (_i.Bit(79) && !relu) || _i.Bit(91) || (op == 0x031 ? (relu ? _i.Bits(85, 2) : _i.Bits(85, 6)) != 0 : _i.Bits(81, 10) != 0)) throw Fail("half arithmetic mode"); if (op == 0x030 && _i.Bit(76)) throw Fail("HADD2 mode"); var a = HalfSrc(0, SA); var b = HalfSrc(op == 0x030 && Form is 2 or 3 ? 2 : 1, SB); uint precision = _i.Bit(76) ? 2u : _i.Bits(80, 1); ulong w = maxwell | ((ulong)a.reg << 8) | ((ulong)b.reg << 20) | ((ulong)b.swz << 28) | ((ulong)a.swz << 47) | ((ulong)_i.Bits(77, 1) << 32) | (uint)(relu ? TexOut : Gpr((uint)Dst)); if (op == 0x030) w |= (b.abs ? 1UL << 30 : 0) | (b.neg ? 1UL << 31 : 0) | (a.neg ? 1UL << 43 : 0) | (a.abs ? 1UL << 44 : 0) | ((ulong)_i.Bits(80, 1) << 39); else if (op == 0x032) w |= (b.abs ? 1UL << 30 : 0) | (a.neg != b.neg ? 1UL << 31 : 0) | (a.abs ? 1UL << 44 : 0) | ((ulong)precision << 39); else { var c = HalfSrc(2, SC); if (a.abs || b.abs || c.abs) throw Fail("HFMA2 absolute operand"); w |= (c.neg ? 1UL << 30 : 0) | (a.neg != b.neg ? 1UL << 31 : 0) | ((ulong)c.swz << 35) | ((ulong)precision << 37) | ((ulong)c.reg << 39); } if (!relu) { EmitG(w); return; } // RELU on both half lanes of the sum x at once: each half h becomes |h| (its sign bit cleared), or 0 when h is // a negative number (sign set, not NaN). |h| > 0x7c00 means NaN, so |h| + 0x3ff carries into bit 15 of its // half exactly for a NaN, and never into the other half (|h| + 0x3ff < 0x10000). A NaN keeps the payload the // host computed, with the sign cleared (the hardware's is 0x7fff). Scratch writes unguarded, Rd guarded. const int x = TexOut, y = TexOut + 1, t = TexOut + 2; Emit(Unguarded(w)); // HFMA2 x Lop32i(y, x, 0x7fff7fff, 0); // y = |h| per half Emit(Unguarded(0x1c00000000000000UL | (0x03ff03ffUL << 20) | ((ulong)y << 8) | t)); // IADD32I t = y + 0x03ff03ff Emit(Unguarded(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)t << 20) | ((ulong)x << 8) | t)); // LOP.AND t = x & ~t Emit(Unguarded(0x3828000000000000UL | (15UL << 20) | ((ulong)t << 8) | t)); // SHR t >>= 15 Lop32i(t, t, 0x00010001, 0); // bit 0 / 16: clear that half Emit(Unguarded(0x5c18000000000000UL | (16UL << 39) | (1UL << 48) | ((ulong)t << 20) | ((ulong)t << 8) | t)); // ISCADD t = (t << 16) - t // LOP.AND Rd = y & ~t (InvB bit 40), no predicate output (PT at 48) EmitG(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)t << 20) | ((ulong)y << 8) | (uint)Gpr((uint)Dst)); } // MOVM.16.MT88 Rd, Ra (0x23a): an 8x8 matrix of 16-bit elements spread over the warp (lane t holds row t/4, // columns 2(t%4) in the low half and 2(t%4)+1 in the high half) is transposed into the same layout. Element (r, c) // lives in lane 4r + c/2, half c%2, so lane t takes both halves from lanes 8(t%4) + t/8 and that + 4, half // (t/4)%2. Maxwell has no matrix move: two SHFL.IDX over the whole warp, the half picked by a shift, ISCADD to pack. // Scratch is written unguarded so every lane takes part in the shuffles; only Rd is guarded. void Movm() { if (_i.W1 != 0 || _i.W2 != 0) throw Fail("MOVM mode"); int src = Gpr(_i.Bits(24, 8)), d = Gpr((uint)Dst); int lane = TexCoord, shift = TexCoord + 1, lo = TexMeta, hi = TexMeta + 1; void shl(int r, int a, uint n) => Emit(Unguarded(0x3848000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)r)); void shr(int r, int a, uint n) => Emit(Unguarded(0x3828000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)r)); // SHFL.IDX r, value, lane register, clamp 31 (whole warp), no predicate output void shfl(int r, int value, int from) => Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)from << 20) | ((ulong)value << 8) | (uint)r)); Emit(Unguarded(0xf0c8000000000000UL | (uint)SX)); // S2R SX, SR_LANEID Lop32i(lane, SX, 3, 0); shl(lane, lane, 3); // 8(t%4) shr(shift, SX, 3); // t/8 Emit(Unguarded(0x5c10000000000000UL | ((ulong)shift << 20) | ((ulong)lane << 8) | (uint)lane)); // IADD shfl(lo, src, lane); Emit(Unguarded(0x1c00000000000000UL | (4UL << 20) | ((ulong)lane << 8) | (uint)lane)); // IADD32I +4: next row shfl(hi, src, lane); shr(shift, SX, 2); Lop32i(shift, shift, 1, 0); shl(shift, shift, 4); // 16 * ((t/4) % 2) Emit(Unguarded(0x5c28000000000000UL | ((ulong)shift << 20) | ((ulong)lo << 8) | (uint)lo)); // SHR by register Lop32i(lo, lo, 0xffff, 0); Emit(Unguarded(0x5c28000000000000UL | ((ulong)shift << 20) | ((ulong)hi << 8) | (uint)hi)); EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)lo << 20) | ((ulong)hi << 8) | (uint)d); // ISCADD (hi << 16) + lo } // POPC Rd, b (bit 63: of ~b). void Popc() { if ((_i.W2 & ~0x08000000u) != 0) throw Fail("POPC mode"); var (b, _, inv) = Raw(1, SB); EmitG(0x5c08000000000000UL | ((ulong)b << 20) | (inv ? 1UL << 40 : 0) | (uint)Gpr((uint)Dst)); } // F2FP.PACK_AB Rd, a, b: half(a) in the high 16 bits, half(b) in the low (round to nearest). With .MERGE_C // (bit 78) the high half is the low 16 bits of the C operand instead. void F2fp() { if (_i.Bits(62, 2) != 0 || _i.Bits(73, 5) != 0 || _i.Bits(79, 2) != 0 || _i.Bits(81, 10) != 0 || _i.Bit(91) || _i.Bit(72)) throw Fail("F2FP mode"); bool merge = _i.Bit(78); int a = merge ? Plain(2, SC) : Plain(0, SA), b = Plain(1, SB); if (merge) Mov(247, a, false); else Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)a << 20) | 247)); Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)b << 20) | 246)); EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)Dst)); } // SUST.P / SULD.P (formatted, bound descriptor): coordinates from Ra, data (stores) at 32-39 or destination // (loads), channel mask 72-75, dimension 61-63 in Maxwell's surface numbering (1D, BUFFER_1D, ARRAY_1D, 2D, // ARRAY_2D, 3D), descriptor offset 40-53 in bank 54-58. Strong stores (0x0010a000) are taken as plain ones. void SurfaceAccess(bool store) { uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4), handle = Handle(true), bank = _i.Bits(54, 5); uint mode = _i.W2 & ~0xf00u; bool okMode = store ? mode is 0x00100000 or 0x0010a000 : mode == 0x001e0000; if (!okMode || mask == 0 || dim > 5 || _i.Bits(59, 2) != 0 || handle > 0x1fff || !TexBank(bank)) throw Fail("surface access mode"); int coord = Gpr(_i.Bits(24, 8)); int data = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst); EmitG((store ? 0xeb28000000000000UL : 0xeb08000000000000UL) | (uint)data | ((ulong)coord << 8) | ((ulong)mask << 20) | ((ulong)dim << 33) | ((ulong)handle << 36)); } // Descriptor offset (bits 40-53, in words) of a bound texture or image. NVN2 compute programs keep their handles // in one table in the driver bank: texture slot k at start + 8k, image slot j at start + 8 * (textures + j), the // start and counts from the control section (job "nvn2_table", runtime/gfx_shader.c). It is renumbered into the // NVN1 layout the translator's host binds by: textures at 0x20 + 8k, images at 0x120 + 8j. uint Handle(bool image) { uint h = _i.Bits(40, 14); if (!_compute || _table == null) return h; int entry = ((int)h * 4 - _table[0]) / 8; if ((int)h * 4 < _table[0] || ((int)h * 4 - _table[0]) % 8 != 0 || entry >= _table[1] + _table[2]) throw Fail($"handle word {h} outside the binding table (start 0x{_table[0]:x}, {_table[1]} textures, {_table[2]} images)"); if (image != (entry >= _table[1])) throw Fail($"handle word {h} is a {(image ? "texture" : "image")} entry"); return image ? (uint)(0x120 + 8 * (entry - _table[1])) / 4 : (uint)(0x20 + 8 * entry) / 4; } // Bound texture descriptors: bank 2 for graphics stages, bank 0 for compute (NVN2 keeps the compute handles in // the driver bank). The translator leaves the bank to the host, which reads the handle table as NVN1's c2. bool TexBank(uint bank) => bank == 2 || (bank == 0 && _compute); // TLD4 (gather), 2D or ARRAY_2D: component 87-88, offset mode 76-77 (1: one offset register, 2: per-tap offsets // in two registers), depth compare 78, FP16 results 79, operands packed as TEX's. Maxwell TLD4: component 56-57, // offset mode 54-55 (same numbering), DC 50. void Tld4() { uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4), offMode = _i.Bits(76, 2); bool dc = _i.Bit(78); if (dim is not 1 and not 5 || offMode == 3 || _i.Bits(81, 3) != 7 || _i.Bit(59) || mask == 0 || _i.Bit(89)) throw Fail("TLD4 mode"); uint handle = Handle(false), bank = _i.Bits(54, 5); if (handle > 0x1fff || !TexBank(bank)) throw Fail($"TLD4 descriptor c[{bank}] 0x{handle:x}"); int coords = TexCoords[dim]; int count = coords + (int)offMode + (dc ? 1 : 0); int inA = count <= 4 ? (count + 1) / 2 : 4; int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))); for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false); int next = coords, meta = TexMeta; for (int k = 0; k < offMode; k++) Mov(meta++, Value(next++), false); if (dc) Mov(meta++, Value(next++), false); EmitG(0xc800000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20) | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49) | (dc ? 1UL << 50 : 0) | (7UL << 51) | ((ulong)offMode << 54) | ((ulong)_i.Bits(87, 2) << 56)); TexResults(mask, _i.Bit(79)); } // PLOP3.LUT Pu, Pv, a, b, c: table in bits 64-66 and 72-76, inputs a 87-90, b 77-80, c 68-71 (index | negate << 3), // outputs 81-83 and 84-86 (the second with the table in bits 16-23). Each input becomes 0 or ~0 (SEL), LOP3 combines // them and ISETP.NE sets the output. Bit 67 (c a uniform predicate) is not lowered for the vector form. void Plop3() { uint lut = _i.Bits(64, 3) | _i.Bits(72, 5) << 3, t1 = _i.Bits(81, 3), t2 = _i.Bits(84, 3), lut2 = _i.Bits(16, 8); const int ones = 240, ra = 241, rb = 242, rc = 243, r1 = 244, r2 = 245; Mov32i(ones, 0xffffffff); Sel(ra, ones, RZ, _i.Bits(87, 4), false); Sel(rb, ones, RZ, _i.Bits(77, 4), false); uint c = _i.Bits(68, 4); if (_i.Bit(67) && !_uniform) { // c is UPn: 0 - ((RUP >> n) & 1) gives 0 or ~0 straight from the uniform predicate state; UPT is ~0. if ((c & 7) == 7) Mov(rc, ones, false); else { Emit(Unguarded(0x3828000000000000UL | ((ulong)(c & 7) << 20) | ((ulong)RUP << 8) | rc)); // SHR imm Lop32i(rc, rc, 1, 0); Iadd3(rc, RZ, rc, RZ, false, negB: true); } if ((c & 8) != 0) Lop32i(rc, rc, 0xffffffff, 2); } else Sel(rc, ones, RZ, c, false); ulong lop3(int dst, uint table) => Unguarded(0x5be0000000000000UL | ((ulong)rc << 39) | ((ulong)rb << 20) | ((ulong)ra << 8) | (uint)dst | ((ulong)table << 28) | (7UL << 48)); Emit(lop3(r1, lut)); if (t2 != 7) Emit(lop3(r2, lut2)); ulong ne(uint p, int r) => 0x5b60000000000000UL | (5UL << 49) | ((ulong)PT << 39) | ((ulong)RZ << 20) | ((ulong)r << 8) | ((ulong)p << 3) | PT; if (t1 != 7) EmitG(ne(t1, r1)); if (t2 != 7) EmitG(ne(t2, r2)); } // LDS / STS: [Ra << (stride ? stride + 1 : 0) (+ a uniform register when bit 91 is set: bits 32-39 for loads, // 64-71 for stores) + offset 40-63], size 73-75 (as Maxwell's), stride 78-79. STS data at 32-39. void SharedAccess(uint op) { bool store = op != 0x984; uint width = _i.Bits(73, 3), stride = _i.Bits(78, 2); bool uniformBase = op is 0x984 or 0x988 && _i.Bit(91); uint ignored = uniformBase ? (op == 0x988 ? 0x080000ffu : 0x08000000u) : 0u; if (width > 6 || (_i.W2 & ~ignored) != (width << 9 | stride << 14) || (op == 0x984 && !uniformBase && _i.Bits(32, 8) != 0) || (op == 0x988 && !uniformBase)) throw Fail("shared memory addressing mode"); int addr = Gpr(_i.Bits(24, 8)); if (stride != 0 && addr != RZ) { Emit(Unguarded(0x3848000000000000UL | ((ulong)(stride + 1) << 20) | ((ulong)addr << 8) | SA)); // SHL imm addr = SA; } if (uniformBase) { int u = Ur(_i.Bits(op == 0x988 ? 64 : 32, 8)); if (u != RZ) { Iadd3(SA, addr, u, RZ, false); addr = SA; } } int data = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst); EmitG((store ? 0xef58000000000000UL : 0xef48000000000000UL) | (uint)data | ((ulong)addr << 8) | ((ulong)_i.Bits(40, 24) << 20) | ((ulong)width << 48)); } // BAR.SYNC 0 for the whole workgroup (bit 80, .DEFER_BLOCKING, needs nothing here). void Bar() { if (_i.W1 != 0 || (_i.W2 & ~0x10000u) != 0 || _i.Bits(12, 4) != 7) throw Fail("BAR mode"); Emit(Unguarded(0xf0a8000000000000UL | (1UL << 43) | (1UL << 44) | (7UL << 39))); } // MEMBAR: Maxwell MEMBAR.GL whatever the scope. A barrier at least as wide as the one asked for orders the same // accesses; the decoder reads MEMBAR's level from bits 8-9 (CTA 0, GL 1, SYS 2), and the SPIR-V it makes is a // device-scope memory barrier for GL and SYS alike. void Membar() => Emit(Unguarded(0xef98000000000000UL | (1UL << 8))); // ISETP.EX: the high-word half of a 64-bit compare. The low words' result is the predicate of bits 68-71; EQ is // (high equal AND low), NE (high different OR low), an ordering (high strictly ordered OR (high equal AND low)), // signed with bit 73. The result is combined with Pc (87-90) by the operation in 74-75 into P (81-83) and Q // (84-86), as ISETP does: Maxwell ISETPs into a saved scratch predicate (P6, or the highest one the instruction // does not name), then PSETP. void SetpExtended() { uint cmp = _i.Bits(76, 3), combine = _i.Bits(74, 2), low = _i.Bits(68, 4); if (cmp is 0 or 7 || combine == 3) throw Fail("ISETP.EX mode"); var (a, _, _) = Raw(0, SA); var (b, _, _) = Raw(1, SB); // the scratch predicate: P6 unless the instruction names it (low result, Pc, guard or a destination) int t = 6; while (t >= 0 && (t == (int)(low & 7) || t == (int)_i.Bits(87, 3) || t == (int)_i.Bits(12, 3) || t == (int)_i.Bits(81, 3) || t == (int)_i.Bits(84, 3))) t--; if (t < 0) throw Fail("ISETP.EX: no free scratch predicate"); Emit(Unguarded(0x38e8000000000000UL | (1UL << (20 + t)) | ((ulong)RZ << 8) | SX)); // P2R: save P6 ulong isetp(uint c, uint input, uint bop, bool signedCmp) => 0x5b60000000000000UL | ((ulong)t << 3) | PT | ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)(input & 7) << 39) | ((ulong)(input >> 3) << 42) | ((ulong)bop << 45) | (signedCmp ? 1UL << 48 : 0) | ((ulong)c << 49); bool sg = _i.Bit(73); const uint And = 0, Or = 1, Lt = 1, Eq = 2, Gt = 4, Ne = 5; if (cmp == Eq) EmitG(isetp(Eq, low, And, sg)); else if (cmp == Ne) EmitG(isetp(Ne, low, Or, sg)); else { EmitG(isetp(Eq, low, And, sg)); EmitG(isetp(cmp <= 3 ? Lt : Gt, (uint)t, Or, sg)); } // PSETP P, Q = (P6 AND PT) combine Pc EmitG(0x5090000000000000UL | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3) | ((ulong)t << 12) | ((ulong)PT << 29) | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)combine << 45)); Emit(Unguarded(0x38f0000000000000UL | (1UL << (20 + t)) | ((ulong)SX << 8))); // R2P: restore P6 } // UBREV URd, URb: BREV on uniform registers (bit 91 marks the uniform datapath). void Ubrev() { if ((_i.W2 & ~0x08000000u) != 0 || _i.W1 > 255) throw Fail("UBREV mode"); int d = Ur((uint)Dst); if (d == RZ) return; EmitG(0x3828000000000000UL | ((ulong)Ur(_i.Bits(32, 8)) << 8) | (1UL << 39) | (1UL << 40) | (uint)d); // SHR.BREV by 0 } // UPOPC URd, URb: POPC on uniform registers, lowered through POPC (0x309) with register fields naming uniform // registers (as UniformAlu does). void Upopc() { if ((_i.W2 & ~0x08000000u) != 0 || _i.W1 > 255) throw Fail("UPOPC mode"); Insn saved = _i; _i.Lo = (_i.Lo & ~0xfffUL) | 0x309; _i.Hi &= ~(1UL << 27); _uniform = true; try { Popc(); } finally { _uniform = false; _i = saved; } } // UP2UR URd, UPR, URa, mask: P2R over the uniform predicates, byte 0 only: URd = (URa & ~mask) | (UP & mask), // the mask in bits 32-38. RUP holds UPk in bit k. void Up2ur() { if (_i.W1 > 0x7f || (_i.W2 & ~0x08000000u) != 0) throw Fail("UP2UR mode"); int d = Ur((uint)Dst), a = Ur(_i.Bits(24, 8)); if (d == RZ) return; uint mask = _i.W1; Lop32i(SX, RUP, mask, 0); // AND if (a == RZ) { Mov(d, SX); return; } Lop32i(SA, a, ~mask, 0); Iadd3(d, SA, SX, RZ, true); // disjoint bits: the sum is the OR } // F2IP.U8.F32 Rd, Ra, Rb, RZ (opcode 0x243): two floats converted to unsigned bytes and packed into the low // half of Rd. Rb's byte goes in bits 0-7, Ra's byte in bits 8-15, and bits 16-31 are zero. With Ra = RZ only // Rb's byte is written. W1 holds only Rb (bits 32-39), and the C operand (bits 64-71) must be RZ. Rounding is // in bits 78-79, using the same codes as Maxwell's F2I RoundMode (bits 39-40): // W2 0x0000c4ff: bits 78-79 = 3, round toward zero; // W2 0x000004ff: bits 78-79 = 0, round to nearest even. The compiler uses this form for unorm values it has // already multiplied by 255. // Each byte goes through NaN -> 0, a float clamp to [0, 255], then F2I.U32.F32 with that rounding. The clamp // has to be explicit. The translator's F2I (Instructions/InstEmitConversion.cs, EmitF2I) clamps only // U16/S16 destinations, and IDstFmt has no U8, so a plain F2I.U8 lets values above 255 through. It also feeds // NaN to FMax, whose SPIR-V result is undefined. void F2ipByte() { if ((_i.W1 & ~0xffu) != 0 || (_i.W2 != 0x0000c4ff && _i.W2 != 0x000004ff)) throw Fail("packed float-to-byte mode"); int a = Gpr(_i.Bits(24, 8)), b = Gpr(_i.Bits(32, 8)), d = Gpr((uint)Dst); if (d == RZ) return; uint round = _i.Bits(78, 2); // Convert src to an integer 0-255. Steps 1-4 work in scratch register t without a guard. Step 5 writes dst // and is guarded only when dst is the guest register. void ToByte(int src, int t, int dst, bool guarded) { // FSET.NUM.AND t = (src is not NaN) ? 0xffffffff : 0 (FComp 7 in 48-51, source predicate PT in 39-41, // BVal clear so the result is an integer mask) Emit(Unguarded(0x5800000000000000UL | (7UL << 48) | ((ulong)PT << 39) | ((ulong)src << 20) | ((ulong)src << 8) | (uint)t)); // LOP.AND t = src & t: NaN becomes +0.0 (destination predicate PT in 48-50) Emit(Unguarded(0x5c40000000000000UL | ((ulong)PT << 48) | ((ulong)t << 20) | ((ulong)src << 8) | (uint)t)); // FMNMX t = max(t, 0.0): immediate form, predicate !PT (39-42) selects the maximum Emit(Unguarded(0x3860000000000000UL | ((ulong)PT << 39) | (1UL << 42) | ((ulong)t << 8) | (uint)t)); // FMNMX t = min(t, 255.0): 255.0f is 0x437f0000, whose top 20 bits 0x437f0 fit the immediate field Emit(Unguarded(0x3860000000000000UL | (0x437f0UL << 20) | ((ulong)PT << 39) | ((ulong)t << 8) | (uint)t)); // F2I.U32.F32 dst = t, with the instruction's rounding (IDstFmt U32 in 8-9, SrcFmt F32 in 10-11, FTZ 44) Put(0x5cb0000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)round << 39) | (1UL << 44) | ((ulong)t << 20) | (uint)dst, guarded); } if (a == RZ) { // Single byte: convert straight into Rd. Steps 1-4 read b and write only SA, so this is safe when d == b. ToByte(b, SA, d, true); return; } // Both bytes go to scratch before Rd is written, so d may alias a or b. ToByte(b, SA, SA, false); ToByte(a, SB, SB, false); // ISCADD Rd = (SB << 8) + SA. Both values are 0-255, so the sum equals the OR of the two bytes. EmitG(0x5c18000000000000UL | (8UL << 39) | ((ulong)SA << 20) | ((ulong)SB << 8) | (uint)d); } // TXD (0xb6c): a sample with explicit derivatives, 2D or ARRAY_2D, bound descriptor. Fields as TEX's: Rd 16-23, // Ra 24-31 (x, y, then for ARRAY_2D the layer as an integer in its low 16 bits), Rb 32-39 (the four derivatives, // interleaved: dx.x, dy.x, dx.y, dy.y), descriptor word offset 40-53 in bank 54-58, dimension 61-63, second // destination 64-71, mask 72-75, bit 90 no-dependency. Offsets (76), FP16 results (79) and the bindless forms // (59-60) are not lowered. Bits 81-84 are set in every TXD seen (81-83: no sparse predicate). // Maxwell TXD (0xde00) takes the coordinates and the layer the same way in Ra.., but the translator's EmitTxd reads Rb.. // as dPdx (2 values) then dPdy (2 values), so the derivatives are reordered on the way into TexMeta. // With no second destination the third and fourth channels are dropped (SM70+ register pairs), so the mask // handed to Maxwell keeps only its two lowest channels and TexResults writes Rd, Rd+1; otherwise TexResults // writes Rd, Rd+1 and then the second destination, which need not follow Rd. void Txd() { uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4); if ((_i.W2 & ~(0xfffu | 1u << 26)) != 0x001e0000 || _i.Bits(59, 2) != 0 || dim is not 1 and not 5 || mask == 0) throw Fail("TXD mode"); uint handle = Handle(false), bank = _i.Bits(54, 5); if (handle > 0x1fff) throw Fail("TXD descriptor offset"); if (!TexBank(bank)) throw Fail($"TXD descriptor bank {bank}"); int d2 = (int)_i.Bits(64, 8); if (Dst == RZ && d2 == RZ) return; // nothing is written, and sampling has no side effects if (d2 == RZ) { uint upper = mask & (mask - 1); // drop the lowest channel... upper &= upper - 1; // ...and the next: what is left has no destination mask ^= upper; } int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); int coords = dim == 5 ? 3 : 2; for (int k = 0; k < coords; k++) Mov(TexCoord + k, ra == RZ ? RZ : Gpr((uint)(ra + k)), false); // SM86 order dx.x, dy.x, dx.y, dy.y; the translator's Maxwell order dx.x, dx.y, dy.x, dy.y. int[] order = { 0, 2, 1, 3 }; for (int k = 0; k < 4; k++) Mov(TexMeta + k, rb == RZ ? RZ : Gpr((uint)(rb + order[k])), false); EmitG(0xde00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20) | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49) | (7UL << 51)); TexResults(mask, false); } // LDSM.16.M88.{1,2,4} and LDSM.16.MT88.{1,2,4} (ldmatrix): count = 1 << bits 72-73 (1, 2 or 4) 8x8 matrices of // 16-bit elements from shared memory into Rd..Rd+count-1 (Rd aligned to count). Lanes 8m..8m+7 hold, in Ra, the // byte addresses of rows 0-7 of matrix m; bit 91 adds a uniform register (bits 32-39, as in SharedAccess) and every // row adds the immediate offset 40-63. Without transpose (bit 78 clear) lane t receives in Rd+m the 32 bits at row // t/4, bytes 4(t%4)..4(t%4)+3 of matrix m. With transpose (.MT88) it receives column t/4 of row 2(t%4) in the low // half and of row 2(t%4)+1 in the high half. // Maxwell has no matrix load, so each lane fetches the row address it needs from the lane that holds it // (SHFL.IDX across the whole warp), adds its own column and loads with LDS: one LDS.32 per matrix, or for .MT88 two // LDS.U16 per matrix joined by ISCADD (high << 16) + low. Address arithmetic and the shuffles are unguarded // scratch writes so every lane takes part in the shuffles (the hardware needs the whole warp too); the loads and // the writes to Rd are guarded by the instruction's predicate. // Scratch: SX lane id, then the column bytes (non-transposed); TexCoord the source lane of row 0 (or of row // 2(t%4)), TexCoord+1 the lane of later matrices (or the column bytes for .MT88), TexCoord+2 the lane for .MT88; // SA the fetched row address; SC the base address when it needs a copy; TexMeta/TexMeta+1 the two halves. void Ldsm() { uint sizeCode = _i.Bits(72, 2); bool transpose = _i.Bit(78), uniformOffset = _i.Bit(91); // Allowed in word 2: size 72-73 (0x300), transpose 78 (0x4000), uniform offset 91 (0x08000000). if (sizeCode == 3 || (_i.W2 & ~0x08004300u) != 0 || (!uniformOffset && _i.Bits(32, 8) != 0)) throw Fail("LDSM mode"); int count = 1 << (int)sizeCode; if (Dst == RZ) return; // a load into RZ has no visible effect if (Dst % count != 0 || Dst + count > RZ) throw Fail("LDSM destination alignment"); var dst = new int[count]; for (int m = 0; m < count; m++) dst[m] = Gpr((uint)(Dst + m)); int addr = Gpr(_i.Bits(24, 8)); if (uniformOffset) { int u = Ur(_i.Bits(32, 8)); if (u != RZ) { Iadd3(SC, addr, u, RZ, false); addr = SC; } } // A destination that is also the base would be overwritten before the later matrices fetch their rows. if (addr != RZ && Array.IndexOf(dst, addr) >= 0) { Mov(SC, addr, false); addr = SC; } ulong offset = (ulong)_i.Bits(40, 24) << 20; // Unguarded Maxwell helpers: SHR/SHL by an immediate (unsigned), IADD32I, and SHFL.IDX of the base address // from a lane held in a register, clamp 31 with segment mask 0 (the whole warp), valid predicate to PT. void shr(int d, int a, int n) => Emit(Unguarded(0x3828000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)d)); void shl(int d, int a, int n) => Emit(Unguarded(0x3848000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)d)); void addImm(int d, int a, uint imm) => Emit(Unguarded(0x1c00000000000000UL | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)d)); void fetchRow(int d, int lane) => Emit(Unguarded(0xef10000000000000UL | (uint)d | ((ulong)addr << 8) | ((ulong)lane << 20) | (0x1fUL << 34) | (1UL << 29) | (7UL << 48))); // LDS d, [a + offset], size as Maxwell's LsSize2 (2 U16, 4 B32), guarded. void lds(int d, int a, ulong size) => EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | offset | (size << 48)); Emit(Unguarded(0xf0c8000000000000UL | SX)); // S2R SX, SR_LANEID (0) if (!transpose) { shr(TexCoord, SX, 2); // lane holding row t/4 of matrix 0 Lop32i(SX, SX, 3, 0); // t & 3 (LOP32I.AND) shl(SX, SX, 2); // column bytes 4(t%4) for (int m = 0; m < count; m++) { int lane = TexCoord; if (m != 0) { addImm(TexCoord + 1, TexCoord, (uint)(8 * m)); lane = TexCoord + 1; } fetchRow(SA, lane); Iadd3(SA, SA, SX, RZ, false); lds(dst[m], SA, 4); // LDS.32 } return; } Lop32i(TexCoord, SX, 3, 0); shl(TexCoord, TexCoord, 1); // lane holding row 2(t%4) of matrix 0 shr(TexCoord + 1, SX, 2); shl(TexCoord + 1, TexCoord + 1, 1); // column bytes 2(t/4) for (int m = 0; m < count; m++) { for (int half = 0; half < 2; half++) { int lane = TexCoord; uint step = (uint)(8 * m + half); if (step != 0) { addImm(TexCoord + 2, TexCoord, step); lane = TexCoord + 2; } fetchRow(SA, lane); Iadd3(SA, SA, TexCoord + 1, RZ, false); lds(TexMeta + half, SA, 2); // LDS.U16 (zero-extended) } // ISCADD Rd+m = (high << 16) + low EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)TexMeta << 20) | ((ulong)(TexMeta + 1) << 8) | (uint)dst[m]); } } // HMMA.1688 / HMMA.16816 Rd, Ra, Rb, Rc with F16 inputs: a warp-wide matrix multiply-accumulate D(16x8) = // A(16xK) * B(Kx8) + C(16x8), K = 8, or 16 when bit 75 is set. Bit 76 selects F32 accumulators and results // (.F32: C and D are four registers each) instead of F16 ones (.F16: a register pair). Fields: D 16-23, A 24-31 // (K/4 registers), B 32-39 (K/8 registers), C 64-71 (RZ means a zero accumulator). With g = lane / 4 and // q = lane % 4, each lane holds (the lower-numbered column or k in the low half of every packed register): // A register 2j + r: row g + 8r, k 8j + 2q and 8j + 2q + 1 (j is 0 only when K = 8) // B register j: k 8j + 2q and 8j + 2q + 1, column g // C / D, .F16: register r holds row g + 8r, columns 2q and 2q + 1 // C / D, .F32: register 2r + c holds row g + 8r, column 2q + c // Maxwell has no tensor instructions, so the product is expanded per lane. For each k pair (2p, 2p + 1) the lane // fetches the A word of its own rows from lane 4g + p % 4 and, for each of its two columns n = 2q + c, the B word // from lane 4n + p % 4 (SHFL.IDX), widens the halves to F32 (F2F.F32.F16, exact) and accumulates with FFMA in // F32 (an F16 x F16 product is exact in F32, so the step rounds once whether or not the host fuses it). // The .F16 form rounds to F16 once, after all K products, as the hardware does: it does not round between the // two eight-k halves of HMMA.16816. Measured on an NVIDIA GPU (tests/tensor_lowering_check.py compares this // lowering with those results): a first-half sum of 65536 that the second half cancels gives 0, which an F16 // rounding after the first half turns into +inf, and 1 + 2^-11 + 2^-11 split over the halves gives 1 + 2^-10, // which that rounding turns into 1. The F16 results are then packed as F2fp in this file does (F2F.F16 writes // the low half and zeroes the high half, ISCADD merges the column pair). // The work runs unguarded because every lane must supply its fragments to the shuffles; only the result writes // take the instruction's guard. Scratch R239-R252 (this file's scratch range, see the header); the guest operands // are never there (guest and uniform registers end below R237, spill temporaries and uniform registers are R156-R235). void Hmma() { bool k16 = _i.Bit(75), f32 = _i.Bit(76); uint d = (uint)Dst, a = _i.Bits(24, 8), b = _i.Bits(32, 8), c = _i.Bits(64, 8); int aRegs = k16 ? 4 : 2, bRegs = k16 ? 2 : 1, cdRegs = f32 ? 4 : 2; // A fragment base aligned to its size and the whole fragment below RZ (RZ itself reads as zeros). bool fits(uint reg, int n) => reg == RZ || (reg % (uint)n == 0 && reg + (uint)n <= RZ); // Word 1 above B, word 2 other than C, the shape (75) and the accumulator type (76), and word 3 bits 96-104 / // 126-127 select modes (bit 74 another shape, HMMA.1684; other input types) that are not lowered. if ((_i.W1 & ~0xffu) != 0 || (_i.W2 & ~0x18ffu) != 0 || ((uint)(_i.Hi >> 32) & 0xc00001ffu) != 0 || !fits(d, cdRegs) || !fits(a, aRegs) || !fits(b, bRegs) || !fits(c, cdRegs)) throw Fail("HMMA mode"); if (d == RZ) return; // no result and no other effect // Register k of a fragment (RZ stays RZ); Gpr maps spilled guest registers to this instruction's temporaries. int part(uint reg, int k) => reg == RZ ? RZ : Gpr(reg + (uint)k); const int laneA = 239; // 4g: the lane of A's row group for k pair 0 const int laneB = 240; // 8q: the lane holding B column 2q for k pair 0 const int lane = 241; // the lane of the current shuffle const int word = 242; // the shuffled packed-half word const int aw = 243; // 243-246: A element (row g + 8r, k 2p + h) at aw + 2r + h, F32 const int bw = 247; // 247-248: B element (k 2p + h, current column) at bw + h, F32 const int acc = 249; // 249-252: accumulator (row g + 8r, column 2q + c) at acc + 2r + c, F32 void widen(int target, int src, bool high) => // F2F.F32.F16 target, src.H0 / .H1 Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)src << 20) | (high ? 1UL << 41 : 0) | (uint)target)); void narrow(int t) => // F2F.F16.F32 t, t (round to nearest; high half zero) Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)t << 20) | (uint)t)); void shuffle(int target, int src, int laneReg) => // SHFL.IDX PT, target, src, laneReg, 0x1f (as Redux) Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)laneReg << 20) | ((ulong)src << 8) | (uint)target)); int laneAt(int baseLane, uint offset) // baseLane + offset in `lane` (IADD32I), or baseLane itself { if (offset == 0) return baseLane; Emit(Unguarded(0x1c00000000000000UL | ((ulong)offset << 20) | ((ulong)baseLane << 8) | lane)); return lane; } // Accumulators from C, or zero: accumulator 2r + col is register 2r + col of an .F32 C, half col of register // r of an .F16 C. for (int k = 0; k < 4; k++) { if (c == RZ) Mov32i(acc + k, 0); else if (f32) Mov(acc + k, part(c, k), false); else widen(acc + k, part(c, k >> 1), (k & 1) == 1); } // Lane bases: 8q first (it needs the raw lane id), then 4g. Put(0xf0c8000000000000UL | laneA, false); // S2R SR_LANEID Lop32i(laneB, laneA, 3, 0); // q Emit(Unguarded(0x3848000000000000UL | (3UL << 20) | ((ulong)laneB << 8) | laneB)); // SHL imm: 8q Lop32i(laneA, laneA, 0x1c, 0); // 4g int pairs = k16 ? 8 : 4; for (int p = 0; p < pairs; p++) { uint pq = (uint)(p & 3); // the source lanes' q for this k pair int j = p >> 2; // which 8-k block of the fragments int srcA = laneAt(laneA, pq); for (int r = 0; r < 2; r++) { shuffle(word, part(a, 2 * j + r), srcA); widen(aw + 2 * r, word, false); widen(aw + 2 * r + 1, word, true); } for (int col = 0; col < 2; col++) { shuffle(word, part(b, j), laneAt(laneB, 4u * (uint)col + pq)); widen(bw, word, false); widen(bw + 1, word, true); for (int r = 0; r < 2; r++) for (int h = 0; h < 2; h++) { int t = acc + 2 * r + col; // FFMA t = A(r, 2p + h) * B(2p + h, col) + t Emit(Unguarded(0x5980000000000000UL | ((ulong)t << 39) | ((ulong)(bw + h) << 20) | ((ulong)(aw + 2 * r + h) << 8) | (uint)t)); } } } // .F32: D register 2r + col is accumulator 2r + col. if (f32) { for (int k = 0; k < 4; k++) Mov(part(d, k), acc + k); return; } // .F16: one rounding of each sum to F16, then D register r = column 2q + 1 in the high half, column 2q in the // low half (ISCADD hi << 16 + lo). for (int t = acc; t < acc + 4; t++) narrow(t); for (int r = 0; r < 2; r++) EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)(acc + 2 * r) << 20) | ((ulong)(acc + 2 * r + 1) << 8) | (uint)part(d, r)); } // UFLO URd, URb: FLO on uniform registers (bit 74 .SH, bit 63 of ~b). void Uflo() { if ((_i.W2 & ~(0x400u | 0x08000000u)) != 0x000e0000) throw Fail("UFLO mode"); int d = Ur((uint)Dst); if (d == RZ) return; EmitG(0x5c30000000000000UL | ((ulong)Ur(_i.Bits(32, 8)) << 20) | (_i.Bit(63) ? 1UL << 40 : 0) | ((ulong)_i.Bits(74, 1) << 41) | (uint)d); } // I2F: source signed (bit 74) and size (84-85: 8/16/32 bits, its byte or half in 60-61), destination size // (75-76: F16, F32, F64), rounding 78-79. Maxwell I2F: ISrcFmt (size code in 10-11, signed in bit 13), ByteSel, DstFmt. void I2f(uint op) { const uint known = 1u << 10 | 3u << 11 | 3u << 14 | 3u << 20; uint w2 = _i.W2 & ~(op == 0xd06 ? 0x08000000u : 0u); uint src = _i.Bits(84, 2), dst = _i.Bits(75, 2), sel = _i.Bits(60, 2); if ((w2 & ~known) != 0 || src == 3 || dst == 0 || (op != 0xb06 && (_i.W1 & 0xcfffff00u) != 0)) throw Fail("I2F format"); int b = op == 0xb06 ? Plain(1, SB) : op == 0xd06 ? Ur(_i.Bits(32, 8)) : Gpr(_i.Bits(32, 8)); uint bytesel = src == 0 ? sel : src == 1 ? sel * 2 : 0; EmitG(0x5cb8000000000000UL | ((ulong)dst << 8) | ((ulong)src << 10) | (_i.Bit(74) ? 1UL << 13 : 0) | ((ulong)b << 20) | ((ulong)bytesel << 41) | ((ulong)_i.Bits(78, 2) << 39) | (uint)Gpr((uint)Dst)); } // STL / LDL (local memory) at [Ra + offset 40-63], size 73-75 (U8 S8 U16 S16 32 64 128); 64 and 128 bits go as // 32-bit words. STL data at 32-39. With bit 91 the address adds a uniform register URb (STL 0x987, LDL 0x983 in // that form): a store's at 64-69, a load's at 32-37 (its B field; Ocarina's mesh programs write that register just // before the access). Ra + URb goes to a scratch register first, or URb's register is the address when Ra is RZ. void LocalAccess(bool store) { uint size = _i.Bits(73, 3), allowed = store ? 0xe00u : 0x200e00u; bool uniform = _i.Bit(91); uint w2 = uniform ? _i.W2 & ~(store ? 0x0800003fu : 0x08000000u) : _i.W2; uint bField = _i.Bits(32, 8) & (uniform && !store ? 0xc0u : 0xffu); if (size > 6 || (w2 & ~allowed) != 0x00100000 || (!store && bField != 0) || (store && _i.Bits(16, 8) != 0)) throw Fail("local memory mode"); int addr = Gpr(_i.Bits(24, 8)), value = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst); if (uniform) { int ub = Ur(store ? _i.Bits(64, 6) : _i.Bits(32, 6)); if (ub != RZ && addr == RZ) addr = ub; else if (ub != RZ) { Iadd3(SX, addr, ub, RZ, false); addr = SX; } } uint off = _i.Bits(40, 24), words = size == 6 ? 4u : size == 5 ? 2u : 1u; if (!store && words > 1 && addr != RZ && addr >= value && addr < value + words) { Mov(SX, addr, false); addr = SX; } for (uint k = 0; k < words; k++) EmitG((store ? 0xef50000000000000UL : 0xef40000000000000UL) | (uint)(value == RZ ? RZ : value + (int)k) | ((ulong)addr << 8) | ((ulong)((off + 4 * k) & 0xffffff) << 20) | ((ulong)(size < 4 ? size : 4) << 48)); } // IPA with the attribute indexed by a uniform register (bits 24-31): Maxwell IPA.IDX.PASS with the whole byte // address in Ra. As Ipa: the value times 1/w, which the program multiplies by w itself. void IpaIndexed() { if (_i.Bits(32, 8) != RZ || (_i.Bits(76, 4) != 0 && _i.Bits(76, 4) != 4) || _i.Bits(81, 3) != 7) throw Fail("indexed IPA mode"); int idx = Ur(_i.Bits(24, 8)), d = Gpr((uint)Dst); uint attr = _i.Bits(64, 8); if (idx == RZ) { EmitG(0xe000000000000000UL | ((ulong)RZ << 8) | ((ulong)RZ << 20) | ((ulong)attr << 30) | ((ulong)_i.Bits(78, 1) << 55) | ((ulong)PT << 47) | (uint)d); return; } Emit(Unguarded(0x1c00000000000000UL | ((ulong)(attr * 4) << 20) | ((ulong)idx << 8) | SX)); // IADD32I EmitG(0xe000000000000000UL | ((ulong)SX << 8) | ((ulong)RZ << 20) | (1UL << 38) | ((ulong)_i.Bits(78, 1) << 55) | ((ulong)PT << 47) | (uint)d); } // ULDC URd, c[bank][URa + offset]: Maxwell LDC with a register index; width 73-75 as LDC's. void UldcIndexed() { uint width = _i.Bits(73, 3); int d = Ur((uint)Dst), idx = Ur(_i.Bits(24, 8)), bank = Bank(_i.Bits(54, 5)); if (width > 5 || (_i.W2 & ~0xe00u) != 0x08000000 || bank >= 18 || _i.Bits(62, 2) != 0) throw Fail("indexed ULDC mode"); if (d == RZ) return; EmitG(0xef90000000000000UL | (uint)d | ((ulong)idx << 8) | ((ulong)_i.Bits(38, 16) << 20) | ((ulong)bank << 36) | ((ulong)width << 48)); } // BRX Ra / BRXU URa with the program start as base (the offset makes the target Ra itself): a switch. The // compiler loads Ra from a jump table in c1 of native byte addresses, LDC Ra, c[0x1][Rx + base], after // clamping the index (IMNMX.U32 Ry, Ry, bound; SHL Rx, Ry, 2). The translator's BRX compares the register with // the lowered addresses, so it cannot take native ones; each table entry becomes a compare and branch instead: // IADD32I.CC RZ, Ra, -target; BRA CC.EQ to the target's lowered code, the last entry also unconditionally. void Brx(bool uniform) { if (_i.Bits(12, 4) != 7) throw Fail("predicated BRX"); if ((_i.W2 & ~0x3ffffu) != (uniform ? 0x0b800000u : 0x03800000u)) throw Fail("BRX mode"); long raw = (long)(_i.W1 | ((ulong)(_i.W2 & 0x3ffff) << 32)); if ((long)_index * 16 + 16 + ((raw << 14) >> 14) != 0) throw Fail("BRX relative to a base other than the program start"); uint src = _i.Bits(24, 8); var targets = BrxTargets(src, uniform); int reg = uniform ? Ur(src) : Gpr(src); foreach (uint t in targets) { Emit(Unguarded(0x1c00000000000000UL | (1UL << 52) | ((ulong)(uint)(-(int)t) << 20) | ((ulong)reg << 8) | RZ)); _branches.Add((_out.Count, (int)(t / 16))); Emit(0xe240000000000000UL | ((ulong)PT << 16) | 2); // BRA CC.EQ } _branches.Add((_out.Count, (int)(targets[^1] / 16))); Emit(0xe24000000000000fUL | ((ulong)PT << 16)); } // The jump table behind a BRX register: the LDC (or, for BRXU, indexed ULDC) of bank 1 that last wrote it in // the straight-line code before the BRX, and its entry count from the IMNMX bound of the index feeding it. List BrxTargets(uint src, bool uniform) { if (C1 == null || C1.Length == 0) throw Fail("BRX without the program's c1 constants"); int Writer(int from, uint reg, bool ur) { for (int k = from; k >= 0 && k >= from - 64; k--) { var p = _code[k]; uint op = p.Bits(0, 12), b = op & 0x1ff; // control flow: not straight-line. BSSY, BSYNC and WARPSYNC branch nowhere and write no register: the // compiler puts a BSSY for the switch's reconvergence between the IMNMX bound and the SHL. if (b is >= 0x140 and <= 0x15f && op is not 0x945 and not 0x941 and not 0x948) return -1; // uniform datapath: ULDC, UMOV and the U* ALU ops (base 0x080-0x0bf; UISETP/UPLOP3 write predicates) bool isUr = op is 0xab9 or 0xabb or 0x882 or 0xc82 || (b is >= 0x080 and < 0x0c0 && b != 0x08c && b != 0x09c); if (p.Bits(16, 8) == reg && isUr == ur) return k; } return -1; } int ldc = Writer(_index - 1, src, uniform); if (ldc < 0) throw Fail("BRX register not loaded from a jump table"); var l = _code[ldc]; uint lop = l.Bits(0, 12); if (!(lop == 0xb82 && !uniform || lop == 0xabb && uniform) || l.Bits(54, 5) != 1) throw Fail($"BRX register loaded by op {lop:x3}, not from c1"); uint tableBase = l.Bits(38, 16); // the bound: the index register of the load, written by SHL x, 2 from an IMNMX.U32 with an immediate int count = 0; int shl = Writer(ldc - 1, l.Bits(24, 8), uniform); if (shl >= 0) { var sh = _code[shl]; int mn = Writer(shl - 1, sh.Bits(24, 8), uniform); // IMNMX / UIMNMX with an immediate bound, or SEL / USEL Rx, Rx, bound under the compare that clamps it // (the uniform switches: UISETP.GT P0, URx, bound; USEL URx, URx, bound, P0) uint mop = mn >= 0 ? _code[mn].Bits(0, 12) : 0, mb = mop & 0x7f; if (mn >= 0 && (mb == 0x17 || mb == 0x07) && _code[mn].Bits(9, 3) == 4) count = (int)_code[mn].W1 + 1; // the register form, the bound in a register a MOV (UMOV) of an immediate set before it: SEL Rx, Rx, Rb // under ISETP Rx, Rb, or IMNMX Rx, Rx, Rb; either source may be the bound, the other is the index else if (mn >= 0 && (mb == 0x17 || mb == 0x07) && _code[mn].Bits(9, 3) == 1) foreach (uint r in new[] { _code[mn].Bits(32, 8), _code[mn].Bits(24, 8) }) { int mv = r == RZ ? -1 : Writer(mn - 1, r, uniform); if (mv >= 0 && _code[mv].Bits(0, 12) == (uniform ? 0x882u : 0x802u) && _code[mv].Bits(12, 4) == 7) { count = (int)_code[mv].W1 + 1; break; } } } if (count <= 0 || count > 1024) throw Fail("BRX jump table without a bound"); var t = new List(); for (int k = 0; k < count; k++) { int at = (int)tableBase + 4 * k; if (at + 4 > C1.Length) throw Fail("BRX jump table outside c1"); uint v = BitConverter.ToUInt32(C1, at); if (v % 16 != 0 || v / 16 > _code.Length) throw Fail($"BRX jump table entry 0x{v:x}"); if (!t.Contains(v)) t.Add(v); } return t; } // CALL.REL.NOINC (relative, the offset as BRA's) pushes no return address: it is a jump. The compiler makes a // call of it by putting the address after the CALL in a register first (MOV Rn, imm, which the scheduler may // hoist dozens of instructions, within the straight-line code before the CALL) // for the callee's RET.REL.NODEC Rn; such a CALL becomes Maxwell CAL and its RET Maxwell RET (the translator // keeps the call stack, so the link register is not written). Without that MOV it is a plain, possibly // predicated, jump (`@P0 CALL.NOINC` over a BRA) and becomes BRA: as CAL it made functions that call each // other, a recursive call graph Vulkan forbids and NVIDIA's compiler inlines until its stack overflows. bool SetsReturnAddress() { uint ret = (uint)(_index + 1) * 16; for (int k = _index - 1; k >= 0 && k >= _index - 64; k--) { var p = _code[k]; uint op = p.Bits(0, 12); if ((op & 0x1ff) >= 0x140 && (op & 0x1ff) <= 0x15f) break; // control flow: the MOV would not reach the CALL if (op == 0x802 && p.W1 == ret && p.Bits(12, 4) == 7) return true; } return false; } void Call() { if (_i.W2 >> 18 != 0x03c00000 >> 18) throw Fail("CALL mode"); long raw = (long)(_i.W1 | ((ulong)(_i.W2 & 0x3ffff) << 32)); long target = (long)_index * 16 + 16 + ((raw << 14) >> 14); if (target < 0 || target % 16 != 0 || target / 16 > _code.Length) throw Fail("CALL target"); if (!SetsReturnAddress()) { _branches.Add((_out.Count, (int)(target / 16))); Emit(0xe24000000000000fUL | ((ulong)_i.Bits(12, 4) << 16)); // BRA under the CALL's own predicate return; } if (_i.Bits(12, 4) != 7) { // Maxwell CAL has no predicate: skip the call unless the guard holds _branches.Add((_out.Count, _index + 1)); Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16)); } _branches.Add((_out.Count, (int)(target / 16))); // Predicate field 0, as Maxwell CAL encodes it (it has no predicate): the translator's Decoder reads a CAL whose // field says PT as an unconditional branch and decodes nothing after it (Decoder.IsUnconditionalBranch). Emit(0xe260000000000000UL); } void Ret() { if ((_i.W2 & ~0x3ffffu) != 0x03c00000) throw Fail("RET mode"); EmitG(0xe32000000000000fUL); } // ---- control flow ------------------------------------------------------------------------------------------ void Kill() { if (_i.W1 != 0 || _i.W2 != 0x03800000) throw Fail("KILL mode"); EmitG(0xe33000000000000fUL); } // EXIT, with an optional second condition Pc in bits 87-90: exits when the guard and Pc hold. void Exit() { if (_i.Bits(84, 2) != 0) throw Fail("conditional/special EXIT"); uint pc = _i.Bits(87, 4); if (pc == 7) { EmitG(0xe30000000000000fUL); return; } if (_i.Bits(12, 4) != 7) { _branches.Add((_out.Count, _index + 1)); Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16)); // skip unless the guard holds } Emit(0xe30000000000000fUL | ((ulong)pc << 16)); } // BRA: byte offset from the next instruction in bits 34-81 (sign-extended), mode in bits 32-33 (.DIV 2: taken // only on a diverged warp, never here; .CONV 3: taken on a converged warp, always here), an extra condition // predicate in bits 87-90. void Bra() { uint mode = _i.Bits(32, 2); if (mode <= 1 && _i.Bit(91)) throw Fail("BRA uniform condition"); if (mode == 2) return; long raw = (long)((_i.W1 & ~3u) | ((ulong)(_i.W2 & 0x3ffff) << 32)); long rel = (raw << 14) >> 14; long target = (long)_index * 16 + 16 + rel; if (target < 0 || target % 16 != 0 || target / 16 > _code.Length) throw Fail("BRA target"); ulong pred = _guard; uint extra = _i.Bits(87, 4); if (extra != 7) { if (_i.Bits(12, 4) != 7) { // both conditions: skip the branch unless the guard holds _branches.Add((_out.Count, _index + 1)); Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16)); } pred = (ulong)extra << 16; } _branches.Add((_out.Count, (int)(target / 16))); Emit(0xe24000000000000fUL | pred); } } // code: the NVN2 program as stored (graphics: 0x80-byte header then instructions; compute: instructions only), size its // byte length from the control section (+0x38). Returns the Maxwell program in the translator's input layout. // mesh: a task or mesh program (graphics false): its 0x80-byte header is read for the output map and dropped, and the // program is lowered as a compute program. public static byte[] Lower(byte[] code, int size, bool graphics, int[] table = null, MeshInfo mesh = null) => Lower(code, size, graphics, out _, table, mesh); // c1: the program's c1 constants (BRX jump tables); set before Lower by the caller. [ThreadStatic] public static byte[] C1Bytes; // Diagnostics only (Program.cs LowerFile, the --lower mode): the register UR0 was mapped to by the last Lowerer // built on this thread (Lowerer.UrBase; SpillFrom in spill mode). Lowering never reads it. [ThreadStatic] public static int LastUrBase; // localBytes: the local memory size the lowered program needs when that changed (spill mode), else 0. Graphics // programs carry it in their header already; compute and task/mesh programs need it from the translator's host // (job "local_mem"). public static byte[] Lower(byte[] code, int size, bool graphics, out int localBytes, int[] table = null, MeshInfo mesh = null) { localBytes = 0; int header = graphics || mesh != null ? 0x80 : 0; if (size <= header || size > code.Length || (size - header) % 16 != 0) throw new LoweringException($"sm86 program size 0x{size:x} (file 0x{code.Length:x})"); var insns = new Insn[(size - header) / 16]; for (int k = 0; k < insns.Length; k++) insns[k] = new Insn { Lo = BitConverter.ToUInt64(code, header + k * 16), Hi = BitConverter.ToUInt64(code, header + k * 16 + 8) }; byte[] sph = null; if (graphics && ((BitConverter.ToUInt32(code, 0) >> 10) & 15) == 5) { sph = new byte[0x50]; Buffer.BlockCopy(code, 0, sph, 0, 0x50); } if (mesh != null) { var h = new byte[0x80]; Buffer.BlockCopy(code, 0, h, 0, 0x80); mesh.Layout(h); } List lowered; List<(int output, int targetSource)> branches; int[] first; int spillAt = -1; try { lowered = new Lowerer(insns, sph, !graphics, table, mesh) { C1 = C1Bytes }.Run(out branches, out first); } catch (RegisterPressureException) { // Local memory after the program's own: the header's sizes (translator: low + high + CRS / 32), or the // translator's compute default (0x1000) for compute programs. spillAt = 0x1000; if (header != 0) spillAt = ((int)(BitConverter.ToUInt32(code, 4) & 0xffffff) + (int)(BitConverter.ToUInt32(code, 8) & 0xffffff) + (int)(BitConverter.ToUInt32(code, 12) & 0xffffff) / 32 + 15) & ~15; try { lowered = new Lowerer(insns, sph, !graphics, table, mesh, spillAt, Spill) { C1 = C1Bytes }.Run(out branches, out first); localBytes = spillAt + Spill.SpillBytes; } catch (UrPoolException) { lowered = new Lowerer(insns, sph, !graphics, table, mesh, spillAt, SpillWide) { C1 = C1Bytes }.Run(out branches, out first); localBytes = spillAt + SpillWide.SpillBytes; } } int start = graphics ? 0x50 : 0; var address = new int[lowered.Count + 1]; int pos = start; for (int k = 0; k <= lowered.Count; k++) { if (((pos - start) & 0x1f) == 0) pos += 8; address[k] = pos; pos += 8; } foreach (var (output, targetSource) in branches) { long rel = address[first[targetSource]] - (address[output] + 8); if (rel < -0x800000 || rel > 0x7fffff) throw new LoweringException("sm86 lowered branch distance"); lowered[output] |= ((ulong)rel & 0xffffff) << 20; } var result = new byte[pos + 16]; if (graphics) Buffer.BlockCopy(code, 0, result, 0, 0x50); if (graphics && spillAt >= 0) // the header's low local memory size covers the spill area too BitConverter.GetBytes((BitConverter.ToUInt32(result, 4) & 0xff000000u) | (uint)localBytes).CopyTo(result, 4); for (int k = 0; k < lowered.Count; k++) BitConverter.GetBytes(lowered[k]).CopyTo(result, address[k]); // a closing EXIT after the last instruction, so a branch to the end lands somewhere BitConverter.GetBytes(0xe30000000007000fUL).CopyTo(result, address[lowered.Count]); return result; } }