mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 18:09:58 -04:00
The Ampere I2F lowering wrote (size | 4) << 10, so the signed flag landed in bit 12, which the Maxwell decoder does not read (ISrcFmt = (op >> 11) & 4 | (op >> 10) & 3). I2F.F32.S16 and the other signed sources converted as unsigned: in Jabu-Jabu's Belly the walls' deformation grid of signed 16-bit offsets turned -5 into 65531 and the room shell into huge stretched triangles (issue 57). The size code stays in bits 10-11, the flag goes to 13, as I2FP already does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2601 lines
159 KiB
C#
2601 lines
159 KiB
C#
// Switch 2 (NVN2) shader programs are SM86 (Ampere) code; the shader translator decodes Maxwell (SM5x) only. This file
|
|
// rewrites each SM86 instruction as one or more Maxwell instructions with the same meaning, so its Maxwell decoder,
|
|
// IR and SPIR-V backend run unchanged. Instructions this file does not know make the whole program fail with the opcode
|
|
// and the reason, so coverage grows by census (python nvn2_shaders.py over a NX2NAVI_SHADER_DUMP directory).
|
|
//
|
|
// Encoding facts: the SM70+ instruction field layouts; the Maxwell side is the translator's Decoders/InstTable.cs and
|
|
// InstDecoders.cs. Where SM86 and the translator's Maxwell model differ: MUFU.SIN/COS take turns, BSSY/BSYNC are
|
|
// reconvergence hints, bank 24 is never bound.
|
|
//
|
|
// Layout: graphics programs keep the first 0x50 bytes of the 0x80-byte Turing+ header (the same fields as Maxwell's
|
|
// header), then the instructions; Maxwell puts a scheduling word first in every 32 bytes counted from the first
|
|
// instruction (the translator's Decoder skips them), written as 0. Compute programs start at address 0.
|
|
//
|
|
// Registers: guest registers keep their numbers. Uniform registers live just above the highest register the program
|
|
// uses (UrBase, even so pairs stay aligned; URZ is RZ) and must end below R239. R239-R254 are this file's scratch
|
|
// registers; R237 and R238 hold the uniform predicate state (see "uniform predicates" below). A program that leaves
|
|
// no room for them (task and mesh programs use up to R252) is lowered again in spill mode: guest registers from
|
|
// SpillFrom up live in local memory after the program's own, and an instruction that names one works on a block of
|
|
// four temporaries (R188-R203, one block per aligned group of four guest registers, so pairs and quads stay
|
|
// contiguous) loaded before it and stored after it; uniform registers then take blocks of four from R204-R235 (eight
|
|
// groups: an Ocarina mesh program used more than the four R220-R235 held). A program with more than eight groups of
|
|
// uniform registers is lowered once more in the wide spill layout: guest registers from R156 spilled, temporaries
|
|
// R156-R171, uniform registers R172-R235 (sixteen groups, UR0-UR62). Programs that fit keep the first layout.
|
|
using System;
|
|
using System.Collections.Generic;
|
|
|
|
static class AmpereLowering
|
|
{
|
|
const int RZ = 255, PT = 7, ScratchBase = 239;
|
|
// Uniform predicates: UPk is bit k of RUP. An instruction that reads or writes uniform predicates runs with UPk
|
|
// swapped into Pk: the guest's Pk saved to USAVE (P2R), UPk loaded (R2P), the instruction lowered as if its
|
|
// predicate fields named P registers, written UPs merged back into RUP (P2R), the guest's P restored (R2P).
|
|
const int RUP = 238, USAVE = 237, UrLimit = 237;
|
|
// Scratch: operand A/B/C materialisation, a spare, texture coordinates, texture meta values, texture results.
|
|
const int SA = 252, SB = 253, SC = 254, SX = 239, TexCoord = 240, TexMeta = 244, TexOut = 248;
|
|
|
|
public class LoweringException : Exception
|
|
{
|
|
public LoweringException(string message) : base(message) { }
|
|
}
|
|
|
|
// The registers do not fit the plain layout: Lower retries in spill mode.
|
|
sealed class RegisterPressureException : LoweringException
|
|
{
|
|
public RegisterPressureException(string message) : base(message) { }
|
|
}
|
|
|
|
// The uniform registers do not fit the spill layout's pool: Lower retries in the wide spill layout.
|
|
sealed class UrPoolException : LoweringException
|
|
{
|
|
public UrPoolException(string message) : base(message) { }
|
|
}
|
|
|
|
// Spill layouts: guest registers from SpillFrom up in local memory, the temporaries from there, the uniform pool
|
|
// after them up to UrPoolEnd.
|
|
readonly record struct SpillLayout(int SpillFrom, int UrPool)
|
|
{
|
|
public int TempBase => SpillFrom;
|
|
public int SpillBytes => (256 - SpillFrom) * 4;
|
|
}
|
|
const int UrPoolEnd = 236;
|
|
static readonly SpillLayout Spill = new(188, 204), SpillWide = new(156, 172);
|
|
|
|
// Task (NVN2 stage 6) and mesh (stage 7) programs. Both are lowered as compute programs: the stage buffer they
|
|
// write with ISBEWR and read with ISBERD becomes shared memory, laid out by this record, and MeshSpirv.cs turns
|
|
// the compute SPIR-V into a TaskEXT or MeshEXT entry point that hands the buffer to Vulkan.
|
|
// Mesh: vertex outputs at Attr: 32-vertex groups, a 32-word line per output component, word
|
|
// ((v >> 5) * Lines + line) * 32 + (v & 31); the primitive count at Map + 0 (ISBEWR region 1) and the
|
|
// primitive's vertex indices as bytes from Map + 4 (region 0); the task payload at Pay (ISBERD region 7).
|
|
// Task: the payload at Pay: the mesh workgroup count at Pay + 4, the data from Pay + 0x10 (region 7).
|
|
// ISBERD of address 0 in region 15 (no task) or 7 reads the workgroup's index. Control section fields
|
|
// (runtime/gfx_shader.c build_request): +0x58 low 16 bits the workgroup size, bits 24-31 the payload size in 256
|
|
// bytes; +0x5c byte 3 the primitive type (1 points, 2 lines, 3 triangles), byte 2 set when a task stage feeds the
|
|
// mesh stage; +0x60 low 16 bits the vertex limit, high 16 bits the primitive limit.
|
|
public sealed class MeshInfo
|
|
{
|
|
public int Stage, Threads, MaxVertices, MaxPrimitives, Topology, PayloadBytes;
|
|
public bool HasTask;
|
|
// Output lines in order: Maxwell attribute address of each component (0x60.. system values, 0x80 + 16g + 4c
|
|
// generic g component c).
|
|
public readonly List<int> Lines = new();
|
|
public int Attr, Map, Pay, SharedBytes;
|
|
public int Groups => (MaxVertices + 31) / 32;
|
|
public int IndicesPerPrimitive => Topology switch { 1 => 1, 2 => 2, _ => 3 };
|
|
|
|
// Lines from the output map of the program header (Turing+ VTG header, the same bytes as Maxwell's: system
|
|
// values B at byte 0x35, attributes 0x60-0x7c one bit each; generic vectors from byte 0x36, a nibble each,
|
|
// component x in bit 0). System values A (0x34) and C (0x48) are not expected in mesh programs.
|
|
public void Layout(byte[] sph)
|
|
{
|
|
if (Stage == 7)
|
|
{
|
|
if (sph[0x34] != 0 || sph[0x48] != 0 || sph[0x49] != 0) throw new LoweringException("mesh output map: system values A or C");
|
|
for (int b = 0; b < 8; b++) if ((sph[0x35] >> b & 1) != 0) Lines.Add(0x60 + 4 * b);
|
|
for (int g = 0; g < 32; g++)
|
|
for (int c = 0; c < 4; c++)
|
|
if ((sph[0x36 + g / 2] >> (4 * (g & 1) + c) & 1) != 0) Lines.Add(0x80 + 16 * g + 4 * c);
|
|
Attr = 0;
|
|
Map = (Attr + Groups * Lines.Count * 128 + 15) & ~15;
|
|
Pay = (Map + 4 + IndicesPerPrimitive * MaxPrimitives + 15) & ~15;
|
|
SharedBytes = Pay + (HasTask ? PayloadBytes : 0);
|
|
}
|
|
else { Pay = 0; SharedBytes = PayloadBytes; }
|
|
if (SharedBytes == 0) SharedBytes = 16;
|
|
}
|
|
}
|
|
|
|
struct Insn
|
|
{
|
|
public ulong Lo, Hi;
|
|
public uint W0 => (uint)Lo;
|
|
public uint W1 => (uint)(Lo >> 32);
|
|
public uint W2 => (uint)Hi;
|
|
public uint Bits(int offset, int count)
|
|
{
|
|
ulong v = offset >= 64 ? Hi >> (offset - 64) : offset + count <= 64 ? Lo >> offset : (Lo >> offset) | (Hi << (64 - offset));
|
|
return (uint)(v & ((1UL << count) - 1));
|
|
}
|
|
public bool Bit(int offset) => Bits(offset, 1) != 0;
|
|
}
|
|
|
|
sealed class Lowerer
|
|
{
|
|
readonly Insn[] _code;
|
|
readonly List<ulong> _out = new();
|
|
readonly List<(int output, int targetSource)> _branches = new();
|
|
readonly int[] _firstOutput;
|
|
Insn _i;
|
|
int _index;
|
|
ulong _guard; // the SM86 instruction's predicate, in Maxwell position (bits 16-19)
|
|
|
|
readonly byte[] _header; // the program header, null for compute
|
|
readonly int UrBase; // R(UrBase + n) holds URn; guest registers are below it
|
|
readonly bool _compute;
|
|
readonly int[] _table; // NVN2 compute binding table: start byte, texture entries, image entries (or null)
|
|
readonly MeshInfo _mesh; // task or mesh program (lowered as compute), else null
|
|
public byte[] C1; // the program's c1 constants (NVN2: after the code; BRX jump tables), or null
|
|
readonly int _spillAt = -1; // spill mode: byte offset of guest register SpillFrom in local memory; -1 otherwise
|
|
readonly SpillLayout _sl; // spill mode: the layout
|
|
int SpillFrom => _sl.SpillFrom;
|
|
readonly Dictionary<int, int> _temps = new(); // spill mode: guest register group (r & ~3) -> its temporaries, this instruction
|
|
readonly Dictionary<int, int> _urBlocks = new(); // spill mode: uniform register group (n & ~3) -> its registers
|
|
public Lowerer(Insn[] code, byte[] header, bool compute, int[] table, MeshInfo mesh, int spillAt = -1, SpillLayout layout = default)
|
|
{
|
|
_code = code; _header = header; _compute = compute; _table = table; _mesh = mesh; _firstOutput = new int[code.Length + 1];
|
|
_spillAt = spillAt;
|
|
_sl = layout.SpillFrom != 0 ? layout : Spill;
|
|
int highest = HighestGpr(code);
|
|
if (spillAt < 0 && highest >= UrLimit) throw new RegisterPressureException($"guest registers up to R{highest}");
|
|
UrBase = spillAt >= 0 ? SpillFrom : (highest + 2) & ~1;
|
|
LastUrBase = UrBase;
|
|
}
|
|
|
|
// Spill mode: the temporary that holds guest register r for this instruction, loading its group on first use.
|
|
int Spilled(int r)
|
|
{
|
|
int g = r & ~3;
|
|
if (!_temps.TryGetValue(g, out int t))
|
|
{
|
|
if (_temps.Count == 4) throw Fail("more than four spilled register groups in one instruction");
|
|
t = _sl.TempBase + 4 * _temps.Count;
|
|
_temps[g] = t;
|
|
for (int k = 0; k < 4; k++) // LDL.32 t+k, [RZ + slot]
|
|
Emit(Unguarded(0xef40000000000000UL | (uint)(t + k) | ((ulong)RZ << 8) | ((ulong)(uint)(_spillAt + 4 * (g - SpillFrom + k)) << 20) | (4UL << 48)));
|
|
}
|
|
return t + (r & 3);
|
|
}
|
|
|
|
// Spill mode: the temporaries of this instruction back to local memory.
|
|
void SpillStore()
|
|
{
|
|
foreach (var (g, t) in _temps)
|
|
for (int k = 0; k < 4; k++) // STL.32 [RZ + slot], t+k
|
|
Emit(Unguarded(0xef50000000000000UL | (uint)(t + k) | ((ulong)RZ << 8) | ((ulong)(uint)(_spillAt + 4 * (g - SpillFrom + k)) << 20) | (4UL << 48)));
|
|
_temps.Clear();
|
|
}
|
|
|
|
// The highest general register the program names (destination, A, B in the register form, C), counting the
|
|
// register fields of uniform-datapath instructions too: an overestimate only moves the uniform registers up.
|
|
static int HighestGpr(Insn[] code)
|
|
{
|
|
int max = -1;
|
|
foreach (var i in code)
|
|
{
|
|
void see(uint r) { if (r < RZ && r > max) max = (int)r; }
|
|
uint op = i.Bits(0, 12), b = op & 0x1ff;
|
|
if (b is >= 0x140 and <= 0x15f) continue; // branches, calls, exits, barriers: no register fields
|
|
see(i.Bits(16, 8)); see(i.Bits(24, 8));
|
|
// Register ranges the base fields do not show: LDSM writes Rd..Rd+count-1; TXD reads Ra..Ra+2 and Rb..Rb+3
|
|
// and writes the second destination pair; HMMA reads A (K/4 registers), B (K/8) and C and writes D, where
|
|
// C and D are a pair (.F16) or four registers (.F32, bit 76).
|
|
if (op == 0x83b && i.Bits(16, 8) != RZ) see(i.Bits(16, 8) + (1u << (int)i.Bits(72, 2)) - 1);
|
|
if (op == 0xb6c) { see(i.Bits(16, 8) + 1); see(i.Bits(24, 8) + 2); see(i.Bits(32, 8) + 3); see(i.Bits(64, 8) + 1); }
|
|
if (op == 0x23c)
|
|
{
|
|
uint k16 = i.Bits(75, 1), cd = i.Bit(76) ? 3u : 1u;
|
|
see(i.Bits(16, 8) + cd); see(i.Bits(24, 8) + 1 + 2 * k16); see(i.Bits(32, 8) + k16); see(i.Bits(64, 8) + cd);
|
|
}
|
|
if (op == 0x3a9) see(i.Bits(64, 8)); // ATOMG.CAS: the new value
|
|
if (op == 0x593 || (op == 0xb66 && i.Bit(78))) see(i.Bits(32, 8)); // SUATOM data, TLD.MS sample index
|
|
if (b < 0x080 || b is >= 0x160 and <= 0x16f) see(i.Bits(64, 8)); // C of ALU forms, TEX second destinations
|
|
if (i.Bits(9, 3) == 1) see(i.Bits(32, 8));
|
|
}
|
|
return max;
|
|
}
|
|
|
|
LoweringException Fail(string reason) =>
|
|
new($"sm86 instruction {_index} (0x{_index * 16:x}) op {_i.Bits(0, 12):x3} [{_i.W0:x8} {_i.W1:x8} {_i.W2:x8} {(uint)(_i.Hi >> 32):x8}]: {reason}");
|
|
|
|
void Emit(ulong word) => _out.Add(word);
|
|
// Guarded by the instruction's own predicate.
|
|
void EmitG(ulong word) => _out.Add(word | _guard);
|
|
static ulong Unguarded(ulong word) => word | ((ulong)PT << 16);
|
|
|
|
// ---- Maxwell building blocks -------------------------------------------------------------------------------
|
|
// guarded: under the instruction's predicate (a guest-visible write); otherwise unconditional (scratch).
|
|
void Put(ulong word, bool guarded) => _out.Add(guarded ? word | _guard : Unguarded(word));
|
|
void Mov(int dst, int src, bool guarded = true) => Put(0x5c98000000000000UL | (15UL << 39) | ((ulong)src << 20) | (uint)dst, guarded);
|
|
void Mov32i(int dst, uint imm, bool guarded = false) => Put(0x0100000000000000UL | (15UL << 12) | ((ulong)imm << 20) | (uint)dst, guarded);
|
|
void MovC(int dst, int bank, int byteOffset, bool guarded = false) =>
|
|
Put(0x4c98000000000000UL | (15UL << 39) | ((ulong)(byteOffset / 4) << 20) | ((ulong)bank << 34) | (uint)dst, guarded);
|
|
void Lop32i(int dst, int a, uint imm, int op) => Emit(Unguarded(0x0400000000000000UL | ((ulong)op << 53) | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)dst));
|
|
void Fmul32i(int dst, int a, uint imm) => Emit(Unguarded(0x1e00000000000000UL | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)dst));
|
|
void Iadd3(int dst, int a, int b, int c, bool guarded, bool negB = false) =>
|
|
Put(0x5cc0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst | (negB ? 1UL << 50 : 0), guarded);
|
|
// Rd = p ? a : b (p: index | negate << 3)
|
|
void Sel(int dst, int a, int b, uint p, bool guarded) =>
|
|
Put(0x5ca0000000000000UL | ((ulong)(p & 7) << 39) | ((ulong)(p >> 3) << 42) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst, guarded);
|
|
// Pd = a < b, unsigned (the carry out of a + b is sum < a)
|
|
void IsetpLtU(int pd, int a, int b) =>
|
|
EmitG(0x5b60000000000000UL | (1UL << 49) | ((ulong)PT << 39) | ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)pd << 3) | PT);
|
|
|
|
// ---- SM86 operand decoding ---------------------------------------------------------------------------------
|
|
uint Form => _i.Bits(9, 3);
|
|
int Dst => (int)_i.Bits(16, 8);
|
|
int RegA => (int)_i.Bits(24, 8);
|
|
|
|
int Ur(uint reg)
|
|
{
|
|
if (reg == 63) return RZ;
|
|
if (reg > 62) throw Fail($"UR{reg}");
|
|
if (_spillAt >= 0)
|
|
{
|
|
int q = (int)reg & ~3;
|
|
if (!_urBlocks.TryGetValue(q, out int b))
|
|
{
|
|
b = _sl.UrPool + 4 * _urBlocks.Count;
|
|
if (b + 4 > UrPoolEnd) throw new UrPoolException(Fail("more uniform register groups than spill mode has room for").Message);
|
|
_urBlocks[q] = b;
|
|
}
|
|
return b + ((int)reg & 3);
|
|
}
|
|
if (UrBase + (int)reg >= UrLimit) throw new RegisterPressureException($"UR{reg} has no free register (guest registers up to R{UrBase - 1})");
|
|
return UrBase + (int)reg;
|
|
}
|
|
|
|
// While a uniform-datapath instruction is lowered through its vector twin (UniformAlu), register fields name
|
|
// uniform registers.
|
|
bool _uniform;
|
|
|
|
int Gpr(uint reg)
|
|
{
|
|
if (_uniform) return reg == RZ ? RZ : Ur(reg);
|
|
if (reg != RZ && _spillAt >= 0 && reg >= SpillFrom) return Spilled((int)reg);
|
|
if (reg != RZ && reg >= UrBase) throw Fail($"R{reg} above the registers the program was scanned for");
|
|
return (int)reg;
|
|
}
|
|
|
|
// Constant bank 24 holds NVN2 driver data the shaders read (fullscreen passes: texel size, viewport rectangle);
|
|
// Maxwell has 18 banks, so it is bank 17 here and the runtime binds it there.
|
|
static int Bank(uint bank) => bank == 24 ? 17 : (int)bank;
|
|
|
|
// Constant-buffer operand: byte offset 38-53, bank 54-58.
|
|
(int bank, int offset) Cbuf()
|
|
{
|
|
int bank = Bank(_i.Bits(54, 5)), offset = (int)_i.Bits(38, 16);
|
|
if (bank >= 18 || (offset & 3) != 0 || (_i.Bit(91) && !_uniform)) throw Fail($"constant buffer operand c[{_i.Bits(54, 5)}][0x{offset:x}]");
|
|
return (bank, offset);
|
|
}
|
|
|
|
// Source 0 (A), 1 (B) or 2 (C) with its float modifiers. Forms (bits 9-11): 1 B and C registers, 2 C immediate,
|
|
// 3 C constant, 4 B immediate, 5 B constant, 6 B uniform, 7 C uniform. A wide (immediate/constant/uniform) operand
|
|
// uses bits 32-63; the register operand that would sit there then moves to bits 64-71.
|
|
(int reg, bool abs, bool neg) Raw(int which, int scratch)
|
|
{
|
|
if (which == 0) return (Gpr(_i.Bits(24, 8)), _i.Bit(73), _i.Bit(72));
|
|
uint form = Form;
|
|
if (form == 0) throw Fail("source form 0");
|
|
bool wide = which == 1 ? form is 4 or 5 or 6 : form is 2 or 3 or 7;
|
|
if (wide)
|
|
{
|
|
if (form is 4 or 2) { Mov32i(scratch, _i.W1); return (scratch, false, false); }
|
|
if (form is 5 or 3) { var (bank, off) = Cbuf(); MovC(scratch, bank, off); return (scratch, _i.Bit(62), _i.Bit(63)); }
|
|
return (Ur(_i.Bits(32, 8)), _i.Bit(62), _i.Bit(63));
|
|
}
|
|
bool upper = which == 2 || form is 2 or 3 or 7;
|
|
return upper ? (Gpr(_i.Bits(64, 8)), _i.Bit(74), _i.Bit(75)) : (Gpr(_i.Bits(32, 8)), _i.Bit(62), _i.Bit(63));
|
|
}
|
|
|
|
// The second operand of a two-source instruction sits in the B slot, or in the C slot for forms 2, 3 and 7.
|
|
int SecondSlot => Form is 2 or 3 or 7 ? 2 : 1;
|
|
|
|
// Applies float abs/neg in a scratch register (bit operations, so -0.0 and NaN payloads stay exact).
|
|
int Mat(int reg, bool abs, bool neg, int scratch)
|
|
{
|
|
if (!abs && !neg) return reg;
|
|
if (abs) { Lop32i(scratch, reg, 0x7fffffff, 0); reg = scratch; }
|
|
if (neg) { Lop32i(scratch, reg, 0x80000000, 2); reg = scratch; }
|
|
return reg;
|
|
}
|
|
|
|
int Plain(int which, int scratch)
|
|
{
|
|
var (r, _, _) = Raw(which, scratch);
|
|
return r;
|
|
}
|
|
|
|
int FloatMat(int which, int scratch)
|
|
{
|
|
var (r, a, n) = Raw(which, scratch);
|
|
return Mat(r, a, n, scratch);
|
|
}
|
|
|
|
// A uniform guard is swapped in by Run like any other uniform predicate operand.
|
|
void RequireUniformGuardTrue() { }
|
|
|
|
// Uniform-datapath instructions: their guard and predicate fields name uniform predicates.
|
|
static bool IsUniformOp(uint op) => ((op & 0x1ff) is >= 0x080 and <= 0x0bf && op != 0x886) || op == 0x9c3; // VOTEU: Vote
|
|
|
|
// Uniform predicates the instruction reads and writes, as masks of UP indices (UPT excluded).
|
|
(uint inUp, uint outUp) UniformPredicates()
|
|
{
|
|
uint op = _i.Bits(0, 12);
|
|
if (!IsUniformOp(op)) return (0, 0);
|
|
uint inUp = 0, outUp = 0;
|
|
void use(ref uint mask, uint p) { if ((p & 7) != 7) mask |= 1u << (int)(p & 7); }
|
|
use(ref inUp, _i.Bits(12, 3));
|
|
switch (op & 0x1ff)
|
|
{
|
|
case 0x08c: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); use(ref inUp, _i.Bits(87, 3)); break; // UISETP
|
|
case 0x09c: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); // UPLOP3
|
|
use(ref inUp, _i.Bits(87, 3)); use(ref inUp, _i.Bits(77, 3)); use(ref inUp, _i.Bits(68, 3)); break;
|
|
case 0x090: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); // UIADD3
|
|
if (_i.Bit(74)) { use(ref inUp, _i.Bits(87, 3)); use(ref inUp, _i.Bits(77, 3)); } break;
|
|
case 0x092: use(ref outUp, _i.Bits(81, 3)); use(ref inUp, _i.Bits(87, 3)); break; // ULOP3
|
|
case 0x087: use(ref inUp, _i.Bits(87, 3)); break; // USEL
|
|
case 0x091: case 0x0a4: case 0x0a5: use(ref outUp, _i.Bits(81, 3)); if (_i.Bit(74)) use(ref inUp, _i.Bits(87, 3)); break;
|
|
}
|
|
return (inUp, outUp);
|
|
}
|
|
|
|
// ---- driver ----------------------------------------------------------------------------------------------
|
|
public List<ulong> Run(out List<(int output, int targetSource)> branches, out int[] firstOutput)
|
|
{
|
|
Mov32i(RUP, 0); // uniform predicates start false
|
|
for (_index = 0; _index < _code.Length; _index++)
|
|
{
|
|
_i = _code[_index];
|
|
_firstOutput[_index] = _out.Count;
|
|
if (_i.Bits(12, 4) == 15) continue; // @!PT: never runs (compiler padding)
|
|
_guard = (ulong)_i.Bits(12, 4) << 16;
|
|
var (inUp, outUp) = UniformPredicates();
|
|
uint m = inUp | outUp;
|
|
if (m != 0)
|
|
{
|
|
Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RZ << 8) | USAVE)); // P2R: save guest P
|
|
Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)RUP << 8))); // R2P: load UPs
|
|
}
|
|
LowerOne();
|
|
if (_temps.Count != 0) SpillStore();
|
|
if (m != 0)
|
|
{
|
|
if (outUp != 0) Emit(Unguarded(0x38e8000000000000UL | ((ulong)outUp << 20) | ((ulong)RUP << 8) | RUP)); // P2R merge
|
|
Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)USAVE << 8))); // R2P: restore
|
|
}
|
|
}
|
|
_firstOutput[_code.Length] = _out.Count;
|
|
branches = _branches;
|
|
firstOutput = _firstOutput;
|
|
return _out;
|
|
}
|
|
|
|
void LowerOne()
|
|
{
|
|
uint op = _i.Bits(0, 12);
|
|
switch (op)
|
|
{
|
|
// Scheduling, dependency and reconvergence hints: the translator runs each thread on its own.
|
|
case 0x918: // NOP
|
|
case 0x91a: // DEPBAR
|
|
case 0x945: // BSSY
|
|
case 0x941: // BSYNC
|
|
case 0x942: // BREAK
|
|
case 0x948: // WARPSYNC
|
|
case 0x9ab: // ERRBAR: waits for earlier memory errors to be reported, which the host GPU does not do here
|
|
return;
|
|
|
|
case 0x202: case 0x802: case 0xa02: case 0xc02: Mov_(op); return;
|
|
case 0x882: case 0xc82: Umov(op); return;
|
|
case 0xab9: Uldc(); return;
|
|
case 0x290: case 0x890: case 0xa90: // UIADD3
|
|
case 0x2a4: case 0x4a4: case 0x8a4: case 0xaa4: case 0x2a5: case 0x8a5: // UIMAD, UIMAD.WIDE
|
|
case 0x292: case 0x892: case 0xa92: // ULOP3
|
|
case 0x299: case 0x499: case 0x899: // USHF
|
|
case 0x291: case 0x891: // ULEA
|
|
case 0x287: case 0x887: // USEL
|
|
case 0x28c: case 0x88c: case 0xa8c: // UISETP
|
|
case 0x89c: // UPLOP3
|
|
case 0x296: case 0x896: // UPRMT
|
|
UniformAlu(op); return;
|
|
case 0x81c: Plop3(); return;
|
|
case 0xb63: Tld4(); return;
|
|
case 0x388: case 0x984: case 0x988: SharedAccess(op); return;
|
|
case 0x83b: Ldsm(); return;
|
|
case 0xb1d: Bar(); return;
|
|
case 0x992: Membar(); return;
|
|
case 0x305: case 0x905: case 0xb05: case 0xd05: F2i(op); return;
|
|
case 0xb82: Ldc(); return;
|
|
case 0x381: case 0x981: Ldg(op); return;
|
|
case 0x386: case 0x986: Stg(op); return;
|
|
case 0xb66: Tld(); return;
|
|
case 0xb6c: Txd(); return;
|
|
|
|
case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xc21: case 0xe21: Fadd(); return;
|
|
case 0x220: case 0x420: case 0x620: case 0x820: case 0xa20: case 0xc20: Fmul(); return;
|
|
case 0x223: case 0x423: case 0x623: case 0x823: case 0xa23: case 0xc23: case 0xe23: Ffma(); return;
|
|
case 0x209: case 0x409: case 0x609: case 0x809: case 0xa09: case 0xc09: Fmnmx(); return;
|
|
case 0x308: case 0x908: case 0xb08: case 0xd08: Mufu(op); return;
|
|
case 0x307: case 0xb07: Frnd(op); return;
|
|
case 0x20b: case 0x40b: case 0x60b: case 0x80b: case 0xa0b: case 0xc0b: Setp(true); return;
|
|
case 0x20c: case 0x40c: case 0x60c: case 0x80c: case 0xa0c: case 0xc0c: Setp(false); return;
|
|
case 0x20a: case 0x40a: case 0x60a: case 0x80a: case 0xa0a: case 0xc0a: Fset(); return;
|
|
case 0x207: case 0x807: case 0xa07: case 0xc07: Sel_(false); return;
|
|
case 0x208: case 0x808: case 0xa08: case 0xc08: Sel_(true); return;
|
|
case 0x245: case 0x845: case 0xa45: case 0xc45: I2fp(); return;
|
|
|
|
case 0x210: case 0x410: case 0x610: case 0x810: case 0xa10: case 0xc10: Iadd3_(); return;
|
|
case 0x224: case 0x424: case 0x624: case 0x824: case 0xa24: case 0xc24: case 0xe24: Imad(); return;
|
|
case 0x212: case 0x812: case 0xa12: case 0xc12: Lop3(); return;
|
|
case 0x217: case 0x817: case 0xa17: case 0xc17: Imnmx(); return;
|
|
case 0x219: case 0x419: case 0x619: case 0x819: case 0xa19: case 0xc19: Shf(); return;
|
|
case 0x23e: case 0x43e: case 0x83e: case 0xa3e: case 0xc3e: F2fp(); return;
|
|
case 0x216: case 0x416: case 0x616: case 0x816: case 0xa16: case 0xc16: Prmt(); return;
|
|
case 0x309: case 0xd09: Popc(); return;
|
|
case 0x300: case 0xd00: Flo(); return;
|
|
case 0x301: Brev(); return;
|
|
case 0x240: case 0x840: Hmnmx2(); return;
|
|
case 0x234: case 0x434: Hsetp2(); return;
|
|
case 0x803: case 0x804: PredicateRegister(op == 0x803); return;
|
|
case 0x59f: Sured(); return;
|
|
case 0x593: Suatom(); return;
|
|
case 0x3a9: AtomgCas(); return;
|
|
case 0x3c4: Redux(); return;
|
|
case 0x81a: Sgxt(); return;
|
|
case 0x806: case 0x886: Vote(op == 0x886); return;
|
|
case 0x3a1: Match(); return;
|
|
case 0xf8c: AtomsPopcInc(); return;
|
|
case 0x3a8: case 0x9a8: Atomg(op == 0x9a8); return;
|
|
case 0x38c: Atoms(); return;
|
|
case 0x98e: Red(); return;
|
|
case 0x389: case 0x589: case 0x989: case 0xf89: Shfl(op); return;
|
|
case 0x822: Fswzadd(); return;
|
|
case 0x230: case 0x430: case 0x630: Half(0x5d10000000000000UL); return; // HADD2
|
|
case 0x232: case 0x832: case 0xa32: Half(0x5d08000000000000UL); return; // HMUL2
|
|
case 0x231: case 0x431: case 0x631: case 0x831: case 0xa31: Half(0x5d00000000000000UL); return; // HFMA2
|
|
case 0xb9b: SurfaceAccess(true); return;
|
|
case 0xb97: SurfaceAccess(false); return;
|
|
case 0x211: case 0x811: case 0xa11: case 0xc11: Lea(); return;
|
|
|
|
case 0x321: case 0x322: AttributeAccess(op == 0x322); return;
|
|
case 0x326: Ipa(); return;
|
|
case 0x805: Cs2r(); return;
|
|
case 0x919: case 0x9c3: S2r(op == 0x9c3); return;
|
|
case 0x927: Isbewr(); return;
|
|
case 0x949: Brx(false); return;
|
|
case 0x958: Brx(true); return;
|
|
case 0x923: Isberd(); return;
|
|
case 0x3c2: R2ur(); return;
|
|
case 0x225: case 0x227: case 0x625: case 0x825: case 0x827: case 0xa27: ImadWide(op); return;
|
|
case 0x213: case 0xa13: case 0xc13: Iabs(); return;
|
|
case 0xb6f: Txq(); return;
|
|
case 0xb60: case 0xf60: case 0x361: Tex(op); return;
|
|
case 0xb69: case 0xf69: case 0x36a: Tmml(op); return;
|
|
case 0xf66: TldBindless(); return;
|
|
case 0x370: TxqBindless(); return;
|
|
case 0x95b: Kill(); return;
|
|
case 0x94d: Exit(); return;
|
|
case 0x947: Bra(); return;
|
|
case 0x944: Call(); return;
|
|
case 0x950: Ret(); return;
|
|
case 0x2bd: Uflo(); return;
|
|
case 0x2be: Ubrev(); return;
|
|
case 0x2bf: Upopc(); return;
|
|
case 0x883: Up2ur(); return;
|
|
case 0x243: F2ipByte(); return;
|
|
case 0x23c: Hmma(); return;
|
|
case 0x23a: Movm(); return;
|
|
case 0x306: case 0xb06: case 0xd06: I2f(op); return;
|
|
case 0x387: case 0x987: case 0x983: LocalAccess(op != 0x983); return;
|
|
case 0x526: IpaIndexed(); return;
|
|
case 0xabb: UldcIndexed(); return;
|
|
}
|
|
throw Fail("opcode not lowered yet");
|
|
}
|
|
|
|
// ---- moves and uniform datapath ----------------------------------------------------------------------------
|
|
void Mov_(uint op)
|
|
{
|
|
if (_i.Bits(72, 4) != 15) throw Fail("MOV lane mask");
|
|
int d = Gpr((uint)Dst);
|
|
switch (op)
|
|
{
|
|
case 0x802: Mov32i(d, _i.W1, true); return;
|
|
case 0x202: Mov(d, Gpr(_i.Bits(32, 8))); return;
|
|
case 0xc02: Mov(d, Ur(_i.Bits(32, 8))); return;
|
|
default: { var (bank, off) = Cbuf(); MovC(d, bank, off, true); return; }
|
|
}
|
|
}
|
|
|
|
void Umov(uint op)
|
|
{
|
|
RequireUniformGuardTrue();
|
|
int d = Ur((uint)Dst);
|
|
if (d == RZ) return;
|
|
if (op == 0x882) Mov32i(d, _i.W1, true);
|
|
else EmitG(0x5c98000000000000UL | (15UL << 39) | ((ulong)Ur(_i.Bits(32, 8)) << 20) | (uint)d);
|
|
}
|
|
|
|
// ULDC: width (bits 73-75) 4 = 32 bit, 5 = 64 bit into a uniform pair.
|
|
void Uldc()
|
|
{
|
|
RequireUniformGuardTrue();
|
|
uint width = _i.Bits(73, 3);
|
|
int d = Ur((uint)Dst);
|
|
if (width < 4) // U8, S8, U16, S16: Maxwell LDC with the same width code, immediate address (any alignment)
|
|
{
|
|
// (this form, 0xab9, has no register index: bits 24-29 are not read, as for the 32- and 64-bit loads
|
|
// below; compilers leave 0 there as well as 63, and the 0 cases failed. 0xabb is the indexed form.)
|
|
int cb = Bank(_i.Bits(54, 5)), at = (int)_i.Bits(38, 16);
|
|
if (cb >= 18 || at >= 0x8000) throw Fail($"ULDC width {width} c[{_i.Bits(54, 5)}][0x{at:x}]");
|
|
if (d != RZ)
|
|
EmitG(0xef90000000000000UL | ((ulong)RZ << 8) | ((ulong)at << 20) | ((ulong)cb << 36) | ((ulong)width << 48) | (uint)d);
|
|
return;
|
|
}
|
|
var (bank, off) = Cbuf();
|
|
if (width == 4) { if (d != RZ) MovC(d, bank, off, true); return; }
|
|
if (width == 5 && (Dst & 1) == 0) { if (d != RZ) { MovC(d, bank, off, true); MovC(d + 1, bank, off + 4, true); } return; }
|
|
throw Fail($"ULDC width {width}");
|
|
}
|
|
|
|
// Uniform-datapath arithmetic (UIADD3, UIMAD, UIMAD.WIDE, ULOP3, USHF, ULEA, USEL) has the encoding of its vector
|
|
// twin (base opcode - 0x80) with uniform registers in the register fields and bit 91 set; it is lowered as the
|
|
// twin with the fields read as uniform registers; its predicate fields name uniform predicates, which Run has
|
|
// swapped into the P registers of the same numbers.
|
|
void UniformAlu(uint op)
|
|
{
|
|
uint twin = (op & 0x1ff) - 0x80;
|
|
Insn saved = _i;
|
|
_i.Lo = (_i.Lo & ~0x1ffUL) | twin;
|
|
_i.Hi &= ~(1UL << 27); // bit 91
|
|
_uniform = true;
|
|
try { LowerOne(); }
|
|
finally { _uniform = false; _i = saved; }
|
|
}
|
|
|
|
// ---- floating point ----------------------------------------------------------------------------------------
|
|
void Fadd()
|
|
{
|
|
if (_i.Bits(81, 10) != 0) throw Fail("FADD mode");
|
|
var (a, aAbs, aNeg) = Raw(0, SA);
|
|
var (b, bAbs, bNeg) = Raw(SecondSlot, SB);
|
|
EmitG(0x5c58000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)
|
|
| (aNeg ? 1UL << 48 : 0) | (aAbs ? 1UL << 46 : 0) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0)
|
|
| ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) | ((ulong)_i.Bits(77, 1) << 50));
|
|
}
|
|
|
|
// FMUL: scale (bits 84-86) 1-3 divide by 8/4/2, 4 none, 5-7 multiply by 2/4/8; Maxwell's scale field is 0 none,
|
|
// 1-3 divide by 2/4/8, 4-6 multiply by 8/4/2.
|
|
static readonly uint[] FmulScale = { 0, 3, 2, 1, 0, 6, 5, 4 };
|
|
void Fmul()
|
|
{
|
|
if (_i.Bit(76) || _i.Bits(81, 3) != 0 || _i.Bits(87, 4) != 0 || _i.Bits(84, 3) == 0) throw Fail("FMUL mode");
|
|
var (a, aAbs, aNeg) = Raw(0, SA);
|
|
var (b, bAbs, bNeg) = Raw(1, SB);
|
|
a = Mat(a, aAbs, false, SA);
|
|
b = Mat(b, bAbs, false, SB);
|
|
EmitG(0x5c68000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)
|
|
| (aNeg != bNeg ? 1UL << 48 : 0) | ((ulong)FmulScale[_i.Bits(84, 3)] << 41)
|
|
| ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) | ((ulong)_i.Bits(77, 1) << 50));
|
|
}
|
|
|
|
void Ffma()
|
|
{
|
|
if (_i.Bit(76) || _i.Bits(81, 10) != 0) throw Fail("FFMA mode");
|
|
var (a, aAbs, aNeg) = Raw(0, SA);
|
|
var (b, bAbs, bNeg) = Raw(1, SB);
|
|
var (c, cAbs, cNeg) = Raw(2, SC);
|
|
a = Mat(a, aAbs, false, SA);
|
|
b = Mat(b, bAbs, false, SB);
|
|
c = Mat(c, cAbs, false, SC);
|
|
EmitG(0x5980000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)
|
|
| (aNeg != bNeg ? 1UL << 48 : 0) | (cNeg ? 1UL << 49 : 0)
|
|
| ((ulong)_i.Bits(77, 1) << 50) | ((ulong)_i.Bits(78, 2) << 51) | ((ulong)_i.Bits(80, 1) << 53));
|
|
}
|
|
|
|
// FMNMX: Rd = p ? min : max (p in bits 87-90). .NAN (bit 81) is not lowered.
|
|
void Fmnmx()
|
|
{
|
|
if (_i.Bits(74, 6) != 0 || _i.Bits(81, 6) != 0) throw Fail("FMNMX mode");
|
|
var (a, aAbs, aNeg) = Raw(0, SA);
|
|
var (b, bAbs, bNeg) = Raw(SecondSlot, SB);
|
|
EmitG(0x5c60000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)
|
|
| (aNeg ? 1UL << 48 : 0) | (aAbs ? 1UL << 46 : 0) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0)
|
|
| ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(80, 1) << 44));
|
|
}
|
|
|
|
// MUFU: operation in bits 74-79, numbered as Maxwell's (0 COS, 1 SIN, 2 EX2, 3 LG2, 4 RCP, 5 RSQ, 8 SQRT).
|
|
// SM70+ COS/SIN take the angle in turns (the compiler multiplies by 1/2pi first); the translator's take radians.
|
|
// MUFU.F16 (bit 73, register form): the half in the low or, with bit 60, the high 16 bits of Rb, abs 62 and
|
|
// neg 63; the result is a half in the low 16 bits. The upscaler's colour decode raises halves to a power this
|
|
// way (LG2.F16, HMUL2, EX2.F16, the two lanes packed by PRMT). Lowered as F2F to F32, the F32 function and F2F
|
|
// back to F16 (round to nearest), which writes the half with the high 16 bits clear.
|
|
void Mufu(uint op)
|
|
{
|
|
uint operation = _i.Bits(74, 6);
|
|
bool half = _i.Bit(73);
|
|
if (operation > 8 || operation is 6 or 7 || (half && (op != 0x308 || _i.Bit(61))))
|
|
throw Fail($"MUFU operation {operation}{(half ? ".F16" : "")}");
|
|
int src; bool abs, neg;
|
|
if (op == 0x908) { Mov32i(SB, _i.W1); (src, abs, neg) = (SB, false, false); }
|
|
else if (half)
|
|
{
|
|
Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)Gpr(_i.Bits(32, 8)) << 20)
|
|
| (_i.Bit(60) ? 1UL << 41 : 0) | (_i.Bit(63) ? 1UL << 45 : 0) | (_i.Bit(62) ? 1UL << 49 : 0) | SB)); // F2F.F32.F16
|
|
(src, abs, neg) = (SB, false, false);
|
|
}
|
|
else (src, abs, neg) = Raw(1, SB);
|
|
if (operation <= 1)
|
|
{
|
|
src = Mat(src, abs, neg, SB);
|
|
Fmul32i(SB, src, 0x40c90fdb); // 2pi
|
|
src = SB; abs = neg = false;
|
|
}
|
|
ulong mufu = 0x5080000000000000UL | ((ulong)operation << 20) | ((ulong)src << 8) | (abs ? 1UL << 46 : 0) | (neg ? 1UL << 48 : 0);
|
|
if (!half) { EmitG(mufu | (uint)Gpr((uint)Dst)); return; }
|
|
Emit(Unguarded(mufu | SB));
|
|
EmitG(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)SB << 20) | (uint)Gpr((uint)Dst)); // F2F.F16.F32
|
|
}
|
|
|
|
// FRND.F32: rounding (bits 78-79: nearest/floor/ceil/trunc) as Maxwell F2F.F32.F32 with integer rounding.
|
|
void Frnd(uint op)
|
|
{
|
|
if (_i.Bits(75, 2) != 2 || _i.Bits(84, 2) != 2 || _i.Bits(81, 3) != 0 || _i.Bits(86, 6) != 0) throw Fail("FRND format");
|
|
var (b, bAbs, bNeg) = Raw(1, SB);
|
|
EmitG(0x5ca8000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)b << 20) | (uint)Gpr((uint)Dst)
|
|
| ((ulong)(8 + _i.Bits(78, 2)) << 39) | ((ulong)_i.Bits(80, 1) << 44) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0));
|
|
}
|
|
|
|
// FSETP / ISETP: P (bits 81-83), Q (84-86), combined with Pc (87-90) by the operation in 74-75.
|
|
void Setp(bool floating)
|
|
{
|
|
if (!floating && _i.Bit(72)) { SetpExtended(); return; }
|
|
uint combine = _i.Bits(74, 2);
|
|
if (combine > 2) throw Fail("predicate combination");
|
|
var (a, aAbs, aNeg) = Raw(0, SA);
|
|
var (b, bAbs, bNeg) = Raw(1, SB);
|
|
ulong w = (floating ? 0x5bb0000000000000UL : 0x5b60000000000000UL) | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3)
|
|
| ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)combine << 45);
|
|
if (floating)
|
|
w |= ((ulong)_i.Bits(80, 1) << 47) | ((ulong)_i.Bits(76, 4) << 48)
|
|
| (aNeg ? 1UL << 43 : 0) | (aAbs ? 1UL << 7 : 0) | (bNeg ? 1UL << 6 : 0) | (bAbs ? 1UL << 44 : 0);
|
|
else
|
|
w |= ((ulong)_i.Bits(73, 1) << 48) | ((ulong)_i.Bits(76, 3) << 49);
|
|
EmitG(w);
|
|
}
|
|
|
|
// FSET.BF: Rd = 1.0 or 0.0.
|
|
void Fset()
|
|
{
|
|
if (_i.Bits(64, 8) != 0 || _i.Bits(74, 2) == 3 || _i.Bits(81, 6) != 0) throw Fail("FSET output mode");
|
|
var (a, aAbs, aNeg) = Raw(0, SA);
|
|
var (b, bAbs, bNeg) = Raw(1, SB);
|
|
EmitG(0x5800000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)
|
|
| ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(74, 2) << 45)
|
|
| ((ulong)_i.Bits(76, 4) << 48) | (1UL << 52) | ((ulong)_i.Bits(80, 1) << 55)
|
|
| (aNeg ? 1UL << 43 : 0) | (bNeg ? 1UL << 53 : 0) | (aAbs ? 1UL << 54 : 0) | (bAbs ? 1UL << 44 : 0));
|
|
}
|
|
|
|
// SEL / FSEL: Rd = p ? a : b (p in bits 87-90). FSEL applies the float modifiers first.
|
|
void Sel_(bool floating)
|
|
{
|
|
int a = floating ? FloatMat(0, SA) : Plain(0, SA);
|
|
int b = floating ? FloatMat(1, SB) : Plain(1, SB);
|
|
Sel(Gpr((uint)Dst), a, b, _i.Bits(87, 4), true);
|
|
}
|
|
|
|
// I2FP.F32.{U32,S32}: bit 74 signed, rounding in 78-79.
|
|
void I2fp()
|
|
{
|
|
uint w2 = _i.W2 & ~(0x400u | 0xc000u | 0x08000000u);
|
|
if (w2 != 0x00201000) throw Fail("I2FP format");
|
|
int b = Plain(1, SB);
|
|
EmitG(0x5cb8000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)_i.Bits(74, 1) << 13) | ((ulong)b << 20)
|
|
| ((ulong)_i.Bits(78, 2) << 39) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// ---- integer -----------------------------------------------------------------------------------------------
|
|
// IADD3 Rd, [Pu,] a, b, c. Negations are arithmetic (Maxwell IADD3's too). One carry out (bits 81-83) with c = RZ,
|
|
// or .X (bit 74) adding the carry-in predicate of bits 87-90.
|
|
void Iadd3_()
|
|
{
|
|
uint op = _i.Bits(0, 12);
|
|
bool negA = _i.Bit(72), negB = op != 0x810 && _i.Bit(63) && Form is not 2 and not 3, negC = _i.Bit(75), extended = _i.Bit(74);
|
|
uint carry1 = _i.Bits(81, 3), carry2 = _i.Bits(84, 3);
|
|
int a = Plain(0, SA), b = Plain(1, SB), c = Plain(2, SC);
|
|
int d = Gpr((uint)Dst);
|
|
if (Form is 2 or 3 && (_i.Bit(63) || negC)) throw Fail("IADD3 negation with a wide third source");
|
|
if (!extended && carry1 == 7 && carry2 == 7)
|
|
{
|
|
EmitG(0x5cc0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)d
|
|
| (negA ? 1UL << 51 : 0) | (negB ? 1UL << 50 : 0) | (negC ? 1UL << 49 : 0));
|
|
return;
|
|
}
|
|
// The 34-bit sum: low word in R244, carry count in R245 (IADD.CC / IADD.X). A negated source is ~x, plus 1
|
|
// unless .X, whose carry-ins (bits 87-90 and 77-80) are added instead. Pu = bit 32, Pv = bit 33.
|
|
const int lo = 244, hi = 245, one = 246;
|
|
int inv(int r, bool neg, int t) { if (!neg) return r; Lop32i(t, r, 0xffffffff, 2); return t; }
|
|
a = inv(a, negA, 241); b = inv(b, negB, 242); c = inv(c, negC, 243);
|
|
void addc(int r)
|
|
{
|
|
Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)r << 20) | ((ulong)lo << 8) | lo)); // IADD.CC
|
|
Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)RZ << 20) | ((ulong)hi << 8) | hi)); // IADD.X
|
|
}
|
|
Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)b << 20) | ((ulong)a << 8) | lo));
|
|
Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)RZ << 20) | ((ulong)RZ << 8) | hi));
|
|
addc(c);
|
|
if (extended)
|
|
foreach (uint p in new[] { _i.Bits(87, 4), _i.Bits(77, 4) })
|
|
{
|
|
if (p == 15) continue;
|
|
Mov32i(one, 1);
|
|
if (p != 7) Sel(one, one, RZ, p, false);
|
|
addc(one);
|
|
}
|
|
else if (negA || negB || negC)
|
|
{
|
|
Mov32i(one, (uint)((negA ? 1 : 0) + (negB ? 1 : 0) + (negC ? 1 : 0)));
|
|
addc(one);
|
|
}
|
|
if (d != RZ) Mov(d, lo);
|
|
uint guard = _i.Bits(12, 3);
|
|
foreach (int k in guard == carry1 ? new[] { 1, 0 } : new[] { 0, 1 }) // an output that is the guard goes last
|
|
{
|
|
uint p = k == 0 ? carry1 : carry2;
|
|
if (p == 7) continue;
|
|
Lop32i(one, hi, k == 0 ? 1u : 2u, 0);
|
|
EmitG(0x5b60000000000000UL | (5UL << 49) | ((ulong)PT << 39) | ((ulong)RZ << 20) | ((ulong)one << 8) | ((ulong)p << 3) | PT);
|
|
}
|
|
}
|
|
|
|
// IMAD Rd, a, b, c (low word); bit 73 signed, bit 75 negates c (~c with .X), .X (bit 74) adds the carry-in
|
|
// predicate of bits 87-90. A carry out (bits 81-83) is not lowered; .HI/.WIDE are ImadWide.
|
|
void Imad()
|
|
{
|
|
uint known = 0xffu | 1u << 9 | 1u << 10 | 1u << 11 | 7u << 17 | 0xfu << 23 | (Form is 6 or 7 ? 1u << 27 : 0u);
|
|
bool x = _i.Bit(74);
|
|
if ((_i.W2 & ~known) != 0 || _i.Bits(81, 3) != 7 || (!x && _i.Bits(87, 4) != 15)) throw Fail("IMAD mode");
|
|
int a = Plain(0, SA), b = Plain(1, SB), c = Plain(2, SC);
|
|
if (_i.Bit(75) && c != RZ)
|
|
{
|
|
if (x) Lop32i(SC, c, 0xffffffff, 2); // .X: ~c, the +1 comes as the carry
|
|
else Iadd3(SC, RZ, c, RZ, false, negB: true);
|
|
c = SC;
|
|
}
|
|
ulong signed = _i.Bit(73) ? (1UL << 48) | (1UL << 53) : 0;
|
|
int d = Gpr((uint)Dst);
|
|
ulong imad(int dst) => 0x5a00000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst | signed;
|
|
if (!x) { EmitG(imad(d)); return; }
|
|
Mov32i(SX, 1);
|
|
Sel(SX, SX, RZ, _i.Bits(87, 4), false);
|
|
Emit(Unguarded(imad(SA)));
|
|
Iadd3(d, SA, SX, RZ, true);
|
|
}
|
|
|
|
// IMAD.HI Rd / IMAD.WIDE Rd:Rd+1 = a * b + (c+1:c), bit 73 signed. b: a register (bits 32-39), an immediate
|
|
// (0x8xx) or a constant (0xa27); 0x625 takes b from bits 64-71 and adds a constant pair. The 64-bit sum uses
|
|
// Maxwell IMUL.LO/.HI and IADD.CC/.X into R247:R248. A carry out (bits 81-83) is not lowered.
|
|
void ImadWide(uint op)
|
|
{
|
|
bool wide = op is 0x225 or 0x625 or 0x825, signedMul = _i.Bit(73);
|
|
if ((_i.W2 & ~0x000e02ffu) != 0x07800000 || _i.Bits(81, 3) != 7) throw Fail("IMAD.HI/WIDE mode");
|
|
int a = Gpr(_i.Bits(24, 8)), b, cLo, cHi;
|
|
if (op is 0x825 or 0x827) { Mov32i(SB, _i.W1); b = SB; }
|
|
else if (op == 0xa27) { var (bank, off) = Cbuf(); MovC(SB, bank, off); b = SB; }
|
|
else b = Gpr(_i.Bits(op == 0x625 ? 64 : 32, 8));
|
|
if (op == 0x625) { var (bank, off) = Cbuf(); MovC(SC, bank, off); MovC(SX, bank, off + 4); cLo = SC; cHi = SX; }
|
|
else
|
|
{
|
|
int c = Gpr(_i.Bits(64, 8));
|
|
if (c != RZ && (c & 1) != 0) throw Fail("IMAD.WIDE odd addend pair");
|
|
cLo = c; cHi = c == RZ ? RZ : c + 1;
|
|
}
|
|
int d = Gpr((uint)Dst);
|
|
if (wide && (d == RZ || (d & 1) != 0)) throw Fail("IMAD.WIDE destination pair");
|
|
ulong sign = signedMul ? (1UL << 40) | (1UL << 41) : 0;
|
|
Emit(Unguarded(0x5c38000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | 247 | sign)); // IMUL lo
|
|
Emit(Unguarded(0x5c38000000000000UL | (1UL << 39) | ((ulong)b << 20) | ((ulong)a << 8) | 248 | sign)); // IMUL.HI
|
|
Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)cLo << 20) | (247UL << 8) | 247)); // IADD.CC
|
|
Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)cHi << 20) | (248UL << 8) | 248)); // IADD.X
|
|
if (wide) { Mov(d, 247); Mov(d + 1, 248); }
|
|
else Mov(d, 248);
|
|
}
|
|
|
|
// IABS Rd, b: max(b, -b), signed.
|
|
void Iabs()
|
|
{
|
|
uint op = _i.Bits(0, 12);
|
|
if ((_i.W2 & ~(op == 0xc13 ? 0x08000000u : 0u)) != 0) throw Fail("IABS mode");
|
|
int b = Plain(1, SB);
|
|
Iadd3(SX, RZ, b, RZ, false, negB: true);
|
|
EmitG(0x5c20000000000000UL | ((ulong)SX << 20) | ((ulong)b << 8) | (uint)Gpr((uint)Dst) | (15UL << 39) | (1UL << 48));
|
|
}
|
|
|
|
// LOP3.LUT: table in bits 72-79, optional P = result != 0 (bits 81-83).
|
|
void Lop3()
|
|
{
|
|
if (_i.Bit(80) || _i.Bits(87, 4) != 15) throw Fail("LOP3 predicate mode");
|
|
int a = Plain(0, SA), b = Plain(1, SB), c = Gpr(_i.Bits(64, 8));
|
|
uint p = _i.Bits(81, 3);
|
|
EmitG(0x5be0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)
|
|
| ((ulong)_i.Bits(72, 8) << 28) | ((ulong)(p == 7 ? 0 : 3) << 36) | ((ulong)p << 48));
|
|
}
|
|
|
|
// IMNMX: Rd = p ? min : max (p in bits 87-90), bit 73 signed.
|
|
void Imnmx()
|
|
{
|
|
int a = Plain(0, SA), b = Plain(1, SB);
|
|
EmitG(0x5c20000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)
|
|
| ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(73, 1) << 48));
|
|
}
|
|
|
|
// SHF Rd, low (A), shift (B), high (C): bits 73-74 type (0 S64, 1 U64, 2 S32, 3 U32), 75 .W (wrap), 76 right,
|
|
// 80 .HI (the high word of the 64-bit result).
|
|
void Shf()
|
|
{
|
|
if (_i.Bits(81, 10) != 0) throw Fail("SHF mode");
|
|
uint type = _i.Bits(73, 2);
|
|
bool wrap = _i.Bit(75), right = _i.Bit(76), hi = _i.Bit(80), wide = type <= 1, signedType = type == 0 || type == 2;
|
|
int low = Plain(0, SA), s = Plain(1, SB), high = Plain(2, SC);
|
|
int d = Gpr((uint)Dst);
|
|
ulong maxShift = type == 0 ? 3UL : type == 1 ? 2UL : 0UL;
|
|
if (!right && hi) // high word of (high:low) << s
|
|
EmitG(0x5bf8000000000000UL | ((ulong)high << 39) | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 50 : 0) | (maxShift << 37));
|
|
else if (!right) // low word: low << s
|
|
{
|
|
if (wrap && wide) throw Fail("SHF.L.W 64-bit low word");
|
|
EmitG(0x5c48000000000000UL | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 39 : 0));
|
|
}
|
|
else if (!hi) // low word of (high:low) >> s
|
|
EmitG(0x5cf8000000000000UL | ((ulong)high << 39) | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 50 : 0) | (maxShift << 37));
|
|
else // high word: high >> s
|
|
{
|
|
if (wrap && wide) throw Fail("SHF.R.W 64-bit high word");
|
|
EmitG(0x5c28000000000000UL | ((ulong)s << 20) | ((ulong)high << 8) | (uint)d | (wrap ? 1UL << 39 : 0) | (signedType ? 1UL << 48 : 0));
|
|
}
|
|
}
|
|
|
|
// LEA Rd, [Pu,] a, b, s: (a << s) + b, shift in bits 75-79, bit 63 negates b. LEA.HI (bit 80): b + high word of
|
|
// (c:a) << s (bit 73 sign-extends a when c is RZ), .X (bit 74) adds the carry predicate of bits 87-90.
|
|
void Lea()
|
|
{
|
|
uint op = _i.Bits(0, 12);
|
|
int a = Plain(0, SA), b = Plain(1, SB), d = Gpr((uint)Dst);
|
|
uint shift = _i.Bits(75, 5), carry = _i.Bits(81, 3);
|
|
bool negB = op != 0x811 && _i.Bit(63);
|
|
if (_i.Bit(72)) throw Fail("LEA negated a");
|
|
if (!_i.Bit(80))
|
|
{
|
|
if (_i.Bit(74)) throw Fail("LEA.X without .HI");
|
|
ulong iscadd(int dst, int bb) => 0x5c18000000000000UL | ((ulong)shift << 39) | ((ulong)bb << 20) | ((ulong)a << 8) | (uint)dst | (negB ? 1UL << 48 : 0);
|
|
if (carry == 7) { EmitG(iscadd(d, b)); return; }
|
|
if (negB) throw Fail("LEA carry out with negated b");
|
|
// The sum goes to scratch unconditionally; only the destination and the carry are guarded. A guarded
|
|
// copy of b made a constant-buffer base (c[0][0x110] in Ocarina's edge-AA tile list, prog 98) a phi
|
|
// of it and the scratch register's old value, and GlobalToStorage could not find the buffer.
|
|
Mov(SX, b, false);
|
|
Emit(Unguarded(iscadd(SC, SX)));
|
|
if (d != RZ) Mov(d, SC);
|
|
IsetpLtU((int)carry, SC, SX); // carry = sum < b
|
|
return;
|
|
}
|
|
if (carry != 7) throw Fail("LEA.HI carry out");
|
|
int high = Gpr(_i.Bits(64, 8));
|
|
// With c = RZ the high word is a right shift of a by 32 - s (arithmetic when bit 73 sign-extends a);
|
|
// otherwise SHF.L.HI of (c:a) by s.
|
|
Mov32i(SX, high == RZ ? 32 - shift : shift);
|
|
if (high == RZ)
|
|
EmitG(0x5c28000000000000UL | ((ulong)SX << 20) | ((ulong)a << 8) | SC | (_i.Bit(73) ? 1UL << 48 : 0));
|
|
else
|
|
EmitG(0x5bf8000000000000UL | ((ulong)high << 39) | ((ulong)SX << 20) | ((ulong)a << 8) | SC);
|
|
if (_i.Bit(74))
|
|
{
|
|
Mov32i(SX, 1);
|
|
Sel(SX, SX, RZ, _i.Bits(87, 4), false);
|
|
Iadd3(SC, SC, SX, RZ, true);
|
|
}
|
|
Iadd3(d, SC, b, RZ, true, negB);
|
|
}
|
|
|
|
// ---- attributes, special registers ------------------------------------------------------------------------
|
|
// ALD / AST: address bits 40-49, components 74-75 (+1), patch 76, output 79; vertex register 32 (ALD) / 64 (AST).
|
|
void AttributeAccess(bool store)
|
|
{
|
|
bool patch = _i.Bit(76), output = _i.Bit(79);
|
|
uint address = _i.Bits(40, 10), components = _i.Bits(74, 2) + 1;
|
|
if (_i.Bit(77) || patch || (output && store) || (address & 3) != 0 || address + components * 4 > 0x400 || _i.Bits(24, 8) != RZ)
|
|
throw Fail("indexed/patch attribute access");
|
|
// a[0x5c] is an Ampere per-primitive hint (shading rate); nothing reads it back.
|
|
if (store && address == 0x5c && components == 1) return;
|
|
int value = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst);
|
|
if (value == RZ)
|
|
{
|
|
if (!store) return;
|
|
for (int k = 0; k < components; k++) Mov32i(TexOut + k, 0);
|
|
value = TexOut;
|
|
}
|
|
int vertex = Gpr(store ? _i.Bits(64, 8) : _i.Bits(32, 8));
|
|
EmitG((store ? 0xeff0000000000000UL : 0xefd8000000000000UL) | (uint)value | ((ulong)RZ << 8) | ((ulong)address << 20)
|
|
| (output ? 1UL << 32 : 0) | ((ulong)vertex << 39) | ((ulong)(components - 1) << 47));
|
|
}
|
|
|
|
// IPA: attribute (bits 64-71, in words), sample mode 76-77 (default/centroid), bit 78 flat. Maxwell IPA.PASS:
|
|
// like Maxwell's, SM86 IPA of a perspective input returns the value times 1/w, and the NVN2 compiler multiplies
|
|
// by w itself (IPA a[0x7c], MUFU.RCP, FMUL). Treating it as perspective-correct and adding IPA.MULTIPLY by w
|
|
// scaled every texture coordinate by w a second time: Ocarina's floors and walls smeared along the view.
|
|
void Ipa()
|
|
{
|
|
if (_i.Bits(32, 8) != RZ || _i.Bits(76, 2) > 1 || _i.Bit(79) || _i.Bits(81, 3) != 7) throw Fail("IPA mode");
|
|
EmitG(0xe000000000000000UL | ((ulong)RZ << 8) | ((ulong)RZ << 20) | ((ulong)_i.Bits(64, 8) << 30)
|
|
| ((ulong)_i.Bits(76, 2) << 52) | ((ulong)_i.Bits(78, 1) << 55) | ((ulong)PT << 47) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// CS2R Rd, SRZ: a zeroed register pair.
|
|
void Cs2r()
|
|
{
|
|
if (_i.Bits(72, 8) != RZ || !_i.Bit(80)) throw Fail("CS2R special register");
|
|
int d = Gpr((uint)Dst);
|
|
if (d == RZ) return;
|
|
Mov32i(d, 0, true);
|
|
Mov32i(d + 1, 0, true);
|
|
}
|
|
|
|
// ISBEWR [Ra + imm], Rb: a stage buffer write of a task or mesh program (MeshInfo): Ra 24-31, Rb 32-39, the byte
|
|
// offset 40-63, the region 75-78, bit 79 set (a write). Region 15 the vertex outputs, 0 the primitive indices
|
|
// (bytes), 1 the primitive count, 7 the task payload. A shared memory store here.
|
|
void Isbewr()
|
|
{
|
|
if (_mesh == null) throw Fail("ISBEWR outside a task or mesh program");
|
|
uint region = _i.Bits(75, 4), offset = _i.Bits(40, 24);
|
|
if (!_i.Bit(79) || (_i.W2 & ~0xf800u) != 0 || _i.Bits(64, 8) != 0 || offset >= 0x10000) throw Fail("ISBEWR mode");
|
|
int a = Gpr(_i.Bits(24, 8)), data = Gpr(_i.Bits(32, 8));
|
|
int at; uint width = 4;
|
|
if (_mesh.Stage == 7 && region == 15) at = _mesh.Attr + (int)offset;
|
|
else if (_mesh.Stage == 7 && region == 0) { at = _mesh.Map + (int)offset; width = 0; }
|
|
else if (_mesh.Stage == 7 && region == 1)
|
|
{
|
|
if (a != RZ) throw Fail("ISBEWR primitive count at a register address");
|
|
at = _mesh.Map;
|
|
}
|
|
else if (_mesh.Stage == 6 && region == 7) at = _mesh.Pay + (int)offset;
|
|
else throw Fail($"ISBEWR region {region} in a stage {_mesh.Stage} program");
|
|
EmitG(0xef58000000000000UL | (uint)data | ((ulong)a << 8) | ((ulong)(uint)at << 20) | ((ulong)width << 48));
|
|
}
|
|
|
|
// ISBERD Rd, [Ra + imm]: a stage buffer read (fields as ISBEWR's, bit 79 clear). Address 0 of region 15 (a mesh
|
|
// program without a task stage, a task program) or of region 7 (a mesh program after a task stage) is the
|
|
// workgroup's index; region 7 elsewhere is the task payload. With bit 79 set (the output side, as ISBEWR's) a
|
|
// task program reads back the payload it writes (AglShader's task program stores a word and reloads it).
|
|
void Isberd()
|
|
{
|
|
if (_mesh == null) throw Fail("ISBERD outside a task or mesh program");
|
|
uint region = _i.Bits(75, 4), offset = _i.Bits(40, 24);
|
|
bool output = _i.Bit(79);
|
|
if ((_i.W2 & ~0xf800u) != 0 || (output && (region != 7 || _mesh.Stage != 6)) || _i.Bits(32, 8) != 0 || _i.Bits(64, 8) != 0
|
|
|| offset >= 0x10000)
|
|
throw Fail("ISBERD mode");
|
|
int a = Gpr(_i.Bits(24, 8)), d = Gpr((uint)Dst);
|
|
if (d == RZ) return;
|
|
if (output)
|
|
{
|
|
EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | ((ulong)(uint)(_mesh.Pay + (int)offset) << 20) | (4UL << 48));
|
|
return;
|
|
}
|
|
if ((region == 15 || region == 7) && a == RZ && offset == 0)
|
|
{
|
|
Put(0xf0c8000000000000UL | (0x25UL << 20) | (uint)d, true); // S2R SR_CTAID.X
|
|
return;
|
|
}
|
|
if (region != 7 || _mesh.Stage != 7 || !_mesh.HasTask) throw Fail($"ISBERD region {region} at a nonzero address");
|
|
EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | ((ulong)(uint)(_mesh.Pay + (int)offset) << 20) | (4UL << 48));
|
|
}
|
|
|
|
// S2R / S2UR: special register in bits 72-79, numbered as Maxwell's (lane id 0, thread and workgroup ids
|
|
// 33-39, lane masks 56-60, ...). SR 132 (variable-rate shading: fragment width and height in bytes 1 and 2)
|
|
// reads as 1x1.
|
|
void S2r(bool uniform)
|
|
{
|
|
uint special = _i.Bits(72, 8);
|
|
if (_i.W1 != 0 || _i.W2 != special << 8) throw Fail("special register mode");
|
|
if (uniform) RequireUniformGuardTrue();
|
|
int d = uniform ? Ur((uint)Dst) : Gpr((uint)Dst);
|
|
if (d == RZ) return;
|
|
if (special == 132) { Mov32i(d, 0x00010100, true); return; }
|
|
Put(0xf0c8000000000000UL | ((ulong)special << 20) | (uint)d, true);
|
|
}
|
|
|
|
// R2UR: a uniform register takes the value of the first active lane. Uniform registers are per-thread here, so
|
|
// this is a move (the compiler emits R2UR for values that are uniform, or loops until they are).
|
|
void R2ur()
|
|
{
|
|
if (_i.W1 != 0 || _i.W2 != 0x000e0000) throw Fail("R2UR mode");
|
|
int d = Ur((uint)Dst);
|
|
if (d != RZ) Mov(d, Gpr(_i.Bits(24, 8)));
|
|
}
|
|
|
|
// ---- texture ---------------------------------------------------------------------------------------------
|
|
// TEX (bound): Rd 16-23, Ra 24-31, Rb 32-39, descriptor word offset 40-53, its bank 54-58, dimension 61-63
|
|
// (1D, 2D, 3D, CUBE, ARRAY_1D, ARRAY_2D, -, ARRAY_CUBE), second destination 64-71, mask 72-75, offsets 76,
|
|
// depth compare 78, FP16 results 79, LOD mode 87-89 (numbered as Maxwell's), bit 90 no-dependency.
|
|
// Operands: array index, coordinates, then LOD/bias, offsets, depth reference; up to four values are split as
|
|
// halves over Ra and Rb (1+1, 2+1, 2+2), five or more as four in Ra and the rest in Rb. Maxwell TEX takes the
|
|
// coordinates in Ra.. and the rest in Rb.., and writes the masked components to consecutive registers.
|
|
static readonly int[] TexCoords = { 1, 2, 3, 3, 2, 3, 0, 4 };
|
|
static readonly int[] TexMaxwellDim = { 0, 2, 4, 6, 1, 3, 0, 7 };
|
|
// TEX. 0xb60 a bound descriptor; the bindless forms take the 32-bit handle (texture index in bits 0-19, sampler
|
|
// index above) from a uniform register in bits 40-45 (0xf60) or, with bit 59 (.B) set, as one more operand
|
|
// right after the coordinates (0x361; with bit 60 clear Ra holds all coordinates and Rb the handle and the
|
|
// rest). Bindless becomes Maxwell TEX.B, the handle first among the meta registers (offsets bit 36, LOD mode 37):
|
|
// the translator then indexes its texture-pool arrays with it (BindlessElimination GenerateBindlessAccess,
|
|
// the handle being loaded from memory).
|
|
void Tex(uint op)
|
|
{
|
|
bool bindless = op != 0xb60;
|
|
uint dim = _i.Bits(61, 3), lod = _i.Bits(87, 3), mask = _i.Bits(72, 4);
|
|
bool offsets = _i.Bit(76), dc = _i.Bit(78);
|
|
if (dim == 6 || _i.Bit(77) || _i.Bits(81, 3) != 7 || _i.Bit(59) != (op == 0x361) || (lod > 3 && lod != 6 && lod != 7) || mask == 0)
|
|
throw Fail("TEX mode");
|
|
uint handle = 0;
|
|
int handleReg = RZ;
|
|
if (op == 0xb60)
|
|
{
|
|
handle = Handle(false);
|
|
uint bank = _i.Bits(54, 5);
|
|
if (handle > 0x1fff) throw Fail("TEX descriptor offset");
|
|
if (!TexBank(bank)) throw Fail($"TEX descriptor bank {bank}");
|
|
}
|
|
else if (op == 0xf60)
|
|
{
|
|
if (_i.Bits(46, 13) != 0) throw Fail("bindless TEX mode");
|
|
handleReg = Ur(_i.Bits(40, 6));
|
|
}
|
|
else if (_i.Bits(40, 19) != 0) throw Fail("TEX.B mode");
|
|
bool hasLod = lod is 2 or 3 or 6 or 7;
|
|
int coords = TexCoords[dim], handleSlot = op == 0x361 ? 1 : 0;
|
|
int count = coords + handleSlot + (hasLod ? 1 : 0) + (offsets ? 1 : 0) + (dc ? 1 : 0);
|
|
int inA = op == 0x361 && !_i.Bit(60) ? coords : count <= 4 ? (count + 1) / 2 : op == 0x361 ? coords : 4;
|
|
int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8);
|
|
int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA)));
|
|
for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false);
|
|
int next = coords + handleSlot, meta = TexMeta;
|
|
if (bindless) Mov(meta++, op == 0x361 ? Value(coords) : handleReg, false);
|
|
if (hasLod) Mov(meta++, Value(next++), false);
|
|
if (offsets) Mov(meta++, Value(next++), false);
|
|
if (dc) Mov(meta++, Value(next++), false);
|
|
if (bindless)
|
|
EmitG(0xde80000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20) | ((ulong)TexMaxwellDim[dim] << 28)
|
|
| ((ulong)mask << 31) | (offsets ? 1UL << 36 : 0) | ((ulong)lod << 37) | ((ulong)_i.Bits(90, 1) << 49)
|
|
| (dc ? 1UL << 50 : 0) | (7UL << 51));
|
|
else
|
|
EmitG(0xc000000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20)
|
|
| ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49)
|
|
| (dc ? 1UL << 50 : 0) | (7UL << 51) | (offsets ? 1UL << 54 : 0) | ((ulong)lod << 55));
|
|
TexResults(mask, _i.Bit(79));
|
|
}
|
|
|
|
// TMML (LOD query; results 8.8 fixed point as Maxwell's), 1D/2D/3D/CUBE. 0xb69 a bound descriptor; 0xf69 the
|
|
// handle in a uniform register (bits 40-45, bit 91 set); 0x36a (TMML.B, bit 59) the handle in Rb and every
|
|
// coordinate in Ra. Coordinates come from Ra, or from Ra and Rb as TEX's when Rb is a register. Maxwell TMML
|
|
// (0xdf58, descriptor in 36-48) / TMML.B (0xdf60, handle in Rb), dimension in Maxwell's TexDim numbering.
|
|
void Tmml(uint op)
|
|
{
|
|
bool bindless = op != 0xb69;
|
|
uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4);
|
|
int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8);
|
|
int coords = dim < 4 ? new[] { 1, 2, 3, 3 }[dim] : 0;
|
|
if (coords == 0 || _i.Bits(76, 4) != 0 || _i.Bits(80, 10) != 0 || mask == 0 || _i.Bit(59) != (op == 0x36a))
|
|
throw Fail("TMML mode");
|
|
uint handle = 0;
|
|
int handleReg = RZ;
|
|
if (op == 0xb69)
|
|
{
|
|
handle = Handle(false);
|
|
if (handle > 0x1fff || !TexBank(_i.Bits(54, 5))) throw Fail("TMML descriptor");
|
|
}
|
|
else if (op == 0xf69)
|
|
{
|
|
if (_i.Bits(46, 13) != 0 || !_i.Bit(91)) throw Fail("bindless TMML mode");
|
|
handleReg = Ur(_i.Bits(40, 6));
|
|
}
|
|
else
|
|
{
|
|
if (_i.Bits(40, 19) != 0 || _i.Bit(91)) throw Fail("TMML.B mode");
|
|
handleReg = Gpr((uint)rb);
|
|
}
|
|
int inA = rb == RZ || op == 0x36a ? coords : (coords + 1) / 2;
|
|
for (int k = 0; k < coords; k++)
|
|
Mov(TexCoord + k, k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))), false);
|
|
if (bindless) Mov(TexMeta, handleReg, false);
|
|
EmitG((bindless ? 0xdf60000000000000UL : 0xdf58000000000000UL) | TexOut | ((ulong)TexCoord << 8)
|
|
| ((ulong)(bindless ? TexMeta : RZ) << 20) | ((ulong)(dim * 2) << 28) | ((ulong)mask << 31)
|
|
| ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49));
|
|
TexResults(mask, false);
|
|
}
|
|
|
|
// TLD.LZ with the handle in a uniform register (0xf66: bits 40-45, bit 91 and bit 60 set): Maxwell TLD.B, the
|
|
// coordinates x | y, layer, x | y or x, y | z over Ra and Rb (1D, 2D, ARRAY_2D, 3D) and the handle in Rb.
|
|
void TldBindless()
|
|
{
|
|
uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4);
|
|
bool array = dim == 5, volume = dim == 2;
|
|
if (_i.Bits(46, 13) != 0 || !_i.Bit(91) || !_i.Bit(60) || _i.Bit(59) || _i.Bits(76, 5) != 0 || _i.Bits(81, 3) != 7
|
|
|| _i.Bits(87, 3) != 1 || (dim != 0 && dim != 1 && !array && !volume) || mask == 0)
|
|
throw Fail("bindless TLD mode");
|
|
int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8);
|
|
int handleReg = Ur(_i.Bits(40, 6));
|
|
Mov(TexCoord, ra == RZ ? RZ : Gpr((uint)ra), false);
|
|
if (array || volume) Mov(TexCoord + 1, ra == RZ ? RZ : Gpr((uint)(ra + 1)), false);
|
|
if (dim != 0) Mov(array || volume ? TexCoord + 2 : TexCoord + 1, rb == RZ ? RZ : Gpr((uint)rb), false);
|
|
Mov(TexMeta, handleReg, false);
|
|
EmitG(0xdd00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20)
|
|
| ((ulong)(array ? 3u : volume ? 4u : dim == 0 ? 0u : 2u) << 28) | ((ulong)mask << 31)
|
|
| ((ulong)_i.Bits(90, 1) << 49) | (7UL << 51));
|
|
TexResults(mask, false);
|
|
}
|
|
|
|
// TXQ.B (0x370): the handle in Ra and the level in Ra+1, as Maxwell TXQ.B (0xdf50) takes them; query 1
|
|
// (dimensions), results as TXQ's.
|
|
void TxqBindless()
|
|
{
|
|
uint mask = _i.Bits(72, 4);
|
|
if (_i.Bits(59, 5) != 1 || mask == 0 || _i.Bits(76, 14) != 0 || _i.Bit(91) || _i.Bits(32, 27) != 0) throw Fail("TXQ.B mode");
|
|
int ra = (int)_i.Bits(24, 8);
|
|
Mov(TexCoord, ra == RZ ? RZ : Gpr((uint)ra), false);
|
|
Mov(TexCoord + 1, ra == RZ ? RZ : Gpr((uint)(ra + 1)), false);
|
|
EmitG(0xdf50000000000000UL | TexOut | ((ulong)TexCoord << 8) | (1UL << 22) | ((ulong)mask << 31) | ((ulong)_i.Bits(90, 1) << 49));
|
|
TexResults(mask, false);
|
|
}
|
|
|
|
// F2I: destination signed (bit 72) and size (75-76: 8/16/32 bits), rounding 78-79 (nearest, floor, ceil,
|
|
// trunc), FTZ 80, source size 84-85 (F16, F32). Maxwell F2I has the same rounding order.
|
|
void F2i(uint op)
|
|
{
|
|
const uint known = 1u << 8 | 3u << 11 | 1u << 13 | 3u << 14 | 1u << 16 | 3u << 20;
|
|
uint w2 = _i.W2 & ~(op == 0xd05 ? 0x08000000u : 0u);
|
|
uint dstSize = _i.Bits(75, 2), srcSize = _i.Bits(84, 2);
|
|
if ((w2 & ~known) != 0 || dstSize > 2 || srcSize is not 1 and not 2 || (srcSize == 1 && _i.Bits(60, 2) != 0))
|
|
throw Fail("F2I format");
|
|
int b; bool abs, neg;
|
|
if (op == 0x905) { Mov32i(SB, _i.W1); (b, abs, neg) = (SB, false, false); } // F2I of a 32-bit immediate
|
|
else (b, abs, neg) = Raw(1, SB);
|
|
uint fmt = dstSize | (_i.Bit(72) ? 4u : 0u); // ISrcDstFmt: U8 0, U16 1, U32 2, +4 signed
|
|
EmitG(0x5cb0000000000000UL | ((ulong)b << 20) | (uint)Gpr((uint)Dst) | ((ulong)(fmt & 3) << 8) | ((ulong)(fmt >> 2) << 12)
|
|
| ((ulong)srcSize << 10) | ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44)
|
|
| (neg ? 1UL << 45 : 0) | (abs ? 1UL << 49 : 0));
|
|
}
|
|
|
|
// LDC Rd, c[bank][Ra + offset]: offset 38-53, bank 54-58, size 73-75 (as Maxwell's: U8 S8 U16 S16 32 64),
|
|
// mode 78-79 (0, or 2 .IS: Ra addresses all banks).
|
|
void Ldc()
|
|
{
|
|
uint width = _i.Bits(73, 3), mode = _i.Bits(78, 2);
|
|
int bank = Bank(_i.Bits(54, 5));
|
|
if (width > 5 || mode is not 0 and not 2 || (_i.W2 & ~0xce00u) != 0 || (_i.W1 & 0xf800003fu) != 0 || bank >= 18)
|
|
throw Fail("LDC mode");
|
|
int d = Gpr((uint)Dst);
|
|
if (width == 5 && (d & 1) != 0) throw Fail("LDC.64 odd destination");
|
|
EmitG(0xef90000000000000UL | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.Bits(38, 16) << 20) | ((ulong)bank << 36)
|
|
| ((ulong)mode << 44) | ((ulong)width << 48) | (uint)d);
|
|
}
|
|
|
|
// Global memory address of LDG/STG: a 64-bit register pair (Ra, .E), or a uniform pair (bits 32-37 for loads,
|
|
// 64-69 for stores) plus Ra (64-bit when bit 90 is set, else zero-extended), or with the "desc[UR][Ra.64]"
|
|
// forms Ra alone (the uniform pair is a cache-policy descriptor). Sums go to R244:R245.
|
|
int GlobalAddress(bool uniformForm, bool store)
|
|
{
|
|
int ra = (int)_i.Bits(24, 8);
|
|
if (!uniformForm)
|
|
{
|
|
if (store ? (_i.W2 & ~0xe00u) != 0x0010a100 : (_i.W2 & ~0x2e00u) != 0x001e8100) throw Fail("global access mode");
|
|
if (ra != RZ && ((ra & 1) != 0)) throw Fail("global address pair");
|
|
return ra == RZ ? RZ : Gpr((uint)ra);
|
|
}
|
|
uint ur = store ? _i.Bits(64, 6) : _i.Bits(32, 6);
|
|
bool descriptor = store ? (_i.W2 & ~(0x3fu | 0xe00u)) == 0x0c101100
|
|
: (_i.W2 & ~(0x3fu | 0xe00u)) == 0x0c1e1100 || _i.W2 == 0x0c1ebd00;
|
|
// Bit 78 is .STRONG.GPU instead of .STRONG.SM (two agl_common compute programs load their arguments and store
|
|
// their results so): the translator's accesses have no scope, which gives the same values outside a race.
|
|
uint plain = store ? 0x0810a100u : 0x081e8100u;
|
|
if (!descriptor && (_i.W2 & ~(0x04000000u | 0x3fu | 0xe00u | 0x4000u | (store ? 0u : 0x2000u))) != plain) throw Fail("global access uniform mode");
|
|
if (!store && _i.Bits(38, 2) != 0) throw Fail("global access uniform mode");
|
|
if (descriptor)
|
|
{
|
|
if ((ra & 1) != 0 || ra == RZ) throw Fail("global address pair");
|
|
return Gpr((uint)ra);
|
|
}
|
|
int u = Ur(ur);
|
|
if (u != RZ && (ur & 1) != 0) throw Fail("global uniform address pair");
|
|
if (ra == RZ) return u;
|
|
bool wide = _i.Bit(90);
|
|
if (wide && (ra & 1) != 0) throw Fail("global address pair");
|
|
int r = Gpr((uint)ra);
|
|
Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)r << 20) | ((ulong)u << 8) | 244)); // IADD.CC
|
|
Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)(wide ? r + 1 : RZ) << 20) | ((ulong)(u == RZ ? RZ : u + 1) << 8) | 245)); // IADD.X
|
|
return 244;
|
|
}
|
|
|
|
// LDG Rd, [address + offset]: offset 40-63 (signed), size 73-75 (U8 S8 U16 S16 32 64 128, as Maxwell's).
|
|
void Ldg(uint op)
|
|
{
|
|
uint size = _i.Bits(73, 3);
|
|
int d = Gpr((uint)Dst);
|
|
if (size > 6 || (size == 5 && (d & 1) != 0) || (size == 6 && (d & 3) != 0)) throw Fail("LDG size");
|
|
if (op == 0x381 && _i.Bits(32, 8) != 0) throw Fail("LDG mode");
|
|
int addr = GlobalAddress(op == 0x981, false);
|
|
EmitG(0xeed0000000000000UL | (uint)d | ((ulong)addr << 8) | ((ulong)_i.Bits(40, 24) << 20) | (1UL << 45) | ((ulong)size << 48));
|
|
}
|
|
|
|
// STG [address + offset], Rdata: data 32-39, offset 40-63, size 73-75; .E (bit 72) required.
|
|
void Stg(uint op)
|
|
{
|
|
uint size = _i.Bits(73, 3);
|
|
int data = Gpr(_i.Bits(32, 8));
|
|
if (size > 6 || !_i.Bit(72) || (data != RZ && ((size == 5 && (data & 1) != 0) || (size == 6 && (data & 3) != 0)))) throw Fail("STG size");
|
|
if (data == RZ && size > 4)
|
|
{
|
|
for (int k = 0; k < (1 << (int)(size - 4)); k++) Mov32i(TexOut + k, 0);
|
|
data = TexOut;
|
|
}
|
|
int addr = GlobalAddress(op == 0x986, true);
|
|
EmitG(0xeed8000000000000UL | (uint)data | ((ulong)addr << 8) | ((ulong)_i.Bits(40, 24) << 20) | (1UL << 45) | ((ulong)size << 48));
|
|
}
|
|
|
|
// TLD (texel fetch, integer coordinates): fields as TEX; LOD mode 87-89 is 1 (.LZ) or 3 (.LL, the level after
|
|
// the coordinates), offsets bit 76. Maxwell TLD takes its operands as Maxwell TEX does. Bit 78 (.MS) reads one
|
|
// sample of a multisampled 2D texture, its index in the operand after the coordinates (agl_common's 2x resolve:
|
|
// TLD.LZ.MS with samples 0 and 1, averaged); it is lowered for .LZ without offsets only, Maxwell's MS bit 50.
|
|
void Tld()
|
|
{
|
|
uint dim = _i.Bits(61, 3), lod = _i.Bits(87, 3), mask = _i.Bits(72, 4);
|
|
bool offsets = _i.Bit(76), ms = _i.Bit(78);
|
|
if (dim == 6 || _i.Bit(77) || _i.Bit(79) || _i.Bits(81, 3) != 7 || _i.Bit(59) || lod is not 1 and not 3 || mask == 0
|
|
|| (ms && (dim != 1 || lod != 1 || offsets)))
|
|
throw Fail("TLD mode");
|
|
uint handle = Handle(false), bank = _i.Bits(54, 5);
|
|
if (handle > 0x1fff) throw Fail("TLD descriptor offset");
|
|
if (!TexBank(bank)) throw Fail($"TLD descriptor bank {bank}");
|
|
int coords = TexCoords[dim];
|
|
int count = coords + (lod == 3 ? 1 : 0) + (offsets ? 1 : 0) + (ms ? 1 : 0);
|
|
int inA = count <= 4 ? (count + 1) / 2 : 4;
|
|
int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8);
|
|
int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA)));
|
|
for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false);
|
|
int next = coords, meta = TexMeta;
|
|
if (lod == 3) Mov(meta++, Value(next++), false);
|
|
if (offsets) Mov(meta++, Value(next++), false);
|
|
if (ms) Mov(meta++, Value(next++), false);
|
|
EmitG(0xdc00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20)
|
|
| ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | (offsets ? 1UL << 35 : 0) | ((ulong)handle << 36)
|
|
| ((ulong)_i.Bits(90, 1) << 49) | (ms ? 1UL << 50 : 0) | (7UL << 51) | (lod == 3 ? 1UL << 55 : 0));
|
|
TexResults(mask, false);
|
|
}
|
|
|
|
// TXQ (dimensions): level in Ra, descriptor offset 40-53 in bank 54-58 (2), mask 72-75, second destination
|
|
// 64-71 for the third and fourth values. Maxwell TXQ with query 1 (dimension) writes them to consecutive
|
|
// registers.
|
|
void Txq()
|
|
{
|
|
uint mask = _i.Bits(72, 4), handle = Handle(false), bank = _i.Bits(54, 5);
|
|
if (_i.Bits(59, 5) != 0 || mask == 0 || _i.Bits(76, 14) != 0 || _i.Bit(91) || handle > 0x1fff) throw Fail("TXQ mode");
|
|
if (!TexBank(bank)) throw Fail($"TXQ descriptor bank {bank}");
|
|
int ra = Gpr(_i.Bits(24, 8));
|
|
EmitG(0xdf48000000000000UL | TexOut | ((ulong)ra << 8) | (1UL << 22) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49));
|
|
TexResults(mask, false);
|
|
}
|
|
|
|
// Texture results: the components of the mask, from TexOut on, to Rd, Rd+1 and then the second destination
|
|
// (bits 64-71) when there is one. FP16 results pack (c0, c1) into the first register and (c2, c3) into the
|
|
// second, the first of each pair in the low half (F2F.F16 writes its half there; ISCADD merges).
|
|
void TexResults(uint mask, bool f16)
|
|
{
|
|
int d = Dst, d2 = (int)_i.Bits(64, 8);
|
|
int n = System.Numerics.BitOperations.PopCount(mask);
|
|
if (!f16)
|
|
{
|
|
for (int k = 0; k < n; k++)
|
|
{
|
|
int target = d2 == RZ || k < 2 ? (d == RZ ? RZ : d + k) : d2 + (k - 2);
|
|
if (target != RZ) Mov(Gpr((uint)target), TexOut + k);
|
|
}
|
|
return;
|
|
}
|
|
void pack(int target, int lo, int hi)
|
|
{
|
|
if (target == RZ) return;
|
|
Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)lo << 20) | 246));
|
|
if (hi == RZ) Mov32i(247, 0);
|
|
else Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)hi << 20) | 247));
|
|
EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)target));
|
|
}
|
|
pack(d, TexOut, n > 1 ? TexOut + 1 : RZ);
|
|
if (n > 2) pack(d2 != RZ ? d2 : d == RZ ? RZ : d + 1, TexOut + 2, n > 3 ? TexOut + 3 : RZ);
|
|
}
|
|
|
|
// PRMT Rd, a, selector (B slot), c: byte i of the result is byte (s & 7) of c:a, its sign repeated when s & 8;
|
|
// the mode (bits 72-75) other than the default index mode is not lowered. Maxwell PRMT has the same meaning.
|
|
void Prmt()
|
|
{
|
|
if (_i.Bits(72, 4) != 0 || _i.Bits(76, 15) != 0) throw Fail("PRMT mode");
|
|
int a = Plain(0, SA), sel = Plain(1, SB), c = Plain(2, SC);
|
|
EmitG(0x5bc0000000000000UL | ((ulong)c << 39) | ((ulong)sel << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// FLO Rd, b: the index of the highest set bit (bit 73 signed, bit 74 .SH: the shift amount, bit 63 of ~b).
|
|
void Flo()
|
|
{
|
|
if ((_i.W2 & ~(0x600u | 0x08000000u)) != 0x000e0000) throw Fail("FLO mode");
|
|
var (b, _, inv) = Raw(1, SB);
|
|
EmitG(0x5c30000000000000UL | ((ulong)b << 20) | (inv ? 1UL << 40 : 0) | ((ulong)_i.Bits(74, 1) << 41)
|
|
| ((ulong)_i.Bits(73, 1) << 48) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// HSETP2 P, Q, a, b: compare 76-79, FTZ 80, P/Q 81-86, combined with Pc (87-90) by bits 69-70, H_AND 71;
|
|
// A: reg 24, neg 72, abs 73, swizzle 74-75; b: a register (swizzle 60-61, abs 62, neg 63) or an immediate half
|
|
// pair. Maxwell HSETP2 has the same fields at its own positions.
|
|
void Hsetp2()
|
|
{
|
|
uint op = _i.Bits(0, 12);
|
|
ulong source;
|
|
if (op == 0x234)
|
|
{
|
|
if ((_i.W1 & 0x0fffff00u) != 0 || _i.Bits(64, 5) != 0) throw Fail("HSETP2 mode");
|
|
source = ((ulong)Gpr(_i.Bits(32, 8)) << 20) | ((ulong)_i.Bits(60, 2) << 28) | ((ulong)_i.Bits(62, 1) << 30) | ((ulong)_i.Bits(63, 1) << 31);
|
|
}
|
|
else { Mov32i(SB, _i.W1); source = (ulong)SB << 20; }
|
|
EmitG(0x5d20000000000000UL | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3) | ((ulong)_i.Bits(80, 1) << 6)
|
|
| ((ulong)Gpr(_i.Bits(24, 8)) << 8) | source | ((ulong)_i.Bits(76, 4) << 35) | ((ulong)_i.Bits(87, 3) << 39)
|
|
| ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(72, 1) << 43) | ((ulong)_i.Bits(73, 1) << 44)
|
|
| ((ulong)_i.Bits(69, 2) << 45) | ((ulong)_i.Bits(74, 2) << 47) | ((ulong)_i.Bits(71, 1) << 49));
|
|
}
|
|
|
|
// P2R Rd, PR, Ra, mask / R2P PR, Ra, mask: immediate mask (bits 32-38), byte select 76-77, Ra the merge source
|
|
// (P2R) or the value (R2P), as Maxwell's P2R/R2P with an immediate.
|
|
void PredicateRegister(bool toRegister)
|
|
{
|
|
if (_i.W1 > 0x7f || (_i.W2 & ~0x3000u) != 0 || (!toRegister && _i.Bits(16, 8) != 0)) throw Fail("P2R/R2P mode");
|
|
ulong w = (toRegister ? 0x38e8000000000000UL | (uint)Gpr((uint)Dst) : 0x38f0000000000000UL)
|
|
| ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.W1 << 20) | ((ulong)_i.Bits(76, 2) << 41);
|
|
EmitG(w);
|
|
}
|
|
|
|
// SURED.D.op U32 (bound descriptor): data 32-39, coordinates 24-31, operation 87-89 (ADD MIN MAX INC DEC AND OR
|
|
// XOR, as Maxwell's), dimension 61-63 (1D buffer or 2D), descriptor offset 40-53 in bank 54-58. Maxwell's SURED
|
|
// is 1110101101011 in the top bits (0xeb58, InstTable Sured); 0xeb48 matched no instruction, so the
|
|
// translator skipped every SURED: Ocarina's edge AA (the blend-weight pass ORs its weights into an image) never
|
|
// wrote anything.
|
|
void Sured()
|
|
{
|
|
uint dim = _i.Bits(61, 3), handle = Handle(true), bank = _i.Bits(54, 5);
|
|
if ((_i.W2 & ~(7u << 23)) != 0x0010e000 || dim is not 1 and not 3 || _i.Bits(59, 2) != 0 || handle > 0x1fff || !TexBank(bank))
|
|
throw Fail("SURED mode");
|
|
EmitG(0xeb58000000000000UL | (uint)Gpr(_i.Bits(32, 8)) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.Bits(87, 3) << 24)
|
|
| ((ulong)dim << 33) | ((ulong)handle << 36));
|
|
}
|
|
|
|
// SUATOM.D.op U32 (bound descriptor): Rd = the image word's old value, fields as SURED's plus the destination and
|
|
// the result predicate 81-83 (PT only). agl_technique_pfx's append counters add to word 0 of a 1D image this
|
|
// way. Maxwell's SUATOM.D is 11101010 in the top bits with bit 54 clear (InstTable Suatom): operation 29-32,
|
|
// dimension 33-35, descriptor 36-48, size 51-53 (U32 0).
|
|
void Suatom()
|
|
{
|
|
uint dim = _i.Bits(61, 3), handle = Handle(true), bank = _i.Bits(54, 5);
|
|
if ((_i.W2 & ~(0xfu << 23)) != 0x001ee000 || _i.Bits(87, 4) > 8 || dim > 5 || _i.Bits(59, 2) != 0 || handle > 0x1fff
|
|
|| !TexBank(bank))
|
|
throw Fail("SUATOM mode");
|
|
EmitG(0xea00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20)
|
|
| ((ulong)_i.Bits(87, 4) << 29) | ((ulong)dim << 33) | ((ulong)handle << 36));
|
|
}
|
|
|
|
// ATOMG.E.CAS U32: Rd = [Ra.64 + offset 40-63], replaced by Rc (64-71) where it equals Rb (32-39). The foliage
|
|
// compute programs claim hash slots with it. Maxwell ATOM.E.CAS takes the compare value in Rb and the new value
|
|
// in Rb + 1 (xlate/patches/0037-global-compare-and-swap.patch, which also makes the translator emit it), so both
|
|
// go to SB and SB + 1 first. Only a zero offset is lowered.
|
|
void AtomgCas()
|
|
{
|
|
if ((_i.W2 & ~0xffu) != 0x001ee100 || _i.Bits(40, 24) != 0 || (_i.Bits(24, 8) & 1) != 0 || _i.Bits(24, 8) == RZ)
|
|
throw Fail("ATOMG.CAS mode");
|
|
Mov(SB, Gpr(_i.Bits(32, 8)), false);
|
|
Mov(SB + 1, Gpr(_i.Bits(64, 8)), false);
|
|
EmitG(0xeef0000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)SB << 20) | (1UL << 48));
|
|
}
|
|
|
|
// REDUX URd, Ra: Ra reduced over the warp's active lanes into a uniform register. Mode (bits 78-80): SUM 3
|
|
// (modulo 2^32), MIN 4, MAX 5 (unsigned, signed with bit 73), OR 1, XOR 2. Lowered as VOTE.ANY (the active-lane
|
|
// mask) and one SHFL.IDX per lane; an inactive lane contributes the operation's identity.
|
|
void Redux()
|
|
{
|
|
uint mode = (_i.W2 & ~0x200u) >> 14;
|
|
bool signedOrder = _i.Bit(73);
|
|
if (_i.W1 != 0 || (_i.W2 & ~0x200u & ~(7u << 14)) != 0 || mode is not (1 or 2 or 3 or 4 or 5)
|
|
|| (signedOrder && mode is not (4 or 5)) || _i.Bits(12, 4) != 7)
|
|
throw Fail("REDUX mode");
|
|
int t = Ur((uint)Dst), src = Gpr(_i.Bits(24, 8));
|
|
if (t == RZ) return;
|
|
const int ballot = 240, lanev = 241, keep = 242;
|
|
bool min = mode == 4;
|
|
Emit(Unguarded(0x50d8000000000000UL | (1UL << 48) | (7UL << 45) | ((ulong)PT << 39) | ballot)); // VOTE.ANY ballot
|
|
Mov32i(t, min ? 0xffffffffu : 0u);
|
|
ulong lop3(int d, int a, int b, uint lut) => Unguarded(0x5be0000000000000UL | ((ulong)RZ << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)d | ((ulong)lut << 28) | (7UL << 48));
|
|
for (uint lane = 0; lane < 32; lane++)
|
|
{
|
|
Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | (1UL << 28) | ((ulong)lane << 20) | ((ulong)src << 8) | lanev));
|
|
if (signedOrder) Lop32i(lanev, lanev, 0x80000000, 2); // signed order as unsigned
|
|
Emit(Unguarded(0x3828000000000000UL | ((ulong)lane << 20) | ((ulong)ballot << 8) | keep)); // SHR imm
|
|
Lop32i(keep, keep, 1, 0);
|
|
Iadd3(keep, RZ, keep, RZ, false, negB: true); // 0 or ~0
|
|
Emit(lop3(lanev, lanev, keep, min ? 0xF3u : 0xC0u)); // x | ~m, x & m
|
|
switch (mode)
|
|
{
|
|
case 4: Emit(Unguarded(0x5c20000000000000UL | ((ulong)lanev << 20) | ((ulong)t << 8) | (uint)t | ((ulong)PT << 39))); break; // min
|
|
case 5: Emit(Unguarded(0x5c20000000000000UL | ((ulong)lanev << 20) | ((ulong)t << 8) | (uint)t | (15UL << 39))); break; // max
|
|
case 1: Emit(lop3(t, t, lanev, 0xFC)); break;
|
|
case 2: Emit(lop3(t, t, lanev, 0x3C)); break;
|
|
default: Iadd3(t, t, lanev, RZ, false); break;
|
|
}
|
|
}
|
|
if (signedOrder) Lop32i(t, t, 0x80000000, 2);
|
|
}
|
|
|
|
// MATCH.ALL Pd, Rd, Ra (op 0x3a1 with no mode bits; Ocarina's NVN2 compute programs test subgroupAllEqual with it
|
|
// before an aggregated shared atomic: @!Pd per-lane ATOMS, else REDUX and one): Pd = every active lane holds Ra's
|
|
// value, Rd = the active-lane mask then, else 0. Lowered as VOTE.ANY (the active lanes), FLO (the highest of
|
|
// them), SHFL.IDX (that lane's value), ISETP.EQ into Pd and VOTE.ALL over Pd. Only the unguarded form with a
|
|
// predicate destination is taken. Bit 73 is .U64: Ra:Ra+1 is compared, the high word ANDed into Pd.
|
|
void Match()
|
|
{
|
|
uint pd = _i.Bits(81, 3);
|
|
bool wide = _i.Bit(73);
|
|
if (_i.W1 != 0 || (_i.W2 & ~(7u << 17 | 1u << 9)) != 0 || _i.Bits(12, 4) != 7 || pd == PT)
|
|
throw Fail("MATCH mode");
|
|
int d = Gpr((uint)Dst), src = Gpr(_i.Bits(24, 8));
|
|
if (wide && (src == RZ || (src & 1) != 0)) throw Fail("MATCH.U64 register pair");
|
|
const int ballot = 240, lead = 241, value = 242, valueHi = 243;
|
|
Emit(Unguarded(0x50d8000000000000UL | (1UL << 48) | (7UL << 45) | ((ulong)PT << 39) | ballot)); // VOTE.ANY
|
|
Emit(Unguarded(0x5c30000000000000UL | ((ulong)ballot << 20) | lead)); // FLO.U32
|
|
Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)lead << 20)
|
|
| ((ulong)src << 8) | value)); // SHFL.IDX
|
|
Emit(Unguarded(0x5b60000000000000UL | (2UL << 49) | ((ulong)PT << 39) | ((ulong)value << 20)
|
|
| ((ulong)src << 8) | ((ulong)pd << 3) | PT)); // ISETP.EQ.U32
|
|
if (wide)
|
|
{
|
|
Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)lead << 20)
|
|
| ((ulong)(src + 1) << 8) | valueHi)); // SHFL.IDX
|
|
Emit(Unguarded(0x5b60000000000000UL | (2UL << 49) | ((ulong)pd << 39) | ((ulong)valueHi << 20)
|
|
| ((ulong)(src + 1) << 8) | ((ulong)pd << 3) | PT)); // ISETP.EQ.U32.AND
|
|
}
|
|
Emit(Unguarded(0x50d8000000000000UL | ((ulong)pd << 45) | ((ulong)pd << 39) | RZ)); // VOTE.ALL
|
|
if (d != RZ) Sel(d, ballot, RZ, pd, false);
|
|
}
|
|
|
|
// ATOMS.POPC.INC RZ, [Ra + URb + offset] (op 0xf8c, mode word 0x0d000000; the increment of Ocarina's aggregated
|
|
// shared counters after MATCH and REDUX): memory goes up by the number of active lanes. With no result register
|
|
// that is one ATOMS.ADD of 1 per active lane, which is what this emits; a form that wants the per-lane result (its
|
|
// rank among the active lanes) is not taken. Bits 78-79 scale Ra as LDS / STS's stride does (SharedAccess): the
|
|
// luminance histograms of agl_common use 3, three counters in each 16-byte bin at offsets 0, 4 and 8.
|
|
void AtomsPopcInc()
|
|
{
|
|
uint offset = _i.Bits(40, 24), ur = _i.Bits(64, 6), stride = _i.Bits(78, 2);
|
|
if ((_i.W2 & ~(0x3fu | 3u << 14)) != 0x0d000000u || _i.Bits(32, 8) != 0 || Dst != RZ || (offset & 3) != 0 || offset >= (1u << 22))
|
|
throw Fail("ATOMS.POPC.INC mode");
|
|
int addr = Gpr(_i.Bits(24, 8));
|
|
if (stride != 0 && addr != RZ)
|
|
{
|
|
Emit(Unguarded(0x3848000000000000UL | ((ulong)(stride + 1) << 20) | ((ulong)addr << 8) | SA)); // SHL imm
|
|
addr = SA;
|
|
}
|
|
if (ur != 0x3f)
|
|
{
|
|
Iadd3(SA, addr, Ur(ur), RZ, false);
|
|
addr = SA;
|
|
}
|
|
Mov32i(SB, 1);
|
|
EmitG(0xec00000000000000UL | RZ | ((ulong)addr << 8) | ((ulong)SB << 20) | ((ulong)(offset >> 2) << 30)); // ATOMS.ADD
|
|
}
|
|
|
|
// VOTE Rd, Pu, Pin: ballot into Rd (URd for VOTEU), mode 72-73 (ALL, ANY, EQ) over the vector predicate of bits
|
|
// 87-90, result predicate 81-83 (a uniform predicate for VOTEU). Maxwell VOTE has the same fields. VOTEU's
|
|
// output goes through Pk (k its UP index) with the guest's Pk saved around it; no uniform predicate is read.
|
|
void Vote(bool uniform)
|
|
{
|
|
if (_i.W1 != 0 || _i.Bits(72, 2) > 2 || (_i.W2 & ~0x078e0300u) != 0 || (uniform && (_i.Bits(24, 8) != 0 || _i.Bits(12, 4) != 7)))
|
|
throw Fail("VOTE mode");
|
|
uint pout = _i.Bits(81, 3);
|
|
int d = uniform ? Ur((uint)Dst) : Gpr((uint)Dst);
|
|
ulong vote = 0x50d8000000000000UL | (uint)d | ((ulong)_i.Bits(87, 4) << 39) | ((ulong)pout << 45) | ((ulong)_i.Bits(72, 2) << 48);
|
|
if (!uniform || pout == 7) { EmitG(vote); return; }
|
|
uint m = 1u << (int)pout;
|
|
Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RZ << 8) | USAVE));
|
|
EmitG(vote);
|
|
Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RUP << 8) | RUP));
|
|
Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)USAVE << 8)));
|
|
}
|
|
|
|
// ATOMG.E U32: Rd = atomic op (87-90, numbered as Maxwell ATOM's: ADD MIN MAX INC DEC AND OR XOR EXCH) of
|
|
// [address + offset 40-63] with Rb (32-39); the address a register pair (24-31) or a uniform pair (64-69).
|
|
void Atomg(bool uniformAddress)
|
|
{
|
|
uint operation = _i.Bits(87, 4);
|
|
int offset = (int)(_i.Bits(40, 24) << 8) >> 8;
|
|
uint w2 = _i.W2 & ~(0xfu << 23);
|
|
int addr;
|
|
if (uniformAddress)
|
|
{
|
|
if (_i.Bits(24, 8) != RZ || (w2 & ~0x3fu) != 0x081ee140 || (_i.Bits(64, 6) & 1) != 0) throw Fail("ATOMG mode");
|
|
addr = Ur(_i.Bits(64, 6));
|
|
}
|
|
else
|
|
{
|
|
if (w2 != 0x001ee100 || (_i.Bits(24, 8) & 1) != 0) throw Fail("ATOMG mode");
|
|
addr = Gpr(_i.Bits(24, 8));
|
|
}
|
|
if (operation > 8 || offset < -0x80000 || offset > 0x7ffff) throw Fail("ATOMG operation/offset");
|
|
EmitG(0xed00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)addr << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20)
|
|
| ((ulong)((uint)offset & 0xfffff) << 28) | (1UL << 48) | ((ulong)operation << 52));
|
|
}
|
|
|
|
// RED.E (global reduction, no result): op 87-89 (ADD MIN MAX INC DEC AND OR XOR), type 73-75 (U32, S32, F32 add),
|
|
// data 32-39, address a register pair (24-31) or, with Ra = RZ, a uniform pair (64-69), offset 40-63.
|
|
void Red()
|
|
{
|
|
uint type = _i.Bits(73, 3), operation = _i.Bits(87, 3);
|
|
bool ur = _i.Bits(24, 8) == RZ;
|
|
uint variable = 0x03800e00u | (ur ? 0x3fu : 0u), mode = ur ? 0x0c10e100u : 0x0010e100u;
|
|
int offset = (int)(_i.Bits(40, 24) << 8) >> 8;
|
|
if ((_i.W2 & ~variable) != mode || type is not (0 or 1 or 3) || (type == 3 && operation != 0)) throw Fail("RED mode");
|
|
uint a = ur ? _i.Bits(64, 6) : _i.Bits(24, 8);
|
|
if ((a & 1) != 0 || offset < -0x80000 || offset > 0x7ffff) throw Fail("RED address");
|
|
int addr = ur ? Ur(a) : Gpr(a);
|
|
EmitG(0xebf8000000000000UL | (uint)Gpr(_i.Bits(32, 8)) | ((ulong)addr << 8) | ((ulong)type << 20) | ((ulong)operation << 23)
|
|
| ((ulong)((uint)offset & 0xfffff) << 28) | (1UL << 48));
|
|
}
|
|
|
|
// SHFL Rd, [Pout,] a, lane, mask: lane an immediate (53-57; 0x989, 0xf89) or a register (32-39), the
|
|
// clamp/segment mask an immediate (40-52; 0x589, 0xf89) or a register (64-71), mode 58-59 (IDX UP DOWN BFLY),
|
|
// lane-valid predicate 81-83. Maxwell SHFL has the same choices.
|
|
void Shfl(uint op)
|
|
{
|
|
bool laneImm = op is 0x989 or 0xf89, maskImm = op is 0x589 or 0xf89;
|
|
if ((_i.W2 & ~(0x000e0000u | (maskImm ? 0u : 0xffu))) != 0 || _i.Bits(60, 4) != 0 || (laneImm && _i.Bits(32, 8) != 0) || (!laneImm && _i.Bits(53, 5) != 0))
|
|
throw Fail("SHFL mode");
|
|
ulong lane = laneImm ? ((ulong)_i.Bits(53, 5) << 20) | (1UL << 28) : (ulong)Gpr(_i.Bits(32, 8)) << 20;
|
|
ulong mask = maskImm ? ((ulong)_i.Bits(40, 13) << 34) | (1UL << 29) : (ulong)Gpr(_i.Bits(64, 8)) << 39;
|
|
EmitG(0xef10000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | lane | mask
|
|
| ((ulong)_i.Bits(58, 2) << 30) | ((ulong)_i.Bits(81, 3) << 48));
|
|
}
|
|
|
|
// FSWZADD Rd, a, b (64-71), per-lane operation codes (bits 32-39): 0 a+b, 1 a-b, 2 b-a, 3 b. Maxwell numbers 1
|
|
// and 2 the other way round.
|
|
void Fswzadd()
|
|
{
|
|
uint swz = _i.W1;
|
|
if (swz > 0xff || _i.W2 > 255) throw Fail("FSWZADD mode");
|
|
uint maxwell = ((swz & 0x55) << 1) | ((swz & 0xaa) >> 1);
|
|
EmitG(0x50f8000000000000UL | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(64, 8)) << 20) | ((ulong)maxwell << 28) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// ATOMS (shared) U32/S32 (bit 73): Rd = op (87-90, as Maxwell ATOMS's) of [Ra + offset 40-63] with Rb.
|
|
void Atoms()
|
|
{
|
|
uint offset = _i.Bits(40, 24), operation = _i.Bits(87, 4);
|
|
if ((_i.W2 & ~0x07800200u) != 0 || operation > 8 || (offset & 3) != 0) throw Fail("ATOMS mode");
|
|
EmitG(0xec00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20)
|
|
| ((ulong)_i.Bits(73, 1) << 28) | ((ulong)(offset >> 2) << 30) | ((ulong)operation << 52));
|
|
}
|
|
|
|
// SGXT Rd, a, n (immediate): the low n bits of a, sign-extended with bit 73 (else zero-extended): Maxwell BFE
|
|
// from bit 0, n bits.
|
|
void Sgxt()
|
|
{
|
|
if ((_i.W2 & ~0x200u) != 0 || _i.W1 > 32) throw Fail("SGXT mode");
|
|
EmitG(0x3800000000000000UL | ((ulong)(_i.W1 << 8) << 20) | ((ulong)_i.Bits(73, 1) << 48) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// BREV Rd, b (register): Maxwell SHR.BREV by 0.
|
|
void Brev()
|
|
{
|
|
if (_i.W2 != 0 || _i.W1 > 255) throw Fail("BREV mode");
|
|
EmitG(0x3828000000000000UL | ((ulong)Gpr(_i.Bits(32, 8)) << 8) | (1UL << 39) | (1UL << 40) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// HMNMX2: per half lane, both operands widened to F32 (F2F, exact), FMNMX with the predicate of bits 87-90
|
|
// (true: min), narrowed back and packed (lane 1 in the high half).
|
|
void Hmnmx2()
|
|
{
|
|
uint op = _i.Bits(0, 12);
|
|
if (_i.Bits(76, 11) != 0 || _i.Bit(91)) throw Fail("HMNMX2 mode");
|
|
var a = HalfSrc(0, SA);
|
|
if (a.swz == 1) throw Fail("HMNMX2 F32 operand");
|
|
(int reg, uint swz, bool abs, bool neg) b = op == 0x840 ? (SB, 0u, false, false) : HalfSrc(1, SB);
|
|
if (op == 0x840) Mov32i(SB, _i.W1);
|
|
if (b.swz == 1) throw Fail("HMNMX2 F32 operand");
|
|
void widen(int target, (int reg, uint swz, bool abs, bool neg) h, int lane)
|
|
{
|
|
bool high = h.swz == 3 || (h.swz == 0 && lane == 1);
|
|
Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)h.reg << 20) | (high ? 1UL << 41 : 0)
|
|
| (h.neg ? 1UL << 45 : 0) | (h.abs ? 1UL << 49 : 0) | (uint)target));
|
|
}
|
|
for (int lane = 0; lane < 2; lane++)
|
|
{
|
|
int t = lane == 0 ? 246 : 247;
|
|
widen(244, a, lane);
|
|
widen(245, b, lane);
|
|
Emit(Unguarded(0x5c60000000000000UL | (245UL << 20) | (244UL << 8) | (uint)t | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42)));
|
|
Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)t << 20) | (uint)t));
|
|
}
|
|
EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// A packed-half operand: register, swizzle (0 H1_H0, 1 F32, 2 H0_H0, 3 H1_H1, as Maxwell's), abs, neg.
|
|
// A: reg 24, swizzle 74-75, abs 73, neg 72. B in bits 32-39 (swizzle 60-61, abs 62, neg 63) or, for forms 2/3/7,
|
|
// 64-71 (swizzle 81-82, abs 83, neg 84). C in 64-71 (81-82, 83, 84). Immediates are a half pair as stored;
|
|
// constants take swizzle/abs/neg from 60-63.
|
|
(int reg, uint swz, bool abs, bool neg) HalfSrc(int which, int scratch)
|
|
{
|
|
if (which == 0) return (Gpr(_i.Bits(24, 8)), _i.Bits(74, 2), _i.Bit(73), _i.Bit(72));
|
|
uint form = Form;
|
|
bool wide = which == 1 ? form is 4 or 5 : form is 2 or 3;
|
|
if (wide)
|
|
{
|
|
if (form is 4 or 2) { Mov32i(scratch, _i.W1); return (scratch, 0, false, false); }
|
|
var (bank, off) = Cbuf(); MovC(scratch, bank, off);
|
|
return (scratch, _i.Bits(60, 2), _i.Bit(62), _i.Bit(63));
|
|
}
|
|
if (form is 6 or 7) throw Fail("half operand in a uniform register");
|
|
bool upper = which == 2 || form is 2 or 3;
|
|
return upper ? (Gpr(_i.Bits(64, 8)), _i.Bits(81, 2), _i.Bit(83), _i.Bit(84))
|
|
: (Gpr(_i.Bits(32, 8)), _i.Bits(60, 2), _i.Bit(62), _i.Bit(63));
|
|
}
|
|
|
|
// HADD2 / HMUL2 / HFMA2 with packed-half results: saturate 77, FTZ 80, bit 76 the FMZ precision of HMUL2/HFMA2.
|
|
// HFMA2.RELU (79) clamps each lane at zero where the predicate in bits 87-90 holds; only PT is lowered. Measured
|
|
// on an NVIDIA GPU (tests/tensor_lowering_check.py): a negative result or -0 becomes +0, +inf stays, -inf
|
|
// becomes 0, a NaN result stays NaN (0x7fff). The sum goes to TexOut and integer operations build the mask of
|
|
// the half lanes to clear. Maxwell PRMT would do that in one instruction and the translator implements it
|
|
// (xlate/patches/0021-prmt-byte-permute.patch, Instructions/InstEmitPermute.cs PrmtR); this keeps the integer
|
|
// form, which is what was measured.
|
|
// F32 results (78), the H0_NH1 source (86) and absolute values where Maxwell has none are not lowered. HADD2's
|
|
// second operand is B, or C for forms 2/3.
|
|
void Half(ulong maxwell)
|
|
{
|
|
uint op = _i.Bits(0, 12) & 0x1ff;
|
|
if (op == 0x030 && _i.Bit(78))
|
|
{
|
|
// HADD2.F32 Rd, -RZ, b: the compiler's half-to-float conversion of one lane of b
|
|
var a0 = HalfSrc(0, SA);
|
|
var bh = HalfSrc(Form is 2 or 3 ? 2 : 1, SB);
|
|
if (a0.reg != RZ || bh.swz is 0 or 1 || _i.Bit(77) || _i.Bits(81, 10) != 0) throw Fail("HADD2.F32 mode");
|
|
EmitG(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)bh.reg << 20) | (bh.swz == 3 ? 1UL << 41 : 0)
|
|
| (bh.neg ? 1UL << 45 : 0) | (bh.abs ? 1UL << 49 : 0) | (uint)Gpr((uint)Dst));
|
|
return;
|
|
}
|
|
bool relu = op == 0x031 && _i.Bit(79);
|
|
if (relu && (_i.Bit(77) || _i.Bits(85, 2) != 0 || _i.Bits(87, 4) != 7)) throw Fail("HFMA2.RELU mode");
|
|
if (_i.Bit(78) || (_i.Bit(79) && !relu) || _i.Bit(91)
|
|
|| (op == 0x031 ? (relu ? _i.Bits(85, 2) : _i.Bits(85, 6)) != 0 : _i.Bits(81, 10) != 0))
|
|
throw Fail("half arithmetic mode");
|
|
if (op == 0x030 && _i.Bit(76)) throw Fail("HADD2 mode");
|
|
var a = HalfSrc(0, SA);
|
|
var b = HalfSrc(op == 0x030 && Form is 2 or 3 ? 2 : 1, SB);
|
|
uint precision = _i.Bit(76) ? 2u : _i.Bits(80, 1);
|
|
ulong w = maxwell | ((ulong)a.reg << 8) | ((ulong)b.reg << 20) | ((ulong)b.swz << 28) | ((ulong)a.swz << 47)
|
|
| ((ulong)_i.Bits(77, 1) << 32) | (uint)(relu ? TexOut : Gpr((uint)Dst));
|
|
if (op == 0x030)
|
|
w |= (b.abs ? 1UL << 30 : 0) | (b.neg ? 1UL << 31 : 0) | (a.neg ? 1UL << 43 : 0) | (a.abs ? 1UL << 44 : 0)
|
|
| ((ulong)_i.Bits(80, 1) << 39);
|
|
else if (op == 0x032)
|
|
w |= (b.abs ? 1UL << 30 : 0) | (a.neg != b.neg ? 1UL << 31 : 0) | (a.abs ? 1UL << 44 : 0) | ((ulong)precision << 39);
|
|
else
|
|
{
|
|
var c = HalfSrc(2, SC);
|
|
if (a.abs || b.abs || c.abs) throw Fail("HFMA2 absolute operand");
|
|
w |= (c.neg ? 1UL << 30 : 0) | (a.neg != b.neg ? 1UL << 31 : 0) | ((ulong)c.swz << 35) | ((ulong)precision << 37)
|
|
| ((ulong)c.reg << 39);
|
|
}
|
|
if (!relu) { EmitG(w); return; }
|
|
// RELU on both half lanes of the sum x at once: each half h becomes |h| (its sign bit cleared), or 0 when h is
|
|
// a negative number (sign set, not NaN). |h| > 0x7c00 means NaN, so |h| + 0x3ff carries into bit 15 of its
|
|
// half exactly for a NaN, and never into the other half (|h| + 0x3ff < 0x10000). A NaN keeps the payload the
|
|
// host computed, with the sign cleared (the hardware's is 0x7fff). Scratch writes unguarded, Rd guarded.
|
|
const int x = TexOut, y = TexOut + 1, t = TexOut + 2;
|
|
Emit(Unguarded(w)); // HFMA2 x
|
|
Lop32i(y, x, 0x7fff7fff, 0); // y = |h| per half
|
|
Emit(Unguarded(0x1c00000000000000UL | (0x03ff03ffUL << 20) | ((ulong)y << 8) | t)); // IADD32I t = y + 0x03ff03ff
|
|
Emit(Unguarded(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)t << 20) | ((ulong)x << 8) | t)); // LOP.AND t = x & ~t
|
|
Emit(Unguarded(0x3828000000000000UL | (15UL << 20) | ((ulong)t << 8) | t)); // SHR t >>= 15
|
|
Lop32i(t, t, 0x00010001, 0); // bit 0 / 16: clear that half
|
|
Emit(Unguarded(0x5c18000000000000UL | (16UL << 39) | (1UL << 48) | ((ulong)t << 20) | ((ulong)t << 8) | t)); // ISCADD t = (t << 16) - t
|
|
// LOP.AND Rd = y & ~t (InvB bit 40), no predicate output (PT at 48)
|
|
EmitG(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)t << 20) | ((ulong)y << 8) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// MOVM.16.MT88 Rd, Ra (0x23a): an 8x8 matrix of 16-bit elements spread over the warp (lane t holds row t/4,
|
|
// columns 2(t%4) in the low half and 2(t%4)+1 in the high half) is transposed into the same layout. Element (r, c)
|
|
// lives in lane 4r + c/2, half c%2, so lane t takes both halves from lanes 8(t%4) + t/8 and that + 4, half
|
|
// (t/4)%2. Maxwell has no matrix move: two SHFL.IDX over the whole warp, the half picked by a shift, ISCADD to pack.
|
|
// Scratch is written unguarded so every lane takes part in the shuffles; only Rd is guarded.
|
|
void Movm()
|
|
{
|
|
if (_i.W1 != 0 || _i.W2 != 0) throw Fail("MOVM mode");
|
|
int src = Gpr(_i.Bits(24, 8)), d = Gpr((uint)Dst);
|
|
int lane = TexCoord, shift = TexCoord + 1, lo = TexMeta, hi = TexMeta + 1;
|
|
void shl(int r, int a, uint n) => Emit(Unguarded(0x3848000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)r));
|
|
void shr(int r, int a, uint n) => Emit(Unguarded(0x3828000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)r));
|
|
// SHFL.IDX r, value, lane register, clamp 31 (whole warp), no predicate output
|
|
void shfl(int r, int value, int from) => Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29)
|
|
| ((ulong)from << 20) | ((ulong)value << 8) | (uint)r));
|
|
Emit(Unguarded(0xf0c8000000000000UL | (uint)SX)); // S2R SX, SR_LANEID
|
|
Lop32i(lane, SX, 3, 0);
|
|
shl(lane, lane, 3); // 8(t%4)
|
|
shr(shift, SX, 3); // t/8
|
|
Emit(Unguarded(0x5c10000000000000UL | ((ulong)shift << 20) | ((ulong)lane << 8) | (uint)lane)); // IADD
|
|
shfl(lo, src, lane);
|
|
Emit(Unguarded(0x1c00000000000000UL | (4UL << 20) | ((ulong)lane << 8) | (uint)lane)); // IADD32I +4: next row
|
|
shfl(hi, src, lane);
|
|
shr(shift, SX, 2);
|
|
Lop32i(shift, shift, 1, 0);
|
|
shl(shift, shift, 4); // 16 * ((t/4) % 2)
|
|
Emit(Unguarded(0x5c28000000000000UL | ((ulong)shift << 20) | ((ulong)lo << 8) | (uint)lo)); // SHR by register
|
|
Lop32i(lo, lo, 0xffff, 0);
|
|
Emit(Unguarded(0x5c28000000000000UL | ((ulong)shift << 20) | ((ulong)hi << 8) | (uint)hi));
|
|
EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)lo << 20) | ((ulong)hi << 8) | (uint)d); // ISCADD (hi << 16) + lo
|
|
}
|
|
|
|
// POPC Rd, b (bit 63: of ~b).
|
|
void Popc()
|
|
{
|
|
if ((_i.W2 & ~0x08000000u) != 0) throw Fail("POPC mode");
|
|
var (b, _, inv) = Raw(1, SB);
|
|
EmitG(0x5c08000000000000UL | ((ulong)b << 20) | (inv ? 1UL << 40 : 0) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// F2FP.PACK_AB Rd, a, b: half(a) in the high 16 bits, half(b) in the low (round to nearest). With .MERGE_C
|
|
// (bit 78) the high half is the low 16 bits of the C operand instead.
|
|
void F2fp()
|
|
{
|
|
if (_i.Bits(62, 2) != 0 || _i.Bits(73, 5) != 0 || _i.Bits(79, 2) != 0 || _i.Bits(81, 10) != 0 || _i.Bit(91) || _i.Bit(72))
|
|
throw Fail("F2FP mode");
|
|
bool merge = _i.Bit(78);
|
|
int a = merge ? Plain(2, SC) : Plain(0, SA), b = Plain(1, SB);
|
|
if (merge) Mov(247, a, false);
|
|
else Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)a << 20) | 247));
|
|
Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)b << 20) | 246));
|
|
EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// SUST.P / SULD.P (formatted, bound descriptor): coordinates from Ra, data (stores) at 32-39 or destination
|
|
// (loads), channel mask 72-75, dimension 61-63 in Maxwell's surface numbering (1D, BUFFER_1D, ARRAY_1D, 2D,
|
|
// ARRAY_2D, 3D), descriptor offset 40-53 in bank 54-58. Strong stores (0x0010a000) are taken as plain ones.
|
|
void SurfaceAccess(bool store)
|
|
{
|
|
uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4), handle = Handle(true), bank = _i.Bits(54, 5);
|
|
uint mode = _i.W2 & ~0xf00u;
|
|
bool okMode = store ? mode is 0x00100000 or 0x0010a000 : mode == 0x001e0000;
|
|
if (!okMode || mask == 0 || dim > 5 || _i.Bits(59, 2) != 0 || handle > 0x1fff || !TexBank(bank))
|
|
throw Fail("surface access mode");
|
|
int coord = Gpr(_i.Bits(24, 8));
|
|
int data = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst);
|
|
EmitG((store ? 0xeb28000000000000UL : 0xeb08000000000000UL) | (uint)data | ((ulong)coord << 8) | ((ulong)mask << 20)
|
|
| ((ulong)dim << 33) | ((ulong)handle << 36));
|
|
}
|
|
|
|
// Descriptor offset (bits 40-53, in words) of a bound texture or image. NVN2 compute programs keep their handles
|
|
// in one table in the driver bank: texture slot k at start + 8k, image slot j at start + 8 * (textures + j), the
|
|
// start and counts from the control section (job "nvn2_table", runtime/gfx_shader.c). It is renumbered into the
|
|
// NVN1 layout the translator's host binds by: textures at 0x20 + 8k, images at 0x120 + 8j.
|
|
uint Handle(bool image)
|
|
{
|
|
uint h = _i.Bits(40, 14);
|
|
if (!_compute || _table == null) return h;
|
|
int entry = ((int)h * 4 - _table[0]) / 8;
|
|
if ((int)h * 4 < _table[0] || ((int)h * 4 - _table[0]) % 8 != 0 || entry >= _table[1] + _table[2])
|
|
throw Fail($"handle word {h} outside the binding table (start 0x{_table[0]:x}, {_table[1]} textures, {_table[2]} images)");
|
|
if (image != (entry >= _table[1])) throw Fail($"handle word {h} is a {(image ? "texture" : "image")} entry");
|
|
return image ? (uint)(0x120 + 8 * (entry - _table[1])) / 4 : (uint)(0x20 + 8 * entry) / 4;
|
|
}
|
|
|
|
// Bound texture descriptors: bank 2 for graphics stages, bank 0 for compute (NVN2 keeps the compute handles in
|
|
// the driver bank). The translator leaves the bank to the host, which reads the handle table as NVN1's c2.
|
|
bool TexBank(uint bank) => bank == 2 || (bank == 0 && _compute);
|
|
|
|
// TLD4 (gather), 2D or ARRAY_2D: component 87-88, offset mode 76-77 (1: one offset register, 2: per-tap offsets
|
|
// in two registers), depth compare 78, FP16 results 79, operands packed as TEX's. Maxwell TLD4: component 56-57,
|
|
// offset mode 54-55 (same numbering), DC 50.
|
|
void Tld4()
|
|
{
|
|
uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4), offMode = _i.Bits(76, 2);
|
|
bool dc = _i.Bit(78);
|
|
if (dim is not 1 and not 5 || offMode == 3 || _i.Bits(81, 3) != 7 || _i.Bit(59) || mask == 0 || _i.Bit(89))
|
|
throw Fail("TLD4 mode");
|
|
uint handle = Handle(false), bank = _i.Bits(54, 5);
|
|
if (handle > 0x1fff || !TexBank(bank)) throw Fail($"TLD4 descriptor c[{bank}] 0x{handle:x}");
|
|
int coords = TexCoords[dim];
|
|
int count = coords + (int)offMode + (dc ? 1 : 0);
|
|
int inA = count <= 4 ? (count + 1) / 2 : 4;
|
|
int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8);
|
|
int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA)));
|
|
for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false);
|
|
int next = coords, meta = TexMeta;
|
|
for (int k = 0; k < offMode; k++) Mov(meta++, Value(next++), false);
|
|
if (dc) Mov(meta++, Value(next++), false);
|
|
EmitG(0xc800000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20)
|
|
| ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49)
|
|
| (dc ? 1UL << 50 : 0) | (7UL << 51) | ((ulong)offMode << 54) | ((ulong)_i.Bits(87, 2) << 56));
|
|
TexResults(mask, _i.Bit(79));
|
|
}
|
|
|
|
// PLOP3.LUT Pu, Pv, a, b, c: table in bits 64-66 and 72-76, inputs a 87-90, b 77-80, c 68-71 (index | negate << 3),
|
|
// outputs 81-83 and 84-86 (the second with the table in bits 16-23). Each input becomes 0 or ~0 (SEL), LOP3 combines
|
|
// them and ISETP.NE sets the output. Bit 67 (c a uniform predicate) is not lowered for the vector form.
|
|
void Plop3()
|
|
{
|
|
uint lut = _i.Bits(64, 3) | _i.Bits(72, 5) << 3, t1 = _i.Bits(81, 3), t2 = _i.Bits(84, 3), lut2 = _i.Bits(16, 8);
|
|
const int ones = 240, ra = 241, rb = 242, rc = 243, r1 = 244, r2 = 245;
|
|
Mov32i(ones, 0xffffffff);
|
|
Sel(ra, ones, RZ, _i.Bits(87, 4), false);
|
|
Sel(rb, ones, RZ, _i.Bits(77, 4), false);
|
|
uint c = _i.Bits(68, 4);
|
|
if (_i.Bit(67) && !_uniform)
|
|
{
|
|
// c is UPn: 0 - ((RUP >> n) & 1) gives 0 or ~0 straight from the uniform predicate state; UPT is ~0.
|
|
if ((c & 7) == 7) Mov(rc, ones, false);
|
|
else
|
|
{
|
|
Emit(Unguarded(0x3828000000000000UL | ((ulong)(c & 7) << 20) | ((ulong)RUP << 8) | rc)); // SHR imm
|
|
Lop32i(rc, rc, 1, 0);
|
|
Iadd3(rc, RZ, rc, RZ, false, negB: true);
|
|
}
|
|
if ((c & 8) != 0) Lop32i(rc, rc, 0xffffffff, 2);
|
|
}
|
|
else Sel(rc, ones, RZ, c, false);
|
|
ulong lop3(int dst, uint table) => Unguarded(0x5be0000000000000UL | ((ulong)rc << 39) | ((ulong)rb << 20) | ((ulong)ra << 8) | (uint)dst | ((ulong)table << 28) | (7UL << 48));
|
|
Emit(lop3(r1, lut));
|
|
if (t2 != 7) Emit(lop3(r2, lut2));
|
|
ulong ne(uint p, int r) => 0x5b60000000000000UL | (5UL << 49) | ((ulong)PT << 39) | ((ulong)RZ << 20) | ((ulong)r << 8) | ((ulong)p << 3) | PT;
|
|
if (t1 != 7) EmitG(ne(t1, r1));
|
|
if (t2 != 7) EmitG(ne(t2, r2));
|
|
}
|
|
|
|
// LDS / STS: [Ra << (stride ? stride + 1 : 0) (+ a uniform register when bit 91 is set: bits 32-39 for loads,
|
|
// 64-71 for stores) + offset 40-63], size 73-75 (as Maxwell's), stride 78-79. STS data at 32-39.
|
|
void SharedAccess(uint op)
|
|
{
|
|
bool store = op != 0x984;
|
|
uint width = _i.Bits(73, 3), stride = _i.Bits(78, 2);
|
|
bool uniformBase = op is 0x984 or 0x988 && _i.Bit(91);
|
|
uint ignored = uniformBase ? (op == 0x988 ? 0x080000ffu : 0x08000000u) : 0u;
|
|
if (width > 6 || (_i.W2 & ~ignored) != (width << 9 | stride << 14) || (op == 0x984 && !uniformBase && _i.Bits(32, 8) != 0)
|
|
|| (op == 0x988 && !uniformBase))
|
|
throw Fail("shared memory addressing mode");
|
|
int addr = Gpr(_i.Bits(24, 8));
|
|
if (stride != 0 && addr != RZ)
|
|
{
|
|
Emit(Unguarded(0x3848000000000000UL | ((ulong)(stride + 1) << 20) | ((ulong)addr << 8) | SA)); // SHL imm
|
|
addr = SA;
|
|
}
|
|
if (uniformBase)
|
|
{
|
|
int u = Ur(_i.Bits(op == 0x988 ? 64 : 32, 8));
|
|
if (u != RZ) { Iadd3(SA, addr, u, RZ, false); addr = SA; }
|
|
}
|
|
int data = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst);
|
|
EmitG((store ? 0xef58000000000000UL : 0xef48000000000000UL) | (uint)data | ((ulong)addr << 8)
|
|
| ((ulong)_i.Bits(40, 24) << 20) | ((ulong)width << 48));
|
|
}
|
|
|
|
// BAR.SYNC 0 for the whole workgroup (bit 80, .DEFER_BLOCKING, needs nothing here).
|
|
void Bar()
|
|
{
|
|
if (_i.W1 != 0 || (_i.W2 & ~0x10000u) != 0 || _i.Bits(12, 4) != 7) throw Fail("BAR mode");
|
|
Emit(Unguarded(0xf0a8000000000000UL | (1UL << 43) | (1UL << 44) | (7UL << 39)));
|
|
}
|
|
|
|
// MEMBAR: Maxwell MEMBAR.GL whatever the scope. A barrier at least as wide as the one asked for orders the same
|
|
// accesses; the decoder reads MEMBAR's level from bits 8-9 (CTA 0, GL 1, SYS 2), and the SPIR-V it makes is a
|
|
// device-scope memory barrier for GL and SYS alike.
|
|
void Membar() => Emit(Unguarded(0xef98000000000000UL | (1UL << 8)));
|
|
|
|
// ISETP.EX: the high-word half of a 64-bit compare. The low words' result is the predicate of bits 68-71; EQ is
|
|
// (high equal AND low), NE (high different OR low), an ordering (high strictly ordered OR (high equal AND low)),
|
|
// signed with bit 73. The result is combined with Pc (87-90) by the operation in 74-75 into P (81-83) and Q
|
|
// (84-86), as ISETP does: Maxwell ISETPs into a saved scratch predicate (P6, or the highest one the instruction
|
|
// does not name), then PSETP.
|
|
void SetpExtended()
|
|
{
|
|
uint cmp = _i.Bits(76, 3), combine = _i.Bits(74, 2), low = _i.Bits(68, 4);
|
|
if (cmp is 0 or 7 || combine == 3) throw Fail("ISETP.EX mode");
|
|
var (a, _, _) = Raw(0, SA);
|
|
var (b, _, _) = Raw(1, SB);
|
|
// the scratch predicate: P6 unless the instruction names it (low result, Pc, guard or a destination)
|
|
int t = 6;
|
|
while (t >= 0 && (t == (int)(low & 7) || t == (int)_i.Bits(87, 3) || t == (int)_i.Bits(12, 3) ||
|
|
t == (int)_i.Bits(81, 3) || t == (int)_i.Bits(84, 3)))
|
|
t--;
|
|
if (t < 0) throw Fail("ISETP.EX: no free scratch predicate");
|
|
Emit(Unguarded(0x38e8000000000000UL | (1UL << (20 + t)) | ((ulong)RZ << 8) | SX)); // P2R: save P6
|
|
ulong isetp(uint c, uint input, uint bop, bool signedCmp) =>
|
|
0x5b60000000000000UL | ((ulong)t << 3) | PT | ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)(input & 7) << 39)
|
|
| ((ulong)(input >> 3) << 42) | ((ulong)bop << 45) | (signedCmp ? 1UL << 48 : 0) | ((ulong)c << 49);
|
|
bool sg = _i.Bit(73);
|
|
const uint And = 0, Or = 1, Lt = 1, Eq = 2, Gt = 4, Ne = 5;
|
|
if (cmp == Eq) EmitG(isetp(Eq, low, And, sg));
|
|
else if (cmp == Ne) EmitG(isetp(Ne, low, Or, sg));
|
|
else { EmitG(isetp(Eq, low, And, sg)); EmitG(isetp(cmp <= 3 ? Lt : Gt, (uint)t, Or, sg)); }
|
|
// PSETP P, Q = (P6 AND PT) combine Pc
|
|
EmitG(0x5090000000000000UL | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3) | ((ulong)t << 12) | ((ulong)PT << 29)
|
|
| ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)combine << 45));
|
|
Emit(Unguarded(0x38f0000000000000UL | (1UL << (20 + t)) | ((ulong)SX << 8))); // R2P: restore P6
|
|
}
|
|
|
|
// UBREV URd, URb: BREV on uniform registers (bit 91 marks the uniform datapath).
|
|
void Ubrev()
|
|
{
|
|
if ((_i.W2 & ~0x08000000u) != 0 || _i.W1 > 255) throw Fail("UBREV mode");
|
|
int d = Ur((uint)Dst);
|
|
if (d == RZ) return;
|
|
EmitG(0x3828000000000000UL | ((ulong)Ur(_i.Bits(32, 8)) << 8) | (1UL << 39) | (1UL << 40) | (uint)d); // SHR.BREV by 0
|
|
}
|
|
|
|
// UPOPC URd, URb: POPC on uniform registers, lowered through POPC (0x309) with register fields naming uniform
|
|
// registers (as UniformAlu does).
|
|
void Upopc()
|
|
{
|
|
if ((_i.W2 & ~0x08000000u) != 0 || _i.W1 > 255) throw Fail("UPOPC mode");
|
|
Insn saved = _i;
|
|
_i.Lo = (_i.Lo & ~0xfffUL) | 0x309;
|
|
_i.Hi &= ~(1UL << 27);
|
|
_uniform = true;
|
|
try { Popc(); }
|
|
finally { _uniform = false; _i = saved; }
|
|
}
|
|
|
|
// UP2UR URd, UPR, URa, mask: P2R over the uniform predicates, byte 0 only: URd = (URa & ~mask) | (UP & mask),
|
|
// the mask in bits 32-38. RUP holds UPk in bit k.
|
|
void Up2ur()
|
|
{
|
|
if (_i.W1 > 0x7f || (_i.W2 & ~0x08000000u) != 0) throw Fail("UP2UR mode");
|
|
int d = Ur((uint)Dst), a = Ur(_i.Bits(24, 8));
|
|
if (d == RZ) return;
|
|
uint mask = _i.W1;
|
|
Lop32i(SX, RUP, mask, 0); // AND
|
|
if (a == RZ) { Mov(d, SX); return; }
|
|
Lop32i(SA, a, ~mask, 0);
|
|
Iadd3(d, SA, SX, RZ, true); // disjoint bits: the sum is the OR
|
|
}
|
|
|
|
// F2IP.U8.F32 Rd, Ra, Rb, RZ (opcode 0x243): two floats converted to unsigned bytes and packed into the low
|
|
// half of Rd. Rb's byte goes in bits 0-7, Ra's byte in bits 8-15, and bits 16-31 are zero. With Ra = RZ only
|
|
// Rb's byte is written. W1 holds only Rb (bits 32-39), and the C operand (bits 64-71) must be RZ. Rounding is
|
|
// in bits 78-79, using the same codes as Maxwell's F2I RoundMode (bits 39-40):
|
|
// W2 0x0000c4ff: bits 78-79 = 3, round toward zero;
|
|
// W2 0x000004ff: bits 78-79 = 0, round to nearest even. The compiler uses this form for unorm values it has
|
|
// already multiplied by 255.
|
|
// Each byte goes through NaN -> 0, a float clamp to [0, 255], then F2I.U32.F32 with that rounding. The clamp
|
|
// has to be explicit. The translator's F2I (Instructions/InstEmitConversion.cs, EmitF2I) clamps only
|
|
// U16/S16 destinations, and IDstFmt has no U8, so a plain F2I.U8 lets values above 255 through. It also feeds
|
|
// NaN to FMax, whose SPIR-V result is undefined.
|
|
void F2ipByte()
|
|
{
|
|
if ((_i.W1 & ~0xffu) != 0 || (_i.W2 != 0x0000c4ff && _i.W2 != 0x000004ff)) throw Fail("packed float-to-byte mode");
|
|
int a = Gpr(_i.Bits(24, 8)), b = Gpr(_i.Bits(32, 8)), d = Gpr((uint)Dst);
|
|
if (d == RZ) return;
|
|
uint round = _i.Bits(78, 2);
|
|
|
|
// Convert src to an integer 0-255. Steps 1-4 work in scratch register t without a guard. Step 5 writes dst
|
|
// and is guarded only when dst is the guest register.
|
|
void ToByte(int src, int t, int dst, bool guarded)
|
|
{
|
|
// FSET.NUM.AND t = (src is not NaN) ? 0xffffffff : 0 (FComp 7 in 48-51, source predicate PT in 39-41,
|
|
// BVal clear so the result is an integer mask)
|
|
Emit(Unguarded(0x5800000000000000UL | (7UL << 48) | ((ulong)PT << 39) | ((ulong)src << 20) | ((ulong)src << 8) | (uint)t));
|
|
// LOP.AND t = src & t: NaN becomes +0.0 (destination predicate PT in 48-50)
|
|
Emit(Unguarded(0x5c40000000000000UL | ((ulong)PT << 48) | ((ulong)t << 20) | ((ulong)src << 8) | (uint)t));
|
|
// FMNMX t = max(t, 0.0): immediate form, predicate !PT (39-42) selects the maximum
|
|
Emit(Unguarded(0x3860000000000000UL | ((ulong)PT << 39) | (1UL << 42) | ((ulong)t << 8) | (uint)t));
|
|
// FMNMX t = min(t, 255.0): 255.0f is 0x437f0000, whose top 20 bits 0x437f0 fit the immediate field
|
|
Emit(Unguarded(0x3860000000000000UL | (0x437f0UL << 20) | ((ulong)PT << 39) | ((ulong)t << 8) | (uint)t));
|
|
// F2I.U32.F32 dst = t, with the instruction's rounding (IDstFmt U32 in 8-9, SrcFmt F32 in 10-11, FTZ 44)
|
|
Put(0x5cb0000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)round << 39) | (1UL << 44) | ((ulong)t << 20) | (uint)dst, guarded);
|
|
}
|
|
|
|
if (a == RZ)
|
|
{
|
|
// Single byte: convert straight into Rd. Steps 1-4 read b and write only SA, so this is safe when d == b.
|
|
ToByte(b, SA, d, true);
|
|
return;
|
|
}
|
|
// Both bytes go to scratch before Rd is written, so d may alias a or b.
|
|
ToByte(b, SA, SA, false);
|
|
ToByte(a, SB, SB, false);
|
|
// ISCADD Rd = (SB << 8) + SA. Both values are 0-255, so the sum equals the OR of the two bytes.
|
|
EmitG(0x5c18000000000000UL | (8UL << 39) | ((ulong)SA << 20) | ((ulong)SB << 8) | (uint)d);
|
|
}
|
|
|
|
// TXD (0xb6c): a sample with explicit derivatives, 2D or ARRAY_2D, bound descriptor. Fields as TEX's: Rd 16-23,
|
|
// Ra 24-31 (x, y, then for ARRAY_2D the layer as an integer in its low 16 bits), Rb 32-39 (the four derivatives,
|
|
// interleaved: dx.x, dy.x, dx.y, dy.y), descriptor word offset 40-53 in bank 54-58, dimension 61-63, second
|
|
// destination 64-71, mask 72-75, bit 90 no-dependency. Offsets (76), FP16 results (79) and the bindless forms
|
|
// (59-60) are not lowered. Bits 81-84 are set in every TXD seen (81-83: no sparse predicate).
|
|
// Maxwell TXD (0xde00) takes the coordinates and the layer the same way in Ra.., but the translator's EmitTxd reads Rb..
|
|
// as dPdx (2 values) then dPdy (2 values), so the derivatives are reordered on the way into TexMeta.
|
|
// With no second destination the third and fourth channels are dropped (SM70+ register pairs), so the mask
|
|
// handed to Maxwell keeps only its two lowest channels and TexResults writes Rd, Rd+1; otherwise TexResults
|
|
// writes Rd, Rd+1 and then the second destination, which need not follow Rd.
|
|
void Txd()
|
|
{
|
|
uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4);
|
|
if ((_i.W2 & ~(0xfffu | 1u << 26)) != 0x001e0000 || _i.Bits(59, 2) != 0 || dim is not 1 and not 5 || mask == 0)
|
|
throw Fail("TXD mode");
|
|
uint handle = Handle(false), bank = _i.Bits(54, 5);
|
|
if (handle > 0x1fff) throw Fail("TXD descriptor offset");
|
|
if (!TexBank(bank)) throw Fail($"TXD descriptor bank {bank}");
|
|
int d2 = (int)_i.Bits(64, 8);
|
|
if (Dst == RZ && d2 == RZ) return; // nothing is written, and sampling has no side effects
|
|
if (d2 == RZ)
|
|
{
|
|
uint upper = mask & (mask - 1); // drop the lowest channel...
|
|
upper &= upper - 1; // ...and the next: what is left has no destination
|
|
mask ^= upper;
|
|
}
|
|
int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8);
|
|
int coords = dim == 5 ? 3 : 2;
|
|
for (int k = 0; k < coords; k++) Mov(TexCoord + k, ra == RZ ? RZ : Gpr((uint)(ra + k)), false);
|
|
// SM86 order dx.x, dy.x, dx.y, dy.y; the translator's Maxwell order dx.x, dx.y, dy.x, dy.y.
|
|
int[] order = { 0, 2, 1, 3 };
|
|
for (int k = 0; k < 4; k++) Mov(TexMeta + k, rb == RZ ? RZ : Gpr((uint)(rb + order[k])), false);
|
|
EmitG(0xde00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20) | ((ulong)TexMaxwellDim[dim] << 28)
|
|
| ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49) | (7UL << 51));
|
|
TexResults(mask, false);
|
|
}
|
|
|
|
// LDSM.16.M88.{1,2,4} and LDSM.16.MT88.{1,2,4} (ldmatrix): count = 1 << bits 72-73 (1, 2 or 4) 8x8 matrices of
|
|
// 16-bit elements from shared memory into Rd..Rd+count-1 (Rd aligned to count). Lanes 8m..8m+7 hold, in Ra, the
|
|
// byte addresses of rows 0-7 of matrix m; bit 91 adds a uniform register (bits 32-39, as in SharedAccess) and every
|
|
// row adds the immediate offset 40-63. Without transpose (bit 78 clear) lane t receives in Rd+m the 32 bits at row
|
|
// t/4, bytes 4(t%4)..4(t%4)+3 of matrix m. With transpose (.MT88) it receives column t/4 of row 2(t%4) in the low
|
|
// half and of row 2(t%4)+1 in the high half.
|
|
// Maxwell has no matrix load, so each lane fetches the row address it needs from the lane that holds it
|
|
// (SHFL.IDX across the whole warp), adds its own column and loads with LDS: one LDS.32 per matrix, or for .MT88 two
|
|
// LDS.U16 per matrix joined by ISCADD (high << 16) + low. Address arithmetic and the shuffles are unguarded
|
|
// scratch writes so every lane takes part in the shuffles (the hardware needs the whole warp too); the loads and
|
|
// the writes to Rd are guarded by the instruction's predicate.
|
|
// Scratch: SX lane id, then the column bytes (non-transposed); TexCoord the source lane of row 0 (or of row
|
|
// 2(t%4)), TexCoord+1 the lane of later matrices (or the column bytes for .MT88), TexCoord+2 the lane for .MT88;
|
|
// SA the fetched row address; SC the base address when it needs a copy; TexMeta/TexMeta+1 the two halves.
|
|
void Ldsm()
|
|
{
|
|
uint sizeCode = _i.Bits(72, 2);
|
|
bool transpose = _i.Bit(78), uniformOffset = _i.Bit(91);
|
|
// Allowed in word 2: size 72-73 (0x300), transpose 78 (0x4000), uniform offset 91 (0x08000000).
|
|
if (sizeCode == 3 || (_i.W2 & ~0x08004300u) != 0 || (!uniformOffset && _i.Bits(32, 8) != 0))
|
|
throw Fail("LDSM mode");
|
|
int count = 1 << (int)sizeCode;
|
|
if (Dst == RZ) return; // a load into RZ has no visible effect
|
|
if (Dst % count != 0 || Dst + count > RZ) throw Fail("LDSM destination alignment");
|
|
var dst = new int[count];
|
|
for (int m = 0; m < count; m++) dst[m] = Gpr((uint)(Dst + m));
|
|
|
|
int addr = Gpr(_i.Bits(24, 8));
|
|
if (uniformOffset)
|
|
{
|
|
int u = Ur(_i.Bits(32, 8));
|
|
if (u != RZ) { Iadd3(SC, addr, u, RZ, false); addr = SC; }
|
|
}
|
|
// A destination that is also the base would be overwritten before the later matrices fetch their rows.
|
|
if (addr != RZ && Array.IndexOf(dst, addr) >= 0) { Mov(SC, addr, false); addr = SC; }
|
|
ulong offset = (ulong)_i.Bits(40, 24) << 20;
|
|
|
|
// Unguarded Maxwell helpers: SHR/SHL by an immediate (unsigned), IADD32I, and SHFL.IDX of the base address
|
|
// from a lane held in a register, clamp 31 with segment mask 0 (the whole warp), valid predicate to PT.
|
|
void shr(int d, int a, int n) => Emit(Unguarded(0x3828000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)d));
|
|
void shl(int d, int a, int n) => Emit(Unguarded(0x3848000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)d));
|
|
void addImm(int d, int a, uint imm) => Emit(Unguarded(0x1c00000000000000UL | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)d));
|
|
void fetchRow(int d, int lane) => Emit(Unguarded(0xef10000000000000UL | (uint)d | ((ulong)addr << 8) | ((ulong)lane << 20)
|
|
| (0x1fUL << 34) | (1UL << 29) | (7UL << 48)));
|
|
// LDS d, [a + offset], size as Maxwell's LsSize2 (2 U16, 4 B32), guarded.
|
|
void lds(int d, int a, ulong size) => EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | offset | (size << 48));
|
|
|
|
Emit(Unguarded(0xf0c8000000000000UL | SX)); // S2R SX, SR_LANEID (0)
|
|
if (!transpose)
|
|
{
|
|
shr(TexCoord, SX, 2); // lane holding row t/4 of matrix 0
|
|
Lop32i(SX, SX, 3, 0); // t & 3 (LOP32I.AND)
|
|
shl(SX, SX, 2); // column bytes 4(t%4)
|
|
for (int m = 0; m < count; m++)
|
|
{
|
|
int lane = TexCoord;
|
|
if (m != 0) { addImm(TexCoord + 1, TexCoord, (uint)(8 * m)); lane = TexCoord + 1; }
|
|
fetchRow(SA, lane);
|
|
Iadd3(SA, SA, SX, RZ, false);
|
|
lds(dst[m], SA, 4); // LDS.32
|
|
}
|
|
return;
|
|
}
|
|
Lop32i(TexCoord, SX, 3, 0);
|
|
shl(TexCoord, TexCoord, 1); // lane holding row 2(t%4) of matrix 0
|
|
shr(TexCoord + 1, SX, 2);
|
|
shl(TexCoord + 1, TexCoord + 1, 1); // column bytes 2(t/4)
|
|
for (int m = 0; m < count; m++)
|
|
{
|
|
for (int half = 0; half < 2; half++)
|
|
{
|
|
int lane = TexCoord;
|
|
uint step = (uint)(8 * m + half);
|
|
if (step != 0) { addImm(TexCoord + 2, TexCoord, step); lane = TexCoord + 2; }
|
|
fetchRow(SA, lane);
|
|
Iadd3(SA, SA, TexCoord + 1, RZ, false);
|
|
lds(TexMeta + half, SA, 2); // LDS.U16 (zero-extended)
|
|
}
|
|
// ISCADD Rd+m = (high << 16) + low
|
|
EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)TexMeta << 20) | ((ulong)(TexMeta + 1) << 8) | (uint)dst[m]);
|
|
}
|
|
}
|
|
|
|
// HMMA.1688 / HMMA.16816 Rd, Ra, Rb, Rc with F16 inputs: a warp-wide matrix multiply-accumulate D(16x8) =
|
|
// A(16xK) * B(Kx8) + C(16x8), K = 8, or 16 when bit 75 is set. Bit 76 selects F32 accumulators and results
|
|
// (.F32: C and D are four registers each) instead of F16 ones (.F16: a register pair). Fields: D 16-23, A 24-31
|
|
// (K/4 registers), B 32-39 (K/8 registers), C 64-71 (RZ means a zero accumulator). With g = lane / 4 and
|
|
// q = lane % 4, each lane holds (the lower-numbered column or k in the low half of every packed register):
|
|
// A register 2j + r: row g + 8r, k 8j + 2q and 8j + 2q + 1 (j is 0 only when K = 8)
|
|
// B register j: k 8j + 2q and 8j + 2q + 1, column g
|
|
// C / D, .F16: register r holds row g + 8r, columns 2q and 2q + 1
|
|
// C / D, .F32: register 2r + c holds row g + 8r, column 2q + c
|
|
// Maxwell has no tensor instructions, so the product is expanded per lane. For each k pair (2p, 2p + 1) the lane
|
|
// fetches the A word of its own rows from lane 4g + p % 4 and, for each of its two columns n = 2q + c, the B word
|
|
// from lane 4n + p % 4 (SHFL.IDX), widens the halves to F32 (F2F.F32.F16, exact) and accumulates with FFMA in
|
|
// F32 (an F16 x F16 product is exact in F32, so the step rounds once whether or not the host fuses it).
|
|
// The .F16 form rounds to F16 once, after all K products, as the hardware does: it does not round between the
|
|
// two eight-k halves of HMMA.16816. Measured on an NVIDIA GPU (tests/tensor_lowering_check.py compares this
|
|
// lowering with those results): a first-half sum of 65536 that the second half cancels gives 0, which an F16
|
|
// rounding after the first half turns into +inf, and 1 + 2^-11 + 2^-11 split over the halves gives 1 + 2^-10,
|
|
// which that rounding turns into 1. The F16 results are then packed as F2fp in this file does (F2F.F16 writes
|
|
// the low half and zeroes the high half, ISCADD merges the column pair).
|
|
// The work runs unguarded because every lane must supply its fragments to the shuffles; only the result writes
|
|
// take the instruction's guard. Scratch R239-R252 (this file's scratch range, see the header); the guest operands
|
|
// are never there (guest and uniform registers end below R237, spill temporaries and uniform registers are R156-R235).
|
|
void Hmma()
|
|
{
|
|
bool k16 = _i.Bit(75), f32 = _i.Bit(76);
|
|
uint d = (uint)Dst, a = _i.Bits(24, 8), b = _i.Bits(32, 8), c = _i.Bits(64, 8);
|
|
int aRegs = k16 ? 4 : 2, bRegs = k16 ? 2 : 1, cdRegs = f32 ? 4 : 2;
|
|
// A fragment base aligned to its size and the whole fragment below RZ (RZ itself reads as zeros).
|
|
bool fits(uint reg, int n) => reg == RZ || (reg % (uint)n == 0 && reg + (uint)n <= RZ);
|
|
// Word 1 above B, word 2 other than C, the shape (75) and the accumulator type (76), and word 3 bits 96-104 /
|
|
// 126-127 select modes (bit 74 another shape, HMMA.1684; other input types) that are not lowered.
|
|
if ((_i.W1 & ~0xffu) != 0 || (_i.W2 & ~0x18ffu) != 0 || ((uint)(_i.Hi >> 32) & 0xc00001ffu) != 0
|
|
|| !fits(d, cdRegs) || !fits(a, aRegs) || !fits(b, bRegs) || !fits(c, cdRegs))
|
|
throw Fail("HMMA mode");
|
|
if (d == RZ) return; // no result and no other effect
|
|
|
|
// Register k of a fragment (RZ stays RZ); Gpr maps spilled guest registers to this instruction's temporaries.
|
|
int part(uint reg, int k) => reg == RZ ? RZ : Gpr(reg + (uint)k);
|
|
|
|
const int laneA = 239; // 4g: the lane of A's row group for k pair 0
|
|
const int laneB = 240; // 8q: the lane holding B column 2q for k pair 0
|
|
const int lane = 241; // the lane of the current shuffle
|
|
const int word = 242; // the shuffled packed-half word
|
|
const int aw = 243; // 243-246: A element (row g + 8r, k 2p + h) at aw + 2r + h, F32
|
|
const int bw = 247; // 247-248: B element (k 2p + h, current column) at bw + h, F32
|
|
const int acc = 249; // 249-252: accumulator (row g + 8r, column 2q + c) at acc + 2r + c, F32
|
|
|
|
void widen(int target, int src, bool high) => // F2F.F32.F16 target, src.H0 / .H1
|
|
Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)src << 20) | (high ? 1UL << 41 : 0) | (uint)target));
|
|
void narrow(int t) => // F2F.F16.F32 t, t (round to nearest; high half zero)
|
|
Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)t << 20) | (uint)t));
|
|
void shuffle(int target, int src, int laneReg) => // SHFL.IDX PT, target, src, laneReg, 0x1f (as Redux)
|
|
Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)laneReg << 20) | ((ulong)src << 8) | (uint)target));
|
|
int laneAt(int baseLane, uint offset) // baseLane + offset in `lane` (IADD32I), or baseLane itself
|
|
{
|
|
if (offset == 0) return baseLane;
|
|
Emit(Unguarded(0x1c00000000000000UL | ((ulong)offset << 20) | ((ulong)baseLane << 8) | lane));
|
|
return lane;
|
|
}
|
|
|
|
// Accumulators from C, or zero: accumulator 2r + col is register 2r + col of an .F32 C, half col of register
|
|
// r of an .F16 C.
|
|
for (int k = 0; k < 4; k++)
|
|
{
|
|
if (c == RZ) Mov32i(acc + k, 0);
|
|
else if (f32) Mov(acc + k, part(c, k), false);
|
|
else widen(acc + k, part(c, k >> 1), (k & 1) == 1);
|
|
}
|
|
|
|
// Lane bases: 8q first (it needs the raw lane id), then 4g.
|
|
Put(0xf0c8000000000000UL | laneA, false); // S2R SR_LANEID
|
|
Lop32i(laneB, laneA, 3, 0); // q
|
|
Emit(Unguarded(0x3848000000000000UL | (3UL << 20) | ((ulong)laneB << 8) | laneB)); // SHL imm: 8q
|
|
Lop32i(laneA, laneA, 0x1c, 0); // 4g
|
|
|
|
int pairs = k16 ? 8 : 4;
|
|
for (int p = 0; p < pairs; p++)
|
|
{
|
|
uint pq = (uint)(p & 3); // the source lanes' q for this k pair
|
|
int j = p >> 2; // which 8-k block of the fragments
|
|
int srcA = laneAt(laneA, pq);
|
|
for (int r = 0; r < 2; r++)
|
|
{
|
|
shuffle(word, part(a, 2 * j + r), srcA);
|
|
widen(aw + 2 * r, word, false);
|
|
widen(aw + 2 * r + 1, word, true);
|
|
}
|
|
for (int col = 0; col < 2; col++)
|
|
{
|
|
shuffle(word, part(b, j), laneAt(laneB, 4u * (uint)col + pq));
|
|
widen(bw, word, false);
|
|
widen(bw + 1, word, true);
|
|
for (int r = 0; r < 2; r++)
|
|
for (int h = 0; h < 2; h++)
|
|
{
|
|
int t = acc + 2 * r + col; // FFMA t = A(r, 2p + h) * B(2p + h, col) + t
|
|
Emit(Unguarded(0x5980000000000000UL | ((ulong)t << 39) | ((ulong)(bw + h) << 20) | ((ulong)(aw + 2 * r + h) << 8) | (uint)t));
|
|
}
|
|
}
|
|
}
|
|
|
|
// .F32: D register 2r + col is accumulator 2r + col.
|
|
if (f32)
|
|
{
|
|
for (int k = 0; k < 4; k++) Mov(part(d, k), acc + k);
|
|
return;
|
|
}
|
|
// .F16: one rounding of each sum to F16, then D register r = column 2q + 1 in the high half, column 2q in the
|
|
// low half (ISCADD hi << 16 + lo).
|
|
for (int t = acc; t < acc + 4; t++) narrow(t);
|
|
for (int r = 0; r < 2; r++)
|
|
EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)(acc + 2 * r) << 20) | ((ulong)(acc + 2 * r + 1) << 8) | (uint)part(d, r));
|
|
}
|
|
|
|
|
|
// UFLO URd, URb: FLO on uniform registers (bit 74 .SH, bit 63 of ~b).
|
|
void Uflo()
|
|
{
|
|
if ((_i.W2 & ~(0x400u | 0x08000000u)) != 0x000e0000) throw Fail("UFLO mode");
|
|
int d = Ur((uint)Dst);
|
|
if (d == RZ) return;
|
|
EmitG(0x5c30000000000000UL | ((ulong)Ur(_i.Bits(32, 8)) << 20) | (_i.Bit(63) ? 1UL << 40 : 0) | ((ulong)_i.Bits(74, 1) << 41) | (uint)d);
|
|
}
|
|
|
|
// I2F: source signed (bit 74) and size (84-85: 8/16/32 bits, its byte or half in 60-61), destination size
|
|
// (75-76: F16, F32, F64), rounding 78-79. Maxwell I2F: ISrcFmt (size code in 10-11, signed in bit 13), ByteSel, DstFmt.
|
|
void I2f(uint op)
|
|
{
|
|
const uint known = 1u << 10 | 3u << 11 | 3u << 14 | 3u << 20;
|
|
uint w2 = _i.W2 & ~(op == 0xd06 ? 0x08000000u : 0u);
|
|
uint src = _i.Bits(84, 2), dst = _i.Bits(75, 2), sel = _i.Bits(60, 2);
|
|
if ((w2 & ~known) != 0 || src == 3 || dst == 0 || (op != 0xb06 && (_i.W1 & 0xcfffff00u) != 0)) throw Fail("I2F format");
|
|
int b = op == 0xb06 ? Plain(1, SB) : op == 0xd06 ? Ur(_i.Bits(32, 8)) : Gpr(_i.Bits(32, 8));
|
|
uint bytesel = src == 0 ? sel : src == 1 ? sel * 2 : 0;
|
|
EmitG(0x5cb8000000000000UL | ((ulong)dst << 8) | ((ulong)src << 10) | (_i.Bit(74) ? 1UL << 13 : 0) | ((ulong)b << 20)
|
|
| ((ulong)bytesel << 41) | ((ulong)_i.Bits(78, 2) << 39) | (uint)Gpr((uint)Dst));
|
|
}
|
|
|
|
// STL / LDL (local memory) at [Ra + offset 40-63], size 73-75 (U8 S8 U16 S16 32 64 128); 64 and 128 bits go as
|
|
// 32-bit words. STL data at 32-39. With bit 91 the address adds a uniform register URb (STL 0x987, LDL 0x983 in
|
|
// that form): a store's at 64-69, a load's at 32-37 (its B field; Ocarina's mesh programs write that register just
|
|
// before the access). Ra + URb goes to a scratch register first, or URb's register is the address when Ra is RZ.
|
|
void LocalAccess(bool store)
|
|
{
|
|
uint size = _i.Bits(73, 3), allowed = store ? 0xe00u : 0x200e00u;
|
|
bool uniform = _i.Bit(91);
|
|
uint w2 = uniform ? _i.W2 & ~(store ? 0x0800003fu : 0x08000000u) : _i.W2;
|
|
uint bField = _i.Bits(32, 8) & (uniform && !store ? 0xc0u : 0xffu);
|
|
if (size > 6 || (w2 & ~allowed) != 0x00100000 || (!store && bField != 0) || (store && _i.Bits(16, 8) != 0))
|
|
throw Fail("local memory mode");
|
|
int addr = Gpr(_i.Bits(24, 8)), value = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst);
|
|
if (uniform)
|
|
{
|
|
int ub = Ur(store ? _i.Bits(64, 6) : _i.Bits(32, 6));
|
|
if (ub != RZ && addr == RZ) addr = ub;
|
|
else if (ub != RZ) { Iadd3(SX, addr, ub, RZ, false); addr = SX; }
|
|
}
|
|
uint off = _i.Bits(40, 24), words = size == 6 ? 4u : size == 5 ? 2u : 1u;
|
|
if (!store && words > 1 && addr != RZ && addr >= value && addr < value + words) { Mov(SX, addr, false); addr = SX; }
|
|
for (uint k = 0; k < words; k++)
|
|
EmitG((store ? 0xef50000000000000UL : 0xef40000000000000UL) | (uint)(value == RZ ? RZ : value + (int)k) | ((ulong)addr << 8)
|
|
| ((ulong)((off + 4 * k) & 0xffffff) << 20) | ((ulong)(size < 4 ? size : 4) << 48));
|
|
}
|
|
|
|
// IPA with the attribute indexed by a uniform register (bits 24-31): Maxwell IPA.IDX.PASS with the whole byte
|
|
// address in Ra. As Ipa: the value times 1/w, which the program multiplies by w itself.
|
|
void IpaIndexed()
|
|
{
|
|
if (_i.Bits(32, 8) != RZ || (_i.Bits(76, 4) != 0 && _i.Bits(76, 4) != 4) || _i.Bits(81, 3) != 7) throw Fail("indexed IPA mode");
|
|
int idx = Ur(_i.Bits(24, 8)), d = Gpr((uint)Dst);
|
|
uint attr = _i.Bits(64, 8);
|
|
if (idx == RZ)
|
|
{
|
|
EmitG(0xe000000000000000UL | ((ulong)RZ << 8) | ((ulong)RZ << 20) | ((ulong)attr << 30) | ((ulong)_i.Bits(78, 1) << 55) | ((ulong)PT << 47) | (uint)d);
|
|
return;
|
|
}
|
|
Emit(Unguarded(0x1c00000000000000UL | ((ulong)(attr * 4) << 20) | ((ulong)idx << 8) | SX)); // IADD32I
|
|
EmitG(0xe000000000000000UL | ((ulong)SX << 8) | ((ulong)RZ << 20) | (1UL << 38) | ((ulong)_i.Bits(78, 1) << 55)
|
|
| ((ulong)PT << 47) | (uint)d);
|
|
}
|
|
|
|
// ULDC URd, c[bank][URa + offset]: Maxwell LDC with a register index; width 73-75 as LDC's.
|
|
void UldcIndexed()
|
|
{
|
|
uint width = _i.Bits(73, 3);
|
|
int d = Ur((uint)Dst), idx = Ur(_i.Bits(24, 8)), bank = Bank(_i.Bits(54, 5));
|
|
if (width > 5 || (_i.W2 & ~0xe00u) != 0x08000000 || bank >= 18 || _i.Bits(62, 2) != 0) throw Fail("indexed ULDC mode");
|
|
if (d == RZ) return;
|
|
EmitG(0xef90000000000000UL | (uint)d | ((ulong)idx << 8) | ((ulong)_i.Bits(38, 16) << 20) | ((ulong)bank << 36) | ((ulong)width << 48));
|
|
}
|
|
|
|
// BRX Ra / BRXU URa with the program start as base (the offset makes the target Ra itself): a switch. The
|
|
// compiler loads Ra from a jump table in c1 of native byte addresses, LDC Ra, c[0x1][Rx + base], after
|
|
// clamping the index (IMNMX.U32 Ry, Ry, bound; SHL Rx, Ry, 2). The translator's BRX compares the register with
|
|
// the lowered addresses, so it cannot take native ones; each table entry becomes a compare and branch instead:
|
|
// IADD32I.CC RZ, Ra, -target; BRA CC.EQ to the target's lowered code, the last entry also unconditionally.
|
|
void Brx(bool uniform)
|
|
{
|
|
if (_i.Bits(12, 4) != 7) throw Fail("predicated BRX");
|
|
if ((_i.W2 & ~0x3ffffu) != (uniform ? 0x0b800000u : 0x03800000u)) throw Fail("BRX mode");
|
|
long raw = (long)(_i.W1 | ((ulong)(_i.W2 & 0x3ffff) << 32));
|
|
if ((long)_index * 16 + 16 + ((raw << 14) >> 14) != 0) throw Fail("BRX relative to a base other than the program start");
|
|
uint src = _i.Bits(24, 8);
|
|
var targets = BrxTargets(src, uniform);
|
|
int reg = uniform ? Ur(src) : Gpr(src);
|
|
foreach (uint t in targets)
|
|
{
|
|
Emit(Unguarded(0x1c00000000000000UL | (1UL << 52) | ((ulong)(uint)(-(int)t) << 20) | ((ulong)reg << 8) | RZ));
|
|
_branches.Add((_out.Count, (int)(t / 16)));
|
|
Emit(0xe240000000000000UL | ((ulong)PT << 16) | 2); // BRA CC.EQ
|
|
}
|
|
_branches.Add((_out.Count, (int)(targets[^1] / 16)));
|
|
Emit(0xe24000000000000fUL | ((ulong)PT << 16));
|
|
}
|
|
|
|
// The jump table behind a BRX register: the LDC (or, for BRXU, indexed ULDC) of bank 1 that last wrote it in
|
|
// the straight-line code before the BRX, and its entry count from the IMNMX bound of the index feeding it.
|
|
List<uint> BrxTargets(uint src, bool uniform)
|
|
{
|
|
if (C1 == null || C1.Length == 0) throw Fail("BRX without the program's c1 constants");
|
|
int Writer(int from, uint reg, bool ur)
|
|
{
|
|
for (int k = from; k >= 0 && k >= from - 64; k--)
|
|
{
|
|
var p = _code[k];
|
|
uint op = p.Bits(0, 12), b = op & 0x1ff;
|
|
// control flow: not straight-line. BSSY, BSYNC and WARPSYNC branch nowhere and write no register: the
|
|
// compiler puts a BSSY for the switch's reconvergence between the IMNMX bound and the SHL.
|
|
if (b is >= 0x140 and <= 0x15f && op is not 0x945 and not 0x941 and not 0x948) return -1;
|
|
// uniform datapath: ULDC, UMOV and the U* ALU ops (base 0x080-0x0bf; UISETP/UPLOP3 write predicates)
|
|
bool isUr = op is 0xab9 or 0xabb or 0x882 or 0xc82 || (b is >= 0x080 and < 0x0c0 && b != 0x08c && b != 0x09c);
|
|
if (p.Bits(16, 8) == reg && isUr == ur) return k;
|
|
}
|
|
return -1;
|
|
}
|
|
int ldc = Writer(_index - 1, src, uniform);
|
|
if (ldc < 0) throw Fail("BRX register not loaded from a jump table");
|
|
var l = _code[ldc];
|
|
uint lop = l.Bits(0, 12);
|
|
if (!(lop == 0xb82 && !uniform || lop == 0xabb && uniform) || l.Bits(54, 5) != 1)
|
|
throw Fail($"BRX register loaded by op {lop:x3}, not from c1");
|
|
uint tableBase = l.Bits(38, 16);
|
|
// the bound: the index register of the load, written by SHL x, 2 from an IMNMX.U32 with an immediate
|
|
int count = 0;
|
|
int shl = Writer(ldc - 1, l.Bits(24, 8), uniform);
|
|
if (shl >= 0)
|
|
{
|
|
var sh = _code[shl];
|
|
int mn = Writer(shl - 1, sh.Bits(24, 8), uniform);
|
|
// IMNMX / UIMNMX with an immediate bound, or SEL / USEL Rx, Rx, bound under the compare that clamps it
|
|
// (the uniform switches: UISETP.GT P0, URx, bound; USEL URx, URx, bound, P0)
|
|
uint mop = mn >= 0 ? _code[mn].Bits(0, 12) : 0, mb = mop & 0x7f;
|
|
if (mn >= 0 && (mb == 0x17 || mb == 0x07) && _code[mn].Bits(9, 3) == 4) count = (int)_code[mn].W1 + 1;
|
|
// the register form, the bound in a register a MOV (UMOV) of an immediate set before it: SEL Rx, Rx, Rb
|
|
// under ISETP Rx, Rb, or IMNMX Rx, Rx, Rb; either source may be the bound, the other is the index
|
|
else if (mn >= 0 && (mb == 0x17 || mb == 0x07) && _code[mn].Bits(9, 3) == 1)
|
|
foreach (uint r in new[] { _code[mn].Bits(32, 8), _code[mn].Bits(24, 8) })
|
|
{
|
|
int mv = r == RZ ? -1 : Writer(mn - 1, r, uniform);
|
|
if (mv >= 0 && _code[mv].Bits(0, 12) == (uniform ? 0x882u : 0x802u) && _code[mv].Bits(12, 4) == 7)
|
|
{
|
|
count = (int)_code[mv].W1 + 1;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (count <= 0 || count > 1024) throw Fail("BRX jump table without a bound");
|
|
var t = new List<uint>();
|
|
for (int k = 0; k < count; k++)
|
|
{
|
|
int at = (int)tableBase + 4 * k;
|
|
if (at + 4 > C1.Length) throw Fail("BRX jump table outside c1");
|
|
uint v = BitConverter.ToUInt32(C1, at);
|
|
if (v % 16 != 0 || v / 16 > _code.Length) throw Fail($"BRX jump table entry 0x{v:x}");
|
|
if (!t.Contains(v)) t.Add(v);
|
|
}
|
|
return t;
|
|
}
|
|
|
|
// CALL.REL.NOINC (relative, the offset as BRA's) pushes no return address: it is a jump. The compiler makes a
|
|
// call of it by putting the address after the CALL in a register first (MOV Rn, imm, which the scheduler may
|
|
// hoist dozens of instructions, within the straight-line code before the CALL)
|
|
// for the callee's RET.REL.NODEC Rn; such a CALL becomes Maxwell CAL and its RET Maxwell RET (the translator
|
|
// keeps the call stack, so the link register is not written). Without that MOV it is a plain, possibly
|
|
// predicated, jump (`@P0 CALL.NOINC` over a BRA) and becomes BRA: as CAL it made functions that call each
|
|
// other, a recursive call graph Vulkan forbids and NVIDIA's compiler inlines until its stack overflows.
|
|
bool SetsReturnAddress()
|
|
{
|
|
uint ret = (uint)(_index + 1) * 16;
|
|
for (int k = _index - 1; k >= 0 && k >= _index - 64; k--)
|
|
{
|
|
var p = _code[k];
|
|
uint op = p.Bits(0, 12);
|
|
if ((op & 0x1ff) >= 0x140 && (op & 0x1ff) <= 0x15f) break; // control flow: the MOV would not reach the CALL
|
|
if (op == 0x802 && p.W1 == ret && p.Bits(12, 4) == 7) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void Call()
|
|
{
|
|
if (_i.W2 >> 18 != 0x03c00000 >> 18) throw Fail("CALL mode");
|
|
long raw = (long)(_i.W1 | ((ulong)(_i.W2 & 0x3ffff) << 32));
|
|
long target = (long)_index * 16 + 16 + ((raw << 14) >> 14);
|
|
if (target < 0 || target % 16 != 0 || target / 16 > _code.Length) throw Fail("CALL target");
|
|
if (!SetsReturnAddress())
|
|
{
|
|
_branches.Add((_out.Count, (int)(target / 16)));
|
|
Emit(0xe24000000000000fUL | ((ulong)_i.Bits(12, 4) << 16)); // BRA under the CALL's own predicate
|
|
return;
|
|
}
|
|
if (_i.Bits(12, 4) != 7)
|
|
{
|
|
// Maxwell CAL has no predicate: skip the call unless the guard holds
|
|
_branches.Add((_out.Count, _index + 1));
|
|
Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16));
|
|
}
|
|
_branches.Add((_out.Count, (int)(target / 16)));
|
|
// Predicate field 0, as Maxwell CAL encodes it (it has no predicate): the translator's Decoder reads a CAL whose
|
|
// field says PT as an unconditional branch and decodes nothing after it (Decoder.IsUnconditionalBranch).
|
|
Emit(0xe260000000000000UL);
|
|
}
|
|
|
|
void Ret()
|
|
{
|
|
if ((_i.W2 & ~0x3ffffu) != 0x03c00000) throw Fail("RET mode");
|
|
EmitG(0xe32000000000000fUL);
|
|
}
|
|
|
|
// ---- control flow ------------------------------------------------------------------------------------------
|
|
void Kill()
|
|
{
|
|
if (_i.W1 != 0 || _i.W2 != 0x03800000) throw Fail("KILL mode");
|
|
EmitG(0xe33000000000000fUL);
|
|
}
|
|
|
|
// EXIT, with an optional second condition Pc in bits 87-90: exits when the guard and Pc hold.
|
|
void Exit()
|
|
{
|
|
if (_i.Bits(84, 2) != 0) throw Fail("conditional/special EXIT");
|
|
uint pc = _i.Bits(87, 4);
|
|
if (pc == 7) { EmitG(0xe30000000000000fUL); return; }
|
|
if (_i.Bits(12, 4) != 7)
|
|
{
|
|
_branches.Add((_out.Count, _index + 1));
|
|
Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16)); // skip unless the guard holds
|
|
}
|
|
Emit(0xe30000000000000fUL | ((ulong)pc << 16));
|
|
}
|
|
|
|
// BRA: byte offset from the next instruction in bits 34-81 (sign-extended), mode in bits 32-33 (.DIV 2: taken
|
|
// only on a diverged warp, never here; .CONV 3: taken on a converged warp, always here), an extra condition
|
|
// predicate in bits 87-90.
|
|
void Bra()
|
|
{
|
|
uint mode = _i.Bits(32, 2);
|
|
if (mode <= 1 && _i.Bit(91)) throw Fail("BRA uniform condition");
|
|
if (mode == 2) return;
|
|
long raw = (long)((_i.W1 & ~3u) | ((ulong)(_i.W2 & 0x3ffff) << 32));
|
|
long rel = (raw << 14) >> 14;
|
|
long target = (long)_index * 16 + 16 + rel;
|
|
if (target < 0 || target % 16 != 0 || target / 16 > _code.Length) throw Fail("BRA target");
|
|
ulong pred = _guard;
|
|
uint extra = _i.Bits(87, 4);
|
|
if (extra != 7)
|
|
{
|
|
if (_i.Bits(12, 4) != 7)
|
|
{
|
|
// both conditions: skip the branch unless the guard holds
|
|
_branches.Add((_out.Count, _index + 1));
|
|
Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16));
|
|
}
|
|
pred = (ulong)extra << 16;
|
|
}
|
|
_branches.Add((_out.Count, (int)(target / 16)));
|
|
Emit(0xe24000000000000fUL | pred);
|
|
}
|
|
}
|
|
|
|
// code: the NVN2 program as stored (graphics: 0x80-byte header then instructions; compute: instructions only), size its
|
|
// byte length from the control section (+0x38). Returns the Maxwell program in the translator's input layout.
|
|
// mesh: a task or mesh program (graphics false): its 0x80-byte header is read for the output map and dropped, and the
|
|
// program is lowered as a compute program.
|
|
public static byte[] Lower(byte[] code, int size, bool graphics, int[] table = null, MeshInfo mesh = null) =>
|
|
Lower(code, size, graphics, out _, table, mesh);
|
|
|
|
// c1: the program's c1 constants (BRX jump tables); set before Lower by the caller.
|
|
[ThreadStatic] public static byte[] C1Bytes;
|
|
|
|
// Diagnostics only (Program.cs LowerFile, the --lower mode): the register UR0 was mapped to by the last Lowerer
|
|
// built on this thread (Lowerer.UrBase; SpillFrom in spill mode). Lowering never reads it.
|
|
[ThreadStatic] public static int LastUrBase;
|
|
|
|
// localBytes: the local memory size the lowered program needs when that changed (spill mode), else 0. Graphics
|
|
// programs carry it in their header already; compute and task/mesh programs need it from the translator's host
|
|
// (job "local_mem").
|
|
public static byte[] Lower(byte[] code, int size, bool graphics, out int localBytes, int[] table = null, MeshInfo mesh = null)
|
|
{
|
|
localBytes = 0;
|
|
int header = graphics || mesh != null ? 0x80 : 0;
|
|
if (size <= header || size > code.Length || (size - header) % 16 != 0)
|
|
throw new LoweringException($"sm86 program size 0x{size:x} (file 0x{code.Length:x})");
|
|
var insns = new Insn[(size - header) / 16];
|
|
for (int k = 0; k < insns.Length; k++)
|
|
insns[k] = new Insn { Lo = BitConverter.ToUInt64(code, header + k * 16), Hi = BitConverter.ToUInt64(code, header + k * 16 + 8) };
|
|
byte[] sph = null;
|
|
if (graphics && ((BitConverter.ToUInt32(code, 0) >> 10) & 15) == 5) { sph = new byte[0x50]; Buffer.BlockCopy(code, 0, sph, 0, 0x50); }
|
|
if (mesh != null) { var h = new byte[0x80]; Buffer.BlockCopy(code, 0, h, 0, 0x80); mesh.Layout(h); }
|
|
List<ulong> lowered;
|
|
List<(int output, int targetSource)> branches;
|
|
int[] first;
|
|
int spillAt = -1;
|
|
try { lowered = new Lowerer(insns, sph, !graphics, table, mesh) { C1 = C1Bytes }.Run(out branches, out first); }
|
|
catch (RegisterPressureException)
|
|
{
|
|
// Local memory after the program's own: the header's sizes (translator: low + high + CRS / 32), or the
|
|
// translator's compute default (0x1000) for compute programs.
|
|
spillAt = 0x1000;
|
|
if (header != 0)
|
|
spillAt = ((int)(BitConverter.ToUInt32(code, 4) & 0xffffff) + (int)(BitConverter.ToUInt32(code, 8) & 0xffffff)
|
|
+ (int)(BitConverter.ToUInt32(code, 12) & 0xffffff) / 32 + 15) & ~15;
|
|
try
|
|
{
|
|
lowered = new Lowerer(insns, sph, !graphics, table, mesh, spillAt, Spill) { C1 = C1Bytes }.Run(out branches, out first);
|
|
localBytes = spillAt + Spill.SpillBytes;
|
|
}
|
|
catch (UrPoolException)
|
|
{
|
|
lowered = new Lowerer(insns, sph, !graphics, table, mesh, spillAt, SpillWide) { C1 = C1Bytes }.Run(out branches, out first);
|
|
localBytes = spillAt + SpillWide.SpillBytes;
|
|
}
|
|
}
|
|
|
|
int start = graphics ? 0x50 : 0;
|
|
var address = new int[lowered.Count + 1];
|
|
int pos = start;
|
|
for (int k = 0; k <= lowered.Count; k++)
|
|
{
|
|
if (((pos - start) & 0x1f) == 0) pos += 8;
|
|
address[k] = pos;
|
|
pos += 8;
|
|
}
|
|
foreach (var (output, targetSource) in branches)
|
|
{
|
|
long rel = address[first[targetSource]] - (address[output] + 8);
|
|
if (rel < -0x800000 || rel > 0x7fffff) throw new LoweringException("sm86 lowered branch distance");
|
|
lowered[output] |= ((ulong)rel & 0xffffff) << 20;
|
|
}
|
|
var result = new byte[pos + 16];
|
|
if (graphics) Buffer.BlockCopy(code, 0, result, 0, 0x50);
|
|
if (graphics && spillAt >= 0) // the header's low local memory size covers the spill area too
|
|
BitConverter.GetBytes((BitConverter.ToUInt32(result, 4) & 0xff000000u) | (uint)localBytes).CopyTo(result, 4);
|
|
for (int k = 0; k < lowered.Count; k++) BitConverter.GetBytes(lowered[k]).CopyTo(result, address[k]);
|
|
// a closing EXIT after the last instruction, so a branch to the end lands somewhere
|
|
BitConverter.GetBytes(0xe30000000007000fUL).CopyTo(result, address[lowered.Count]);
|
|
return result;
|
|
}
|
|
}
|