Emulate mesh shaders with compute on GPUs without VK_EXT_mesh_shader (NX2NAVI_MESH_EMULATION, off by default).

Without the extension every mesh draw was skipped ("mesh draws need VK_EXT_mesh_shader: skipped"): on an RX 580 the
game's grass and foliage were missing. The translator already builds each task and mesh program as compute code that
MeshSpirv.cs wraps into TaskEXT / MeshEXT modules; runtime/mesh_emu_spirv.c turns those modules into GLCompute ones:
outputs and payload move to workgroup memory, SetMeshOutputsEXT's counts are kept, and the entry function's end
reserves vertex slots and indices in an arena with atomics and copies the vertices and triangles out (task: the payload
and the mesh group count go to a task record, and the mesh dispatch to the draw record). A generated vertex shader
reads the vertices back for the game's fragment shader. Checked with spirv-val (Vulkan 1.3) on the 1016 task and mesh
programs of Ocarina of Time 1.1.1: 1008 convert and validate; the 8 point-list meshes stay unemulated.

vk_pipeline.c gives an emulated mesh pipeline a layout whose task and mesh bindings are visible to compute, with push
constants and a compute-side descriptor template, builds the generated vertex shader's graphics pipeline and the two
compute pipelines; vk_exec_draw.c prepares each mesh draw as before, then outside the pass zeroes the arena counters
(first emulated draw of a submission), fills a draw record, pushes the same descriptors to the compute bind point,
dispatches the task and mesh stages (direct, or from the indirect ring entry) and draws DrawIndexedIndirect from the
record. mesh_emulation = 1 turns it on; mesh_emu_mb sizes the arena (160 MiB).

(cherry picked from commit efc43b177694d8f8ce5700fcf28d80e889a45589)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tobim6
2026-11-12 16:22:03 +01:00
parent 8913ce1736
commit aec5b5a840
6 changed files with 1105 additions and 16 deletions

View File

@@ -421,6 +421,9 @@
# is read and written by the GPU in system memory instead (0: every pool keeps its video memory copy). For GPUs and
# buses short of bandwidth; on an RX 580 the pools the game refills every frame cost about 8 ms a frame in copies.
# mirror_hot = 0
# GPUs without mesh shaders (VK_EXT_mesh_shader: AMD before RDNA 2, NVIDIA before Turing): 1 runs the game's mesh
# and task shaders (grass and other foliage) as compute shaders and draws what they make; 0 skips those draws. The
# mesh_emulation = 0
# Scaling to the window when it is larger than the game resolution: fsr (AMD FSR 1) or bilinear, and FSR's sharpening
# from 0 (off) to 100. Changes while the game runs take effect at once.
# upscaler = fsr

798
runtime/mesh_emu_spirv.c Normal file
View File

@@ -0,0 +1,798 @@
// Mesh shader emulation: SPIR-V rewriting (mesh_emu_spirv.h). The input is what tools/xlate/MeshSpirv.cs makes: the
// translator's compute main, called from an entry function that copies its shared-memory stage buffer into the mesh
// outputs after SetMeshOutputsEXT (mesh), or into the task payload before EmitMeshTasksEXT (task).
//
// Mesh -> GLCompute: the output arrays and the task payload move to Workgroup memory; SetMeshOutputsEXT is dropped (its
// counts are kept); at the entry function's end thread 0 reserves vertex slots and indices in the arena with atomics,
// and every thread copies vertices and triangles out (indices made absolute vertex slots, relative to the vertex area).
// With a task stage mesh workgroup (x, y) is mesh group x of task record (task base + y): a prologue returns when x is
// past the record's count, loads the payload, and the module's WorkgroupId reads become (x, 0, 0).
// Task -> GLCompute: the payload moves to Workgroup memory; EmitMeshTasksEXT becomes a store of the payload and the
// mesh group count into the task record, and workgroup 0 writes the mesh dispatch {max_mesh, task workgroups, 1}
// into the draw record.
#include "mesh_emu_spirv.h"
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
enum {
OpSourceContinued = 2, OpSource = 3, OpSourceExtension = 4, OpName = 5, OpMemberName = 6, OpString = 7, OpLine = 8,
OpExtension = 10, OpExtInstImport = 11, OpExtInst = 12, OpMemoryModel = 14, OpEntryPoint = 15, OpExecutionMode = 16,
OpCapability = 17, OpTypeVoid = 19, OpTypeBool = 20, OpTypeInt = 21, OpTypeFloat = 22, OpTypeVector = 23,
OpTypeArray = 28, OpTypeRuntimeArray = 29, OpTypeStruct = 30, OpTypePointer = 32, OpTypeFunction = 33,
OpConstant = 43, OpConstantComposite = 44, OpFunction = 54, OpFunctionEnd = 56, OpVariable = 59, OpLoad = 61,
OpStore = 62, OpAccessChain = 65, OpInBoundsAccessChain = 66, OpDecorate = 71, OpMemberDecorate = 72,
OpCompositeConstruct = 80, OpCompositeExtract = 81, OpUConvert = 113, OpConvertUToPtr = 120, OpBitcast = 124,
OpIAdd = 128, OpIMul = 132, OpLogicalAnd = 167, OpSelect = 169, OpIEqual = 170, OpINotEqual = 171,
OpULessThan = 176, OpULessThanEqual = 178, OpControlBarrier = 224, OpAtomicIAdd = 234, OpAtomicUMax = 239,
OpSelectionMerge = 247, OpLabel = 248, OpBranch = 249, OpBranchConditional = 250, OpReturn = 253,
OpNoLine = 317, OpModuleProcessed = 330, OpExecutionModeId = 331, OpConstantNull = 46, OpCopyObject = 83,
OpEmitMeshTasksEXT = 5294, OpSetMeshOutputsEXT = 5295,
};
enum { ScInput = 1, ScOutput = 3, ScWorkgroup = 4, ScPrivate = 6, ScFunction = 7, ScPushConstant = 9,
ScPhysicalStorageBuffer = 5349, ScTaskPayload = 5402 };
enum { DecBlock = 2, DecArrayStride = 6, DecBuiltIn = 11, DecLocation = 30, DecOffset = 35, DecPerPrimitive = 5271 };
BiPrimTri = 5296 };
enum { CapShader = 1, CapInt64 = 11, CapMeshShading = 5283, CapPSBAddresses = 5347 };
enum { ModelVertex = 0, ModelGLCompute = 5, ModelTask = 5364, ModelMesh = 5365 };
enum { ModeLocalSize = 17, ModeOutputVertices = 26, ModeOutputPoints = 27, ModeOutputLines = 5269,
ModeOutputPrimitives = 5270, ModeOutputTriangles = 5298 };
enum { GlslUMin = 38 };
enum { MaxIns = 4096 };
typedef struct { uint32_t *w; size_t n, cap; int bad; } Buf;
static void put(Buf *b, uint32_t x) {
if (b->n == b->cap) {
size_t c = b->cap ? b->cap * 2 : 1024;
uint32_t *w = realloc(b->w, c * sizeof *w);
if (!w) { b->bad = 1; return; }
b->w = w, b->cap = c;
}
b->w[b->n++] = x;
}
static void putn(Buf *b, const uint32_t *w, size_t n) { for (size_t i = 0; i < n; i++) put(b, w[i]); }
static void op(Buf *b, uint32_t opc, int n, ...) {
va_list ap;
va_start(ap, n);
put(b, (uint32_t)(n + 1) << 16 | opc);
for (int i = 0; i < n; i++) put(b, va_arg(ap, uint32_t));
va_end(ap);
}
static void op_str(Buf *b, uint32_t opc, const uint32_t *pre, int npre, const char *s, const uint32_t *post, int npost) {
size_t sl = strlen(s) + 1, sw = (sl + 3) / 4;
put(b, (uint32_t)(1 + npre + sw + npost) << 16 | opc);
for (int i = 0; i < npre; i++) put(b, pre[i]);
for (size_t i = 0; i < sw; i++) {
uint32_t x = 0;
for (size_t k = 0; k < 4 && i * 4 + k < sl; k++) x |= (uint32_t)(uint8_t)s[i * 4 + k] << (8 * k);
put(b, x);
}
for (int i = 0; i < npost; i++) put(b, post[i]);
}
// ---- the module
typedef struct {
const uint32_t *w;
size_t n;
uint32_t bound;
uint32_t t_void, t_bool, t_u32, t_u64, t_f32, t_v3u, t_v4u, t_v4f;
uint32_t glsl, ep_fn, model, wgid, lii, nwg;
int has_int64, has_psb, has_psb_ext;
struct { uint32_t id, sc, to; } ptr[4096];
int nptr;
struct { uint32_t id, type, sc; } var[2048];
int nvar;
struct { uint32_t id, type, val; } cst[8192];
int ncst;
} Mod;
static uint32_t find_ptr(const Mod *m, uint32_t sc, uint32_t to) {
for (int i = 0; i < m->nptr; i++)
if (m->ptr[i].sc == sc && m->ptr[i].to == to) return m->ptr[i].id;
return 0;
}
static uint32_t ptr_to(const Mod *m, uint32_t id) {
for (int i = 0; i < m->nptr; i++)
if (m->ptr[i].id == id) return m->ptr[i].to;
return 0;
}
static const uint32_t *find_type(const Mod *m, uint32_t id) {
for (size_t i = 5; i < m->n;) {
uint32_t o = m->w[i] & 0xffff, len = m->w[i] >> 16;
if (!len || o == OpFunction) break;
if ((o == OpTypeArray || o == OpTypeRuntimeArray || o == OpTypeVector || o == OpTypeStruct) && m->w[i + 1] == id)
return m->w + i;
i += len;
}
return NULL;
}
static int const_u32(const Mod *m, uint32_t id, uint32_t *v) {
for (int i = 0; i < m->ncst; i++)
if (m->cst[i].id == id && m->cst[i].type == m->t_u32) return *v = m->cst[i].val, 1;
return 0;
}
static int is_debug(uint32_t o) {
return o == OpSourceContinued || o == OpSource || o == OpSourceExtension || o == OpName || o == OpMemberName ||
o == OpString || o == OpModuleProcessed;
}
static int is_annot(uint32_t o) { return o == OpDecorate || o == OpMemberDecorate || o == 73 || o == 74 || o == 75 || o == 332 || o == 5632 || o == 5633; }
static int parse(Mod *m, const uint32_t *w, size_t n, const char **why) {
m->w = w, m->n = n;
if (n < 5 || w[0] != 0x07230203) return *why = "not SPIR-V", 0;
m->bound = w[3];
for (size_t i = 5; i < n;) {
uint32_t o = w[i] & 0xffff, len = w[i] >> 16;
if (!len || i + len > n) return *why = "malformed", 0;
if (len > MaxIns) return *why = "an instruction too long", 0;
const uint32_t *a = w + i + 1;
if (o == OpFunction) break;
switch (o) {
case OpCapability:
if (a[0] == CapInt64) m->has_int64 = 1;
if (a[0] == CapPSBAddresses) m->has_psb = 1;
break;
case OpExtension: if (!strcmp((const char *)a, "SPV_KHR_physical_storage_buffer")) m->has_psb_ext = 1; break;
case OpExtInstImport: if (!strcmp((const char *)(a + 1), "GLSL.std.450")) m->glsl = a[0]; break;
case OpEntryPoint:
if (m->ep_fn) return *why = "several entry points", 0;
m->model = a[0], m->ep_fn = a[1];
break;
case OpTypeVoid: if (!m->t_void) m->t_void = a[0]; break;
case OpTypeBool: if (!m->t_bool) m->t_bool = a[0]; break;
case OpTypeInt:
if (a[1] == 32 && !a[2] && !m->t_u32) m->t_u32 = a[0];
if (a[1] == 64 && !a[2] && !m->t_u64) m->t_u64 = a[0];
break;
case OpTypeFloat: if (a[1] == 32 && !m->t_f32) m->t_f32 = a[0]; break;
case OpTypeVector:
if (a[1] == m->t_u32 && a[2] == 3 && !m->t_v3u) m->t_v3u = a[0];
if (a[1] == m->t_u32 && a[2] == 4 && !m->t_v4u) m->t_v4u = a[0];
if (a[1] == m->t_f32 && m->t_f32 && a[2] == 4 && !m->t_v4f) m->t_v4f = a[0];
break;
case OpTypePointer:
if (m->nptr == 4096) return *why = "too many pointer types", 0;
m->ptr[m->nptr].id = a[0], m->ptr[m->nptr].sc = a[1], m->ptr[m->nptr++].to = a[2];
break;
case OpVariable:
if (m->nvar == 2048) return *why = "too many variables", 0;
m->var[m->nvar].id = a[1], m->var[m->nvar].type = a[0], m->var[m->nvar++].sc = a[2];
break;
case OpConstant:
if (len == 4 && m->ncst < 8192) m->cst[m->ncst].id = a[1], m->cst[m->ncst].type = a[0], m->cst[m->ncst++].val = a[2];
break;
case OpDecorate:
if (len >= 4 && a[1] == DecBuiltIn) {
if (a[2] == BiWorkgroupId) m->wgid = a[0];
if (a[2] == BiLocalInvocationIndex) m->lii = a[0];
if (a[2] == BiNumWorkgroups) m->nwg = a[0];
}
break;
}
i += len;
}
if (!m->ep_fn || (m->model != ModelMesh && m->model != ModelTask)) return *why = "not a task or mesh module", 0;
if (!m->t_u32 || !m->t_void || !m->glsl || !m->lii || !m->wgid) return *why = "missing basic types or built-ins", 0;
return 1;
}
// ---- emission over new ids
typedef struct {
Mod *m;
Buf *types, *annot, *code;
uint32_t p_u32_psb, p_v4u_psb, p_words, p_slots, p_pc_u64, p_pc_u32, pc_var, c64_16;
struct { uint32_t id, val; } cc[256];
int ncc;
} Em;
static uint32_t nid(Mod *m) { return m->bound++; }
static uint32_t cu(Em *e, uint32_t v) {
for (int i = 0; i < e->m->ncst; i++)
if (e->m->cst[i].type == e->m->t_u32 && e->m->cst[i].val == v) return e->m->cst[i].id;
for (int i = 0; i < e->ncc; i++)
if (e->cc[i].val == v) return e->cc[i].id;
uint32_t id = nid(e->m);
op(e->types, OpConstant, 3, e->m->t_u32, id, v);
if (e->ncc < 256) e->cc[e->ncc].id = id, e->cc[e->ncc++].val = v;
return id;
}
static uint32_t E1(Em *e, uint32_t o, uint32_t t, uint32_t a) { uint32_t r = nid(e->m); op(e->code, o, 3, t, r, a); return r; }
static uint32_t E2(Em *e, uint32_t o, uint32_t t, uint32_t a, uint32_t b) { uint32_t r = nid(e->m); op(e->code, o, 4, t, r, a, b); return r; }
static uint32_t E3(Em *e, uint32_t o, uint32_t t, uint32_t a, uint32_t b, uint32_t c) { uint32_t r = nid(e->m); op(e->code, o, 5, t, r, a, b, c); return r; }
static uint32_t add(Em *e, uint32_t a, uint32_t b) { return E2(e, OpIAdd, e->m->t_u32, a, b); }
static uint32_t mul(Em *e, uint32_t a, uint32_t b) { return E2(e, OpIMul, e->m->t_u32, a, b); }
static uint32_t ult(Em *e, uint32_t a, uint32_t b) { return E2(e, OpULessThan, e->m->t_bool, a, b); }
static uint32_t ld(Em *e, uint32_t t, uint32_t p) { return E1(e, OpLoad, t, p); }
static uint32_t pc_u64(Em *e) { return ld(e, e->m->t_u64, E2(e, OpAccessChain, e->p_pc_u64, e->pc_var, cu(e, 0))); }
static uint32_t pc_u32(Em *e, uint32_t k) { return ld(e, e->m->t_u32, E2(e, OpAccessChain, e->p_pc_u32, e->pc_var, cu(e, k))); }
static uint32_t words_base(Em *e) { return E1(e, OpConvertUToPtr, e->p_words, pc_u64(e)); }
static uint32_t word_ptr(Em *e, uint32_t base, uint32_t idx) { return E3(e, OpAccessChain, e->p_u32_psb, base, cu(e, 0), idx); }
static uint32_t word_load(Em *e, uint32_t base, uint32_t idx) {
uint32_t p = word_ptr(e, base, idx), r = nid(e->m);
op(e->code, OpLoad, 5, e->m->t_u32, r, p, 2u, 4u); // Aligned 4
return r;
}
static void word_store(Em *e, uint32_t base, uint32_t idx, uint32_t v) { op(e->code, OpStore, 4, word_ptr(e, base, idx), v, 2u, 4u); }
static uint32_t atomic(Em *e, uint32_t o, uint32_t base, uint32_t idx, uint32_t v) {
uint32_t p = word_ptr(e, base, idx), r = nid(e->m);
op(e->code, o, 6, e->m->t_u32, r, p, cu(e, 1), cu(e, 0), v); // Device scope, relaxed
return r;
}
static void barrier(Em *e) { op(e->code, OpControlBarrier, 3, cu(e, 2), cu(e, 2), cu(e, 264)); } // Workgroup, AcqRel|WorkgroupMemory
static void if_begin(Em *e, uint32_t cond, uint32_t *merge) {
uint32_t t = nid(e->m);
*merge = nid(e->m);
op(e->code, OpSelectionMerge, 2, *merge, 0u);
op(e->code, OpBranchConditional, 3, cond, t, *merge);
op(e->code, OpLabel, 1, t);
}
static void if_end(Em *e, uint32_t merge) {
op(e->code, OpBranch, 1, merge);
op(e->code, OpLabel, 1, merge);
}
// The mesh epilogue (replaces the entry function's return)
static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int nov, uint32_t vcount, uint32_t pcount,
Mod *m = e->m;
barrier(e);
uint32_t lid = ld(e, m->t_u32, m->lii), base = words_base(e);
uint32_t mz;
if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz);
{
uint32_t vb = atomic(e, OpAtomicIAdd, base, cu(e, 0), vsz);
uint32_t iok = E2(e, OpULessThanEqual, m->t_bool, add(e, ib, isz), icap);
uint32_t vok = E2(e, OpULessThanEqual, m->t_bool, add(e, vb, vsz), vcap);
uint32_t mi;
if_begin(e, iok, &mi);
atomic(e, OpAtomicIAdd, base, drw, isz); // the draw record's index count
if_end(e, mi);
uint32_t all = E2(e, OpLogicalAnd, m->t_bool, iok, vok);
atomic(e, OpAtomicIAdd, base, cu(e, 3), E3(e, OpSelect, m->t_u32, all, cu(e, 0), cu(e, 1)));
op(e->code, OpStore, 2, sh_vb, E3(e, OpSelect, m->t_u32, vok, vb, cu(e, 0xffffffffu)));
op(e->code, OpStore, 2, sh_ib, E3(e, OpSelect, m->t_u32, iok, ib, cu(e, 0xffffffffu)));
}
if_end(e, mz);
barrier(e);
uint32_t VB = ld(e, m->t_u32, sh_vb), IB = ld(e, m->t_u32, sh_ib);
uint32_t m3;
if_begin(e, E2(e, OpINotEqual, m->t_bool, IB, cu(e, 0xffffffffu)), &m3);
{
uint32_t vbok = E2(e, OpINotEqual, m->t_bool, VB, cu(e, 0xffffffffu));
uint32_t m4;
if_begin(e, vbok, &m4);
{
uint32_t vw = pc_u32(e, 4);
uint32_t a64 = E2(e, OpIAdd, m->t_u64, pc_u64(e), E2(e, OpIMul, m->t_u64, E1(e, OpUConvert, m->t_u64, vw), e->c64_16));
uint32_t sbase = E1(e, OpConvertUToPtr, e->p_slots, a64);
for (uint32_t r = 0; r * T < info->max_v; r++) {
uint32_t v = add(e, lid, cu(e, r * T)), mv;
if_begin(e, ult(e, v, vcount), &mv);
uint32_t s0 = add(e, VB, mul(e, v, cu(e, S)));
for (int k = 0; k < nov; k++) {
if (ov[k].role == 2) continue;
uint32_t dst = E3(e, OpAccessChain, e->p_v4u_psb, sbase, cu(e, 0), add(e, s0, cu(e, ov[k].slot)));
op(e->code, OpStore, 4, dst, bits, 2u, 16u);
}
if_end(e, mv);
}
}
if_end(e, m4);
uint32_t iw = pc_u32(e, 2), idxv = 0;
for (int k = 0; k < nov; k++)
if (ov[k].role == 2) idxv = ov[k].var;
for (uint32_t r = 0; r * T < info->max_p; r++) {
uint32_t p = add(e, lid, cu(e, r * T)), mp;
if_begin(e, ult(e, p, pcount), &mp);
uint32_t val = E3(e, OpSelect, m->t_u32, vbok, add(e, VB, mul(e, li, cu(e, S))), cu(e, 0));
word_store(e, base, add(e, at, cu(e, c)), val);
}
if_end(e, mp);
}
}
if_end(e, m3);
op(e->code, OpReturn, 0);
}
// The task epilogue (replaces EmitMeshTasksEXT x y z)
static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint32_t p_wg_u32, uint32_t nwg,
uint32_t ex, uint32_t ey, uint32_t ez) {
Mod *m = e->m;
uint32_t T = info->threads;
uint32_t cnt = mul(e, ex, mul(e, ey, ez));
uint32_t nc = E3(e, OpSelect, m->t_u32, ult(e, cnt, cu(e, info->max_mesh)), cnt, cu(e, info->max_mesh));
barrier(e);
uint32_t lid = ld(e, m->t_u32, m->lii), wg = ld(e, m->t_v3u, m->wgid), nw = ld(e, m->t_v3u, nwg);
uint32_t wx = E2(e, OpCompositeExtract, m->t_u32, wg, 0u), wy = E2(e, OpCompositeExtract, m->t_u32, wg, 1u),
wz = E2(e, OpCompositeExtract, m->t_u32, wg, 2u);
uint32_t nx = E2(e, OpCompositeExtract, m->t_u32, nw, 0u), ny = E2(e, OpCompositeExtract, m->t_u32, nw, 1u),
nz = E2(e, OpCompositeExtract, m->t_u32, nw, 2u);
uint32_t nxy = mul(e, nx, ny);
uint32_t t = add(e, wx, add(e, mul(e, wy, nx), mul(e, wz, nxy)));
uint32_t base = words_base(e), drw = pc_u32(e, 1);
uint32_t tb = word_load(e, base, add(e, drw, cu(e, 8)));
{
uint32_t recw = add(e, cu(e, MESH_EMU_TOFF / 4), mul(e, slot, cu(e, MESH_EMU_TWORDS)));
uint32_t v = ld(e, m->t_u32, E2(e, OpAccessChain, p_wg_u32, payload, k));
word_store(e, base, add(e, add(e, recw, cu(e, 1)), k), v);
}
uint32_t m2;
if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &m2);
word_store(e, base, recw, nc);
if_end(e, m2);
}
if_end(e, m1);
uint32_t first = E2(e, OpLogicalAnd, m->t_bool, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)),
E2(e, OpIEqual, m->t_bool, t, cu(e, 0)));
uint32_t m3;
if_begin(e, first, &m3);
{
uint32_t total = mul(e, nxy, nz);
// task records past the area are not written: their mesh groups are not dispatched
uint32_t ty = E3(e, OpSelect, m->t_u32, ult(e, total, room), total, room);
word_store(e, base, add(e, drw, cu(e, 5)), cu(e, info->max_mesh));
word_store(e, base, add(e, drw, cu(e, 6)), ty);
word_store(e, base, add(e, drw, cu(e, 7)), cu(e, 1));
}
if_end(e, m3);
op(e->code, OpReturn, 0);
}
uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_t *out_n, const char **why) {
static const char *dummy;
if (!why) why = &dummy;
*why = NULL;
Mod *m = calloc(1, sizeof *m);
uint32_t *ins = malloc(MaxIns * 4), *ret = NULL;
Buf caps = {0}, exts = {0}, imps = {0}, modes = {0}, debug = {0}, annot = {0}, types = {0}, ntypes = {0}, nannot = {0},
code = {0}, out = {0};
if (!m || !ins) { *why = "out of memory"; goto done; }
if (!parse(m, w, n, why)) goto done;
memset(info, 0, sizeof *info);
int mesh = m->model == ModelMesh;
info->task = !mesh;
for (size_t i = 5; i < n;) {
uint32_t o = w[i] & 0xffff, len = w[i] >> 16;
const uint32_t *a = w + i + 1;
if (o == OpFunction) break;
if (o == OpExecutionMode && a[0] == m->ep_fn) {
if (a[1] == ModeLocalSize) info->threads = a[2] * a[3] * a[4];
if (a[1] == ModeOutputVertices) info->max_v = a[2];
if (a[1] == ModeOutputPrimitives) info->max_p = a[2];
if (a[1] == ModeOutputTriangles) info->prim = 3;
if (a[1] == ModeOutputLines) info->prim = 2;
if (a[1] == ModeOutputPoints) info->prim = 1;
}
i += len;
}
if (!info->threads) { *why = "no LocalSize"; goto done; }
if (!m->t_f32 || (mesh && !m->t_v4f)) { *why = "no float types"; goto done; }
// output variables and the payload
OutVar ov[40];
int nov = 0;
uint32_t payload = 0, payload_len = 0;
for (int i = 0; i < m->nvar; i++) {
uint32_t sc = m->var[i].sc;
if (sc != ScOutput && sc != ScTaskPayload) continue;
const uint32_t *arr = find_type(m, ptr_to(m, m->var[i].type));
uint32_t len = 0;
if (!arr || (arr[0] & 0xffff) != OpTypeArray || !const_u32(m, arr[3], &len)) { *why = "an output or payload is not a sized array"; goto done; }
if (sc == ScTaskPayload) {
if (arr[2] != m->t_u32) { *why = "a payload that is not uint[]"; goto done; }
payload = m->var[i].id, payload_len = len;
continue;
}
if (nov == 40) { *why = "too many outputs"; goto done; }
ov[nov] = (OutVar){m->var[i].id, len, 9, 0, 0};
for (size_t k = 5; k < n;) {
uint32_t o = w[k] & 0xffff, l2 = w[k] >> 16;
const uint32_t *a = w + k + 1;
if (o == OpFunction) break;
if (o == OpDecorate && a[0] == m->var[i].id) {
if (a[1] == DecBuiltIn && a[2] == BiPosition) ov[nov].role = 0;
else if (a[1] == DecLocation) ov[nov].role = 1, ov[nov].loc = a[2];
else if (a[1] == DecBuiltIn) { *why = "an unsupported mesh output built-in"; goto done; }
else if (a[1] == DecPerPrimitive) { *why = "per-primitive outputs are not emulated"; goto done; }
}
k += l2;
}
if (ov[nov].role == 9) { *why = "an output without a role"; goto done; }
nov++;
}
info->payload_words = payload_len;
if (payload_len > MESH_EMU_TWORDS - 1) { *why = "payload too large"; goto done; }
if (mesh) {
uint32_t locs = 0;
int have_pos = 0, have_idx = 0;
for (int i = 0; i < nov; i++) {
if (ov[i].role == 1) {
if (ov[i].loc >= 32 || locs >> ov[i].loc & 1) { *why = "bad generic location"; goto done; }
locs |= 1u << ov[i].loc;
}
have_pos |= ov[i].role == 0, have_idx |= ov[i].role == 2;
}
if (!have_pos || !have_idx) { *why = "no position or no indices"; goto done; }
for (uint32_t l = 0; l < 32; l++)
if (locs >> l & 1) info->generic_loc[info->n_generic++] = (uint8_t)l;
info->slots = 1 + info->n_generic;
for (int i = 0; i < nov; i++) {
if (ov[i].role == 0) ov[i].slot = 0;
if (ov[i].role == 1)
for (uint32_t g = 0; g < info->n_generic; g++)
if (info->generic_loc[g] == ov[i].loc) ov[i].slot = 1 + g;
}
}
uint32_t emit_x = 0, emit_y = 0, emit_z = 0;
if (!mesh) {
size_t i = 5;
while (i < n && (w[i] & 0xffff) != OpFunction) i += w[i] >> 16;
for (; i < n; i += w[i] >> 16)
if ((w[i] & 0xffff) == OpEmitMeshTasksEXT) emit_x = w[i + 1], emit_y = w[i + 2], emit_z = w[i + 3];
if (!emit_x || !payload) { *why = "no EmitMeshTasksEXT or no payload"; goto done; }
info->max_mesh = 32;
i = 5;
while (i < n && (w[i] & 0xffff) != OpFunction) i += w[i] >> 16;
for (; i < n; i += w[i] >> 16) {
uint32_t v;
if ((w[i] & 0xffff) == OpExtInst && (w[i] >> 16) == 7 && w[i + 2] == emit_x && w[i + 3] == m->glsl &&
w[i + 4] == GlslUMin && const_u32(m, w[i + 6], &v) && v && v <= 1024)
info->max_mesh = v;
}
}
// ---- new types, constants and variables
Em e = {.m = m, .types = &ntypes, .annot = &nannot, .code = &code};
if (!m->t_bool) m->t_bool = nid(m), op(&ntypes, OpTypeBool, 1, m->t_bool);
if (!m->t_u64) m->t_u64 = nid(m), op(&ntypes, OpTypeInt, 3, m->t_u64, 64u, 0u);
if (!m->t_v3u) m->t_v3u = nid(m), op(&ntypes, OpTypeVector, 3, m->t_v3u, m->t_u32, 3u);
if (!m->t_v4u) m->t_v4u = nid(m), op(&ntypes, OpTypeVector, 3, m->t_v4u, m->t_u32, 4u);
uint32_t rt_u = nid(m), s_words = nid(m), rt_v4 = nid(m), s_slots = nid(m), pc_struct = nid(m);
op(&ntypes, OpTypeRuntimeArray, 2, rt_u, m->t_u32);
op(&ntypes, OpTypeStruct, 2, s_words, rt_u);
op(&ntypes, OpTypeRuntimeArray, 2, rt_v4, m->t_v4u);
op(&ntypes, OpTypeStruct, 2, s_slots, rt_v4);
op(&ntypes, OpTypeStruct, 8, pc_struct, m->t_u64, m->t_u32, m->t_u32, m->t_u32, m->t_u32, m->t_u32, m->t_u32);
e.p_words = nid(m), op(&ntypes, OpTypePointer, 3, e.p_words, (uint32_t)ScPhysicalStorageBuffer, s_words);
e.p_slots = nid(m), op(&ntypes, OpTypePointer, 3, e.p_slots, (uint32_t)ScPhysicalStorageBuffer, s_slots);
#define NEED_PTR(DST, SC, TO) do { DST = find_ptr(m, SC, TO); if (!DST) { DST = nid(m); op(&ntypes, OpTypePointer, 3, DST, (uint32_t)(SC), TO); \
m->ptr[m->nptr].id = DST, m->ptr[m->nptr].sc = SC, m->ptr[m->nptr++].to = TO; } } while (0)
NEED_PTR(e.p_u32_psb, ScPhysicalStorageBuffer, m->t_u32);
NEED_PTR(e.p_v4u_psb, ScPhysicalStorageBuffer, m->t_v4u);
uint32_t p_pc = nid(m);
op(&ntypes, OpTypePointer, 3, p_pc, (uint32_t)ScPushConstant, pc_struct);
NEED_PTR(e.p_pc_u64, ScPushConstant, m->t_u64);
NEED_PTR(e.p_pc_u32, ScPushConstant, m->t_u32);
e.pc_var = nid(m), op(&ntypes, OpVariable, 3, p_pc, e.pc_var, (uint32_t)ScPushConstant);
e.c64_16 = nid(m), op(&ntypes, OpConstant, 4, m->t_u64, e.c64_16, 16u, 0u);
op(&nannot, OpDecorate, 3, rt_u, (uint32_t)DecArrayStride, 4u);
op(&nannot, OpDecorate, 2, s_words, (uint32_t)DecBlock);
op(&nannot, OpMemberDecorate, 4, s_words, 0u, (uint32_t)DecOffset, 0u);
op(&nannot, OpDecorate, 3, rt_v4, (uint32_t)DecArrayStride, 16u);
op(&nannot, OpDecorate, 2, s_slots, (uint32_t)DecBlock);
op(&nannot, OpMemberDecorate, 4, s_slots, 0u, (uint32_t)DecOffset, 0u);
op(&nannot, OpDecorate, 2, pc_struct, (uint32_t)DecBlock);
for (uint32_t k = 0; k < 7; k++) op(&nannot, OpMemberDecorate, 4, pc_struct, k, (uint32_t)DecOffset, k ? 4 + 4 * k : 0u);
// Output and payload pointers become Workgroup pointers: in place when no such pointer exists yet, else mapped
struct { uint32_t from, to; } pmap[512];
int npmap = 0;
for (int i = 0; i < m->nptr; i++) {
if (m->ptr[i].sc != ScOutput && m->ptr[i].sc != ScTaskPayload) continue;
uint32_t have = find_ptr(m, ScWorkgroup, m->ptr[i].to);
if (have) {
if (npmap == 512) { *why = "too many pointer types"; goto done; }
pmap[npmap].from = m->ptr[i].id, pmap[npmap++].to = have;
m->ptr[i].sc = 0xffffffffu; // dropped
} else {
m->ptr[i].sc = ScWorkgroup; // rewritten in place below
}
}
uint32_t p_wg_u32, p_wg_v4f = 0, p_wg_v3u = 0;
NEED_PTR(p_wg_u32, ScWorkgroup, m->t_u32);
uint32_t sh_vb = 0, sh_ib = 0, wgemu = 0, p_priv_v3 = 0, p_priv_u32 = 0, p_in_u32 = find_ptr(m, ScInput, m->t_u32);
if (mesh) {
sh_vb = nid(m), op(&ntypes, OpVariable, 3, p_wg_u32, sh_vb, (uint32_t)ScWorkgroup);
sh_ib = nid(m), op(&ntypes, OpVariable, 3, p_wg_u32, sh_ib, (uint32_t)ScWorkgroup);
}
int remap_wg = mesh && payload;
if (remap_wg) {
NEED_PTR(p_priv_v3, ScPrivate, m->t_v3u);
NEED_PTR(p_priv_u32, ScPrivate, m->t_u32);
wgemu = nid(m), op(&ntypes, OpVariable, 3, p_priv_v3, wgemu, (uint32_t)ScPrivate);
}
uint32_t nwg = m->nwg;
if (!mesh && !nwg) {
uint32_t p_in_v3;
NEED_PTR(p_in_v3, ScInput, m->t_v3u);
nwg = nid(m), op(&ntypes, OpVariable, 3, p_in_v3, nwg, (uint32_t)ScInput);
op(&nannot, OpDecorate, 3, nwg, (uint32_t)DecBuiltIn, (uint32_t)BiNumWorkgroups);
}
#undef NEED_PTR
// ---- one pass over the module
uint32_t moved[48];
int nmoved = 0;
for (int i = 0; i < nov; i++) moved[nmoved++] = ov[i].var;
if (payload) moved[nmoved++] = payload;
uint32_t vcount = 0, pcount = 0;
int in_fn = 0, in_ep = 0, ep_started = 0, ep_done = 0;
for (size_t i = 5; i < n;) {
uint32_t o = w[i] & 0xffff, len = w[i] >> 16;
memcpy(ins, w + i, len * 4);
const uint32_t *a = w + i + 1;
if (o == OpFunction) in_fn = 1, in_ep = a[1] == m->ep_fn;
if (!in_fn) {
if (o == OpCapability) { if (a[0] != CapMeshShading) putn(&caps, ins, len); }
else if (o == OpExtension) { if (strcmp((const char *)a, "SPV_EXT_mesh_shader")) putn(&exts, ins, len); }
else if (o == OpExtInstImport) putn(&imps, ins, len);
else if (o == OpMemoryModel || o == OpEntryPoint) {}
else if (o == OpExecutionMode || o == OpExecutionModeId) {
if (!(o == OpExecutionMode && (a[1] == ModeOutputVertices || a[1] == ModeOutputPrimitives ||
a[1] == ModeOutputTriangles || a[1] == ModeOutputLines || a[1] == ModeOutputPoints)))
putn(&modes, ins, len);
} else if (is_debug(o)) putn(&debug, ins, len);
else if (is_annot(o)) {
int drop = 0;
for (int k = 0; k < nmoved; k++) drop |= o == OpDecorate && a[0] == moved[k];
if (!drop) putn(&annot, ins, len);
} else { // types, constants, global variables
int drop = 0;
if (o == OpTypePointer) {
for (int k = 0; k < npmap; k++) drop |= pmap[k].from == a[0];
if (a[1] == ScOutput || a[1] == ScTaskPayload) ins[2] = ScWorkgroup;
}
if (o == OpVariable && (a[2] == ScOutput || a[2] == ScTaskPayload)) ins[3] = ScWorkgroup;
for (uint32_t k = 1; k < len; k++) {
int idpos = o == OpTypeFunction || o == OpTypeStruct || o == OpTypeArray || o == OpTypeRuntimeArray ||
(o == OpTypePointer && k == 3) || k == 1;
if (o == OpTypePointer && k == 1) idpos = 0;
for (int q = 0; idpos && q < npmap; q++)
if (ins[k] == pmap[q].from) ins[k] = pmap[q].to;
}
if (!drop) putn(&types, ins, len);
}
i += len;
continue;
}
// function code
if (len >= 2)
for (int q = 0; q < npmap; q++)
if (ins[1] == pmap[q].from) ins[1] = pmap[q].to;
if (remap_wg && len >= 4 && (o == OpLoad || o == OpAccessChain || o == OpInBoundsAccessChain) && ins[3] == m->wgid) {
ins[3] = wgemu;
if (o != OpLoad) {
if (!p_in_u32 || ins[1] != p_in_u32) { *why = "an unexpected WorkgroupId access"; goto done; }
ins[1] = p_priv_u32;
}
} else if (remap_wg) {
for (uint32_t k = 1; k < len; k++) { // id operands only: literals of these instructions may equal the id
int lit = (o == OpExtInst && k == 4) || (o == 81 && k >= 4) || (o == 82 && k >= 5) || (o == 79 && k >= 5) ||
(o == OpLoad && k >= 4) || (o == OpStore && k >= 3) || o == 251 /* OpSwitch */ || o == OpSelectionMerge ||
o == 246 /* OpLoopMerge */;
if (!lit && ins[k] == m->wgid) { *why = "an unexpected WorkgroupId use"; goto done; }
}
}
if (o == OpFunctionEnd) in_fn = 0, in_ep = 0;
if (in_ep && o == OpLabel && !ep_started) {
putn(&code, ins, len);
size_t j = i + len; // function variables stay first
while (j < n && (w[j] & 0xffff) == OpVariable) putn(&code, w + j, w[j] >> 16), j += w[j] >> 16;
ep_started = 1;
i = j;
if (remap_wg) {
uint32_t lid = ld(&e, m->t_u32, m->lii), wg = ld(&e, m->t_v3u, m->wgid);
uint32_t x = E2(&e, OpCompositeExtract, m->t_u32, wg, 0u), y = E2(&e, OpCompositeExtract, m->t_u32, wg, 1u);
uint32_t base = words_base(&e), drw = pc_u32(&e, 1);
uint32_t tb = word_load(&e, base, add(&e, drw, cu(&e, 8)));
uint32_t slotc = E3(&e, OpSelect, m->t_u32, ok, slot, cu(&e, 0));
uint32_t recw = add(&e, cu(&e, MESH_EMU_TOFF / 4), mul(&e, slotc, cu(&e, MESH_EMU_TWORDS)));
uint32_t cnt = word_load(&e, base, recw);
uint32_t act = E2(&e, OpLogicalAnd, m->t_bool, ok, ult(&e, x, cnt));
uint32_t cont = nid(m), retl = nid(m);
op(&code, OpSelectionMerge, 2, cont, 0u);
op(&code, OpBranchConditional, 3, act, cont, retl);
op(&code, OpLabel, 1, retl);
op(&code, OpReturn, 0);
op(&code, OpLabel, 1, cont);
uint32_t v3 = nid(m);
op(&code, OpCompositeConstruct, 5, m->t_v3u, v3, x, cu(&e, 0), cu(&e, 0));
op(&code, OpStore, 2, wgemu, v3);
uint32_t v = word_load(&e, base, add(&e, add(&e, recw, cu(&e, 1)), k));
op(&code, OpStore, 2, E2(&e, OpAccessChain, p_wg_u32, payload, k), v);
}
barrier(&e);
}
continue;
}
if (in_ep && o == OpSetMeshOutputsEXT) {
vcount = a[0], pcount = a[1];
i += len;
continue;
}
if (in_ep && mesh && o == OpReturn) {
if (!vcount || ep_done) { *why = "no SetMeshOutputsEXT before the return, or several returns"; goto done; }
ep_done = 1;
i += len;
continue;
}
if (in_ep && !mesh && o == OpEmitMeshTasksEXT) {
task_epilogue(&e, info, payload, p_wg_u32, nwg, emit_x, emit_y, emit_z);
ep_done = 1;
i += len;
continue;
}
putn(&code, ins, len);
i += len;
}
if (!ep_done) { *why = "the entry function's end was not found"; goto done; }
// ---- assemble
put(&out, 0x07230203);
put(&out, w[1] < 0x00010500 ? 0x00010500 : w[1]); // SPIR-V 1.5: physical storage buffers are core
put(&out, w[2]);
put(&out, m->bound);
put(&out, 0);
putn(&out, caps.w, caps.n);
if (!m->has_int64) op(&out, OpCapability, 1, (uint32_t)CapInt64);
if (!m->has_psb) op(&out, OpCapability, 1, (uint32_t)CapPSBAddresses);
putn(&out, exts.w, exts.n);
putn(&out, imps.w, imps.n);
op(&out, OpMemoryModel, 2, 5348u, 1u); // PhysicalStorageBuffer64, GLSL450
{ // entry point: every global variable
Buf ep = {0};
uint32_t pre[2] = {ModelGLCompute, m->ep_fn};
uint32_t ifc[2048 + 16];
int ni = 0;
for (int i = 0; i < m->nvar && ni < 2048; i++) {
int dropped = 0;
if (!dropped && m->var[i].sc != ScFunction) ifc[ni++] = m->var[i].id;
}
// the new ones: scan ntypes for OpVariable
for (size_t i = 0; i < ntypes.n;) {
uint32_t o = ntypes.w[i] & 0xffff, len = ntypes.w[i] >> 16;
if (o == OpVariable && ni < 2048 + 16) ifc[ni++] = ntypes.w[i + 2];
i += len;
}
op_str(&ep, OpEntryPoint, pre, 2, "main", ifc, ni);
putn(&out, ep.w, ep.n);
free(ep.w);
}
putn(&out, modes.w, modes.n);
putn(&out, debug.w, debug.n);
putn(&out, annot.w, annot.n);
putn(&out, nannot.w, nannot.n);
putn(&out, types.w, types.n);
putn(&out, ntypes.w, ntypes.n);
putn(&out, code.w, code.n);
out.w[3] = m->bound;
if (caps.bad || exts.bad || imps.bad || modes.bad || debug.bad || annot.bad || types.bad || ntypes.bad ||
nannot.bad || code.bad || out.bad) { *why = "out of memory"; goto done; }
ret = out.w, *out_n = out.n, out.w = NULL;
done:
free(caps.w), free(exts.w), free(imps.w), free(modes.w), free(debug.w), free(annot.w), free(types.w), free(ntypes.w),
free(nannot.w), free(code.w), free(out.w);
free(m), free(ins);
return ret;
}
// ---- the vertex shader of an emulated mesh pipeline
Buf b = {0}, t = {0}, an = {0}, c = {0};
uint32_t id = 1;
#define N() (id++)
uint32_t glsl = N(), fn = N(), t_void = N(), t_fn = N(), t_u32 = N(), t_i32 = N(), t_f32 = N(), t_u64 = N(),
t_v4u = N(), t_v4f = N(), t_v4i = N(), rt_v4 = N(), s_slots = N(), p_slots = N(), p_v4u_psb = N(),
pc_struct = N(), p_pc = N(), pc_var = N(), p_pc_u64 = N(), p_pc_u32 = N(), p_in_i32 = N(),
(void)glsl;
uint32_t vecs[3][5] = {{0}}; // [type][comps] vector ids (comps 2..4), [type][1] the scalar
vecs[0][1] = t_f32, vecs[1][1] = t_i32, vecs[2][1] = t_u32, vecs[0][4] = t_v4f, vecs[1][4] = t_v4i, vecs[2][4] = t_v4u;
op(&t, OpTypeVoid, 1, t_void);
op(&t, OpTypeFunction, 2, t_fn, t_void);
op(&t, OpTypeInt, 3, t_u32, 32u, 0u);
op(&t, OpTypeInt, 3, t_i32, 32u, 1u);
op(&t, OpTypeFloat, 2, t_f32, 32u);
op(&t, OpTypeInt, 3, t_u64, 64u, 0u);
op(&t, OpTypeVector, 3, t_v4u, t_u32, 4u);
op(&t, OpTypeVector, 3, t_v4f, t_f32, 4u);
op(&t, OpTypeVector, 3, t_v4i, t_i32, 4u);
for (int ty = 0; ty < 3; ty++)
for (int k = 2; k < 4; k++) {
vecs[ty][k] = N();
op(&t, OpTypeVector, 3, vecs[ty][k], vecs[ty][1], (uint32_t)k);
}
op(&t, OpTypeRuntimeArray, 2, rt_v4, t_v4u);
op(&t, OpTypeStruct, 2, s_slots, rt_v4);
op(&t, OpTypePointer, 3, p_slots, (uint32_t)ScPhysicalStorageBuffer, s_slots);
op(&t, OpTypePointer, 3, p_v4u_psb, (uint32_t)ScPhysicalStorageBuffer, t_v4u);
op(&t, OpTypeStruct, 8, pc_struct, t_u64, t_u32, t_u32, t_u32, t_u32, t_u32, t_u32);
op(&t, OpTypePointer, 3, p_pc, (uint32_t)ScPushConstant, pc_struct);
op(&t, OpTypePointer, 3, p_pc_u64, (uint32_t)ScPushConstant, t_u64);
op(&t, OpTypePointer, 3, p_pc_u32, (uint32_t)ScPushConstant, t_u32);
op(&t, OpTypePointer, 3, p_in_i32, (uint32_t)ScInput, t_i32);
op(&t, OpTypePointer, 3, p_out_v4f, (uint32_t)ScOutput, t_v4f);
op(&t, OpConstant, 3, t_u32, c0, 0u);
op(&t, OpConstant, 3, t_u32, c4, 4u);
op(&t, OpConstant, 4, t_u64, c16, 16u, 0u);
op(&t, OpVariable, 3, p_pc, pc_var, (uint32_t)ScPushConstant);
op(&t, OpVariable, 3, p_in_i32, vidx, (uint32_t)ScInput);
op(&t, OpVariable, 3, p_out_v4f, pos, (uint32_t)ScOutput);
op(&an, OpDecorate, 3, rt_v4, (uint32_t)DecArrayStride, 16u);
op(&an, OpDecorate, 2, s_slots, (uint32_t)DecBlock);
op(&an, OpMemberDecorate, 4, s_slots, 0u, (uint32_t)DecOffset, 0u);
op(&an, OpDecorate, 2, pc_struct, (uint32_t)DecBlock);
for (uint32_t k = 0; k < 7; k++) op(&an, OpMemberDecorate, 4, pc_struct, k, (uint32_t)DecOffset, k ? 4 + 4 * k : 0u);
op(&an, OpDecorate, 3, vidx, (uint32_t)DecBuiltIn, (uint32_t)BiVertexIndex);
op(&an, OpDecorate, 3, pos, (uint32_t)DecBuiltIn, (uint32_t)BiPosition);
ifc[ni++] = pc_var, ifc[ni++] = vidx, ifc[ni++] = pos;
uint32_t p = N(), v = N();
op(&t, OpVariable, 3, p, v, (uint32_t)ScOutput);
ifc[ni++] = v;
}
// code: slot = vertex area + VertexIndex; each output from slot + its index
uint32_t lab = N();
op(&c, OpFunction, 4, t_void, fn, 0u, t_fn);
op(&c, OpLabel, 1, lab);
uint32_t e64 = N(), e64p = N(), vw = N(), vwp = N(), vi = N(), viu = N(), s = N(), s64 = N(), off = N(), a = N(), base = N();
op(&c, OpAccessChain, 4, p_pc_u64, e64p, pc_var, c0);
op(&c, OpLoad, 3, t_u64, e64, e64p);
op(&c, OpAccessChain, 4, p_pc_u32, vwp, pc_var, c4);
op(&c, OpLoad, 3, t_u32, vw, vwp);
op(&c, OpLoad, 3, t_i32, vi, vidx);
op(&c, OpBitcast, 3, t_u32, viu, vi);
op(&c, OpIAdd, 4, t_u32, s, vw, viu);
op(&c, OpUConvert, 3, t_u64, s64, s);
op(&c, OpIMul, 4, t_u64, off, s64, c16);
op(&c, OpIAdd, 4, t_u64, a, e64, off);
op(&c, OpConvertUToPtr, 3, p_slots, base, a);
uint32_t slot_of[33];
for (int k = 0; k < 33; k++) slot_of[k] = 0xffffffffu;
slot_of[32] = 0; // position
for (uint32_t g = 0; g < mi->n_generic; g++) slot_of[mi->generic_loc[g]] = 1 + g;
uint32_t val;
if (slot_of[l] == 0xffffffffu) {
val = 0;
} else {
uint32_t ci = N(), p = N();
val = N();
op(&t, OpConstant, 3, t_u32, ci, slot_of[l]);
op(&c, OpAccessChain, 5, p_v4u_psb, p, base, c0, ci);
op(&c, OpLoad, 5, t_v4u, val, p, 2u, 16u);
}
uint32_t f = N();
op(&c, OpBitcast, 3, t_v4f, f, val);
op(&c, OpStore, 2, pos, f);
continue;
}
if (!val) { // not written by the mesh stage: zero
uint32_t z = N();
r = N();
} else {
uint32_t cv = val;
if (ty != 2) cv = N(), op(&c, OpBitcast, 3, v4t, cv, val);
if (k == 4) r = cv;
else {
r = N();
}
}
}
op(&c, OpReturn, 0);
op(&c, OpFunctionEnd, 0);
// assemble
put(&b, 0x07230203), put(&b, 0x00010500), put(&b, 0), put(&b, id), put(&b, 0);
op(&b, OpCapability, 1, (uint32_t)CapShader);
op(&b, OpCapability, 1, (uint32_t)CapInt64);
op(&b, OpCapability, 1, (uint32_t)CapPSBAddresses);
op(&b, OpMemoryModel, 2, 5348u, 1u);
uint32_t pre[2] = {ModelVertex, fn};
op_str(&b, OpEntryPoint, pre, 2, "main", ifc, (int)ni);
putn(&b, an.w, an.n);
putn(&b, t.w, t.n);
putn(&b, c.w, c.n);
b.w[3] = id;
int bad = b.bad || t.bad || an.bad || c.bad;
free(t.w), free(an.w), free(c.w);
if (bad) { free(b.w); return NULL; }
*out_n = b.n;
return b.w;

37
runtime/mesh_emu_spirv.h Normal file
View File

@@ -0,0 +1,37 @@
#pragma once
// Mesh shader emulation for GPUs without VK_EXT_mesh_shader (NX2NAVI_MESH_EMULATION, vk_pipeline.c / vk_exec_draw.c):
// the translator's TaskEXT and MeshEXT modules (tools/xlate/MeshSpirv.cs) become GLCompute modules that write their
// vertices, triangles and task payloads into one GPU buffer (the arena), and a generated vertex shader reads the
// vertices back for the game's fragment shader. Plain C over SPIR-V words; no Vulkan types.
#include <stddef.h>
#include <stdint.h>
// The arena: one buffer, its address in the push constants of every emulation stage.
// MESH_EMU_DOFF: draw records, MESH_EMU_DWORDS words each: DrawIndexedIndirect {indexCount, instanceCount,
// MESH_EMU_TOFF: task records, MESH_EMU_TWORDS words each: mesh workgroup count, then the payload
// index area (uint32 indices), vertex area (16-byte slots: position, then each generic output) at offsets the
// replay passes in the push constants.
#define MESH_EMU_DOFF 256u
#define MESH_EMU_DWORDS 16u
#define MESH_EMU_DRECS 4096u
#define MESH_EMU_TOFF (MESH_EMU_DOFF + MESH_EMU_DRECS * MESH_EMU_DWORDS * 4u)
// Push constants of every emulation stage (compute and the vertex shader), 32 bytes:
// u64 arena address; uint draw record (word index); uint index area (word index); uint index capacity;
#define MESH_EMU_PC_BYTES 32u
typedef struct MeshEmuInfo {
int task; // 1 TaskEXT, 0 MeshEXT
uint32_t threads; // LocalSize x
uint32_t max_v, max_p; // mesh: OutputVertices, OutputPrimitivesEXT
uint32_t payload_words; // the task payload's size (mesh: what it reads, 0 without a task stage)
uint32_t max_mesh; // task: the most mesh workgroups one task workgroup launches (dispatch width)
uint32_t slots; // mesh: 16-byte slots per vertex: position and each generic output
uint32_t n_generic; // mesh: generic outputs, in slot order 1..n
uint8_t generic_loc[32]; // their locations
} MeshEmuInfo;
// The compute module for a TaskEXT or MeshEXT module (malloc'd, *out_n words); NULL with why set when the module is not
// one the converter knows (it then stays unemulated: its draws are skipped as before).
uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_t *out_n, const char **why);
// The vertex shader of an emulated mesh pipeline: position and the generic outputs from the vertex slots, as types[l]

View File

@@ -830,6 +830,10 @@ typedef struct {
// from the start of the set's descriptors. VK_NULL_HANDLE: the set is written with VkWriteDescriptorSet structs.
VkDescriptorUpdateTemplate tpl[2];
int pool; // a stage indexes the texture or sampler pool: bind set 2 (the pools)
// NX2NAVI_MESH_EMULATION: an emulated mesh pipeline (a generated vertex shader and the fragment stage; the task and
// mesh stages are compute pipelines of the same layout, vk_pipeline_mesh_emu), and set 0's compute-side template.
int emu;
VkDescriptorUpdateTemplate emu_tpl_c;
} VkPipeInfo;
// One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the
// bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte.
@@ -866,6 +870,10 @@ typedef struct {
} VkPrewarmProgress;
void vk_pipeline_prewarm_progress(VkPrewarmProgress *p); // any thread, while vk_pipeline_prewarm runs or after
VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info);
// NX2NAVI_MESH_EMULATION: the compute pipelines of an emulated mesh pipeline (built with it); 0 when it has none.
struct MeshEmuInfo;
int vk_pipeline_mesh_emu(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, VkPipeline *mesh,
VkPipeline *task, struct MeshEmuInfo *mi);
// Records every dynamic state that VkPipeState determines, after binding a graphics pipeline: cull mode, front face,
// topology, depth and stencil tests and ops, depth bias enable, rasterizer discard, vertex input, and, where the GPU
// has them dynamic, depth clamp, polygon mode, blend enables, equations, write masks and alpha to coverage. The replay

View File

@@ -33,6 +33,7 @@
#include "vk_mesh_fb.h"
#include "platform/platform.h"
#include "vk_dlaa.h"
#include "mesh_emu_spirv.h"
#define MAX_BINDINGS 128
#define MAX_DESC 1024
@@ -3422,10 +3423,12 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c
// graphics push equal to the last one at its set number, with no other layout or bind between, is not recorded.
// The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update
// template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template.
static uint32_t emu_d_bytes;
static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
DrawCtx *const cx = ex->dc;
VkWriteDescriptorSet *w = cx->bind_w;
VkDescSlot *d = cx->bind_d;
emu_d = d;
VkBufferView *bv = cx->bind_bv;
if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS);
bind_stats(info);
@@ -3567,6 +3570,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
// An empty set 1 (a fragment stage without resources: depth and shadow draws) starts past the last binding, which
// the loop never reaches: d0[1] stayed 0 and set 0 was pushed with 0 bytes, so the driver read stale descriptors
if (n_sets == 2 && lo[1] == hi[1]) d0[1] = nd;
emu_d_bytes = (uint32_t)((n_sets > 1 ? d0[1] : nd) * sizeof *d);
// Under the validation layer, a new graphics layout binds set 1 before set 0 is pushed or bound. The layer (SDK
// 1.4.363, PipelineLayoutCompatDef::operator==) takes any two INDEPENDENT_SETS layouts as compatible, so
// vkCmdPushDescriptorSetKHR keeps the push set it made for an earlier layout and drops writes to bindings that
@@ -4126,6 +4130,7 @@ static void prog_lead(const prog_t *p, int mesh) {
(unsigned long long)PLD.n[m][4], (unsigned long long)PLD.n[m][5]);
}
static const VkPipeInfo *PD_info; // the pipeline of the last draw prepare_draw_ex prepared
// mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing
// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. 2: a mesh draw of
// the mesh fallback (mfb_draw): the vertex stage is DrawCtx.mfb_cur_vs, in an ordinary pipeline without vertex input.
@@ -4328,7 +4333,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
delta = cfg_bool("NX2NAVI_DYN_DELTA");
if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)");
}
if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &cx->B.dyn, mesh && !fb,
// an emulated mesh draw (NX2NAVI_MESH_EMULATION) binds a vertex pipeline: its topology, primitive restart and
// vertex input are dynamic and are set as for any vertex draw
if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &cx->B.dyn, mesh && !fb && !info->emu,
&vk_exec_frame.dyn_left);
else vk_pipeline_set_dynamic(ex->cb, &s);
ex->recorded++;
@@ -4359,6 +4366,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
vk_exec_oq_draw(ex);
DRS.issued[mesh]++;
draw_stats_tick();
PD_info = info;
return 1;
}
@@ -5561,7 +5569,17 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) {
ex->recorded++;
}
static int mesh_emu_on(void) {
static int on = -1;
if (on < 0) {
if (on) rt_log("vk: mesh shaders emulated with compute (NX2NAVI_MESH_EMULATION)");
}
return on;
}
static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) {
PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn();
if (!fn && mesh_emu_on()) {
static int on = -1;
if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis)
if (!d->count || !on) return;
@@ -5570,8 +5588,7 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) {
mfb_draw(ex, d->count, NULL);
return;
}
PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn();
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader (mesh_emulation = 1 emulates them): skipped");
ex->dc->c0_binst = 0;
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;
if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first);
@@ -5690,6 +5707,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan
vk_exec_barrier(ex); // ends the pass (the next draw opens it again) and orders the culling's writes before the copy
vk_gpu_gap_note(VK_GAP_MESH_RING, 12);
vkCmdFillBuffer(ex->cb, mesh_ring, *at + 4, 8, 1);
}
VkBufferCopy cp = {src->offset, *at, 4};
vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp);
mesh_count_note(ex, src);
@@ -5719,8 +5737,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan
.dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT,
.dstAccessMask = VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(ex->cb, &di);
ex->recorded += 3;
vkCmdPipelineBarrier2(cb, &di);
return 1;
}
@@ -5746,7 +5763,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) {
return;
}
VkDeviceSize at;
if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped");
if (!mesh_ring_entry(ex, ex->cb, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped");
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;
vk_exec_frame.draws++;
static uint32_t n_logged;
@@ -5764,6 +5781,86 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) {
VK_MEM_BOUND(ip);
}
// ---- NX2NAVI_MESH_EMULATION: mesh draws on GPUs without VK_EXT_mesh_shader
// Each mesh draw (prepared like one, prepare_draw_ex, for its layout, descriptors and graphics state) runs its task
// and mesh stages as compute pipelines (vk_pipeline_mesh_emu) outside the pass: they write vertices and triangles into
// the arena (mesh_emu_spirv.h) and the draw record's DrawIndexedIndirect, which the generated vertex shader and the
// game's fragment shader then draw. The arena's counters are zeroed at the first emulated draw of each submission;
int failed;
uint64_t draws, skipped;
} ME;
return 1;
}
static int me_arena(void) {
if (ME.buf) return 1;
if (ME.failed) return 0;
VkDeviceSize size = (VkDeviceSize)mb << 20;
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT |
VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
return 0;
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ME.buf, &ME.alloc, NULL) != VK_SUCCESS) {
ME.buf = VK_NULL_HANDLE, ME.failed = 1;
rt_log("vk: mesh emulation: the %d MiB arena could not be created: mesh draws skipped", mb);
return 0;
VkBufferDeviceAddressInfo dai = {.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, .buffer = ME.buf};
ME.addr = vkGetBufferDeviceAddress(vk_ctx()->device, &dai);
uint64_t voff = ioff + ibytes;
ME.iw = (uint32_t)(ioff / 4), ME.icap = (uint32_t)(ibytes / 4), ME.vw = (uint32_t)(voff / 16), ME.vcap = (uint32_t)((size - voff) / 16);
return 1;
static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, VkAccessFlags2 da) {
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(cb, &di);
}
}
// count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from
if (!me_arena()) return 0;
ex->dc->c0_binst = 0;
const VkPipeInfo *info = PD_info;
const GfxShader *ms = ex->dc->B.st.vs, *ts = ex->dc->B.st.ts;
VkPipeline mp = VK_NULL_HANDLE, tp = VK_NULL_HANDLE;
MeshEmuInfo mi;
if (!info || !info->emu || !ex->dc->B.st_valid || !vk_pipeline_mesh_emu(ms, ts, info->layout, &mp, &tp, &mi) || !!ts != !!tp ||
(info->push && !info->emu_tpl_c) || (!info->push && !ex->dc->B.ds_bound[0])) {
ME.skipped++;
return 0;
VkPipelineLayout layout = info->layout;
int push = info->push, pool = info->pool;
VkDescriptorUpdateTemplate tpl_c = info->emu_tpl_c;
VkDescriptorSet set0 = ex->dc->B.ds_bound[0];
}
}
uint32_t ep = vk_texture_epoch();
if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over
ME.epoch = ep;
me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT,
VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
uint32_t rec = ME.rec++ % MESH_EMU_DRECS;
VkDeviceSize ro = MESH_EMU_DOFF + (VkDeviceSize)rec * MESH_EMU_DWORDS * 4;
vkCmdFillBuffer(cb, ME.buf, ro, 4, 0);
vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1);
cb = ex->cb;
vkCmdBindIndexBuffer(cb, ME.buf, (VkDeviceSize)ME.iw * 4, VK_INDEX_TYPE_UINT32);
vkCmdPushConstants(cb, layout, VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES, pc);
vkCmdDrawIndexedIndirect(cb, ME.buf, ro, 1, 20);
ex->recorded += 3;
vk_exec_frame.draws++;
// the index buffer and push constants are the emulation's: the next draw binds everything again
vk_exec_draw_new_cb();
ex->dirty = VK_EXEC_DIRTY_ALL;
ME.draws++;
static uint64_t log_frame;
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
if ((rt_trace & RT_TRACE_PERF) && f / 600 != log_frame / 600) {
log_frame = f;
ME.draws = ME.skipped = 0;
return 1;
}
static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType type) {
VkBuffer b;
VkDeviceSize o;

View File

@@ -43,6 +43,7 @@
#include <string.h>
#include "cfg.h"
#include "mesh_emu_spirv.h"
#include "gfx.h"
#include "host.h"
#include "nvn_hmap.h"
@@ -130,6 +131,8 @@ typedef struct Layout {
uint32_t n0; // independent layouts: b[0..n0) are set 0, the rest set 1
VkPipelineLayout half[2]; // GPL libraries: {set 0, null}, {null, set 1} (made on first use)
VkDescriptorUpdateTemplate tpl[2]; // per set (make_tpl); VK_NULL_HANDLE: written with write structs
uint8_t emu; // NX2NAVI_MESH_EMULATION: a mesh layout whose task and mesh stages run as compute
VkDescriptorUpdateTemplate tpl_c; // emu: set 0's template for the compute bind point
VkDescriptorSetLayoutBinding b[];
} Layout;
@@ -2588,8 +2591,8 @@ static int layout_covers(const VkPipeState *s, const Layout *l, uint32_t fs_off,
// layouts hold the compute stage only). VK_NULL_HANDLE when templates are off, for an empty set, for a push set
// without vkCmdPushDescriptorSetWithTemplateKHR or when creation fails: the replay then writes the set with write
// structs.
static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl,
int push, VkPipelineLayout pl, uint32_t set) {
static VkDescriptorUpdateTemplate make_tpl_bp(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl,
int push, VkPipelineLayout pl, uint32_t set, int bp) {
if (!P.tpl || !n || (push && !P.tpl_push)) return VK_NULL_HANDLE;
VkDescriptorUpdateTemplateEntry e[MAX_BINDINGS];
size_t off = 0;
@@ -2604,7 +2607,8 @@ static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b
.pDescriptorUpdateEntries = e,
.templateType = push ? VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR : VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET,
.descriptorSetLayout = sl,
.pipelineBindPoint = stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS,
.pipelineBindPoint = bp >= 0 ? (VkPipelineBindPoint)bp
: stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS,
.pipelineLayout = pl, .set = set};
VkDescriptorUpdateTemplate t;
if (P.tpl_create(P.device, &ci, NULL, &t) != VK_SUCCESS) {
@@ -2613,6 +2617,10 @@ static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b
}
return t;
}
static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl,
int push, VkPipelineLayout pl, uint32_t set) {
return make_tpl_bp(b, n, sl, push, pl, set, -1);
}
// Finds or creates the layout of a binding list (sorted here); NULL on failure. Caller holds P.lock.
// Sorts a binding list by binding and hashes it into *h; the descriptor total, or -1 when a binding repeats.
@@ -2667,6 +2675,124 @@ static Layout *layout_get(Bindings *B) {
return l;
}
// NX2NAVI_MESH_EMULATION (GPUs without VK_EXT_mesh_shader): the layout of an emulated mesh pipeline. The task and
// mesh stages' bindings are visible to the compute stage (they run as compute pipelines of this layout), the
// fragment stage's stay; push constants (MESH_EMU_PC_BYTES) for the compute stages and the generated vertex shader;
// set 0 gets a template per bind point. Caller holds P.lock.
static int mesh_emu_on(void) {
static int on = -1;
return on;
}
static Layout *layout_get_emu(Bindings *B) {
for (uint32_t i = 0; i < B->n; i++) {
VkShaderStageFlags f = B->b[i].stageFlags;
if (f & (VK_SHADER_STAGE_MESH_BIT_EXT | VK_SHADER_STAGE_TASK_BIT_EXT))
f = (f & ~(VK_SHADER_STAGE_MESH_BIT_EXT | VK_SHADER_STAGE_TASK_BIT_EXT)) | VK_SHADER_STAGE_COMPUTE_BIT;
B->b[i].stageFlags = f;
B->pb[i].stages = f;
}
uint64_t h = fnv(FNV0, "mesh-emu", 8);
int64_t total = bindings_sort(B, &h);
if (total < 0) return NULL;
Layout **head = &layouts[h % LAYOUT_BUCKETS];
for (Layout *l = *head; l; l = l->next)
if (l->hash == h && l->emu && l->n == B->n && !memcmp(l->b, B->b, B->n * sizeof *B->b)) return l;
Layout *l = calloc(1, sizeof *l + B->n * sizeof *B->b);
if (!l) return NULL;
l->hash = h, l->n = B->n, l->emu = 1;
memcpy(l->b, B->b, B->n * sizeof *B->b);
l->push = total <= (int64_t)P.max_push;
VkDescriptorSetLayoutCreateInfo si = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0,
.bindingCount = l->n, .pBindings = l->b};
if (vkCreateDescriptorSetLayout(P.device, &si, NULL, &l->set) != VK_SUCCESS) {
free(l);
return NULL;
}
VkDescriptorSetLayout sets3[3] = {l->set, vk_empty_set_layout(), vk_pool_set_layout()};
VkPushConstantRange pcr = {VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES};
VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 3,
.pSetLayouts = sets3, .pushConstantRangeCount = 1, .pPushConstantRanges = &pcr};
if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) {
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
free(l);
return NULL;
}
l->tpl[0] = make_tpl_bp(l->b, l->n, l->set, l->push, l->layout, 0, VK_PIPELINE_BIND_POINT_GRAPHICS);
l->tpl_c = make_tpl_bp(l->b, l->n, l->set, l->push, l->layout, 0, VK_PIPELINE_BIND_POINT_COMPUTE);
l->next = *head;
*head = l;
P.stats.layouts++;
return l;
}
// The compute pipelines of an emulated mesh pipeline, per (mesh, task, layout): made by the pipeline's build
// (create_graphics, any thread) and found by its draws. ok 0: the stages could not be converted or built.
typedef struct MeshEmuEnt {
struct MeshEmuEnt *next;
const GfxShader *ms, *ts;
VkPipelineLayout layout;
int ok;
VkPipeline mesh, task;
MeshEmuInfo mi, ti;
} MeshEmuEnt;
static PlatMutex emu_lock = PLAT_MUTEX_INIT;
return e;
}
static VkPipeline emu_compute(const uint32_t *w, size_t n, VkPipelineLayout layout) {
VkShaderModule m = make_module(w, n, 0);
if (!m) return VK_NULL_HANDLE;
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
.layout = layout};
VkPipeline p = VK_NULL_HANDLE;
if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &p) != VK_SUCCESS) p = VK_NULL_HANDLE;
vkDestroyShaderModule(P.device, m, NULL);
return p;
}
static const MeshEmuEnt *emu_get(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, uint32_t toff, int create) {
e = calloc(1, sizeof *e);
e->ms = ms, e->ts = ts, e->layout = layout;
const char *why = NULL;
uint64_t n = 0;
size_t cn = 0;
const uint32_t *w = gfx_shader_spirv(ms, &n);
uint32_t *c = w ? mesh_emu_compute(w, (size_t)n, &e->mi, &cn, &why) : NULL;
if (c && (e->mesh = emu_compute(c, cn, layout))) e->ok = 1;
else if (!why) why = "the mesh stage's compute pipeline was not created";
free(c);
if (e->ok && e->mi.payload_words && !ts) e->ok = 0, why = "a mesh stage that reads a task payload without a task stage";
if (e->ok && ts) {
w = gfx_shader_spirv(ts, &n);
c = w ? mesh_emu_compute(w, (size_t)n, &e->ti, &cn, &why) : NULL;
uint32_t *r = c && toff ? spirv_rebind(c, cn, toff) : NULL; // the task stage's bindings follow the fragment stage's
if (c && (e->task = emu_compute(r ? r : c, cn, layout))) {
if (e->ti.payload_words != e->mi.payload_words) e->ok = 0, why = "task and mesh payload sizes differ";
} else {
e->ok = 0;
if (!why) why = "the task stage's compute pipeline was not created";
}
free(r), free(c);
}
plat_mutex_unlock(&emu_lock);
return e;
}
int vk_pipeline_mesh_emu(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, VkPipeline *mesh,
VkPipeline *task, MeshEmuInfo *mi) {
const MeshEmuEnt *e = emu_get(ms, ts, layout, 0, 0);
if (!e || !e->ok) return 0;
*mesh = e->mesh, *task = e->task;
*mi = e->mi;
if (ts) mi->max_mesh = e->ti.max_mesh;
return 1;
}
// Independent layouts (P.indep): the vertex stage's bindings in set 0, the fragment stage's in set 1 (empty without a
// fragment shader), each with its own support buffer at binding 0, the pipeline layout created with
// VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT so stage libraries can use it. Only the P.push_stage set is a
@@ -2793,6 +2919,7 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel
p->info.n_sets = l && l->set1 ? 2 : 1;
p->info.pool = B ? B->pool : 0;
if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1];
if (l && l->emu) p->info.emu = 1, p->info.emu_tpl_c = l->tpl_c;
p->lay = l;
Pipe **head = &pipes[h % PIPE_BUCKETS];
p->next = *head;
@@ -3111,11 +3238,28 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay
uint64_t vw, fw, vm_hash = 0, fm_hash = 0; // the hashes: of the final modules, for the PIPE_FEEDBACK 2 line
const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw);
uint32_t *vpatched = NULL, *fpatched = NULL;
int mesh = is_mesh(s);
int mesh = is_mesh(s), emu = mesh && l->emu;
// the water pair's vertex edit is for a translated vertex stage: an emulated mesh draw's generated one has none
int wmode = mesh || !l->set1 ? 0 : water_mode(s, k); // the edited fragment modules are in their set 1 form
uint64_t wn = 0;
const uint32_t *vwater = water_pair_vertex(vcode, vw, wmode, &wn);
if (vwater) vcode = vwater, vw = wn;
if (emu) { // NX2NAVI_MESH_EMULATION: the task and mesh stages as compute pipelines, a generated vertex shader here
uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0);
const MeshEmuEnt *me = emu_get(s->vs, s->ts, l->layout, toff, 1);
if (!me || !me->ok) return VK_NULL_HANDLE;
GfxShaderIo fin[32];
uint8_t ty[32], cp[32];
uint32_t need = s->fs ? gfx_shader_inputs(s->fs, fin) : 0;
for (int i = 0; i < 32; i++) {
ty[i] = need >> i & 1 && !fin[i].packed && fin[i].type <= GFX_TYPE_UINT ? (uint8_t)fin[i].type : 0;
cp[i] = need >> i & 1 && !fin[i].packed && fin[i].comps ? (uint8_t)fin[i].comps : 4;
}
size_t vn = 0;
if (!vpatched) return VK_NULL_HANDLE;
vcode = vpatched, vw = vn;
mesh = 0; // a vertex pipeline from here on
}
VkShaderModule tm = VK_NULL_HANDLE;
if (mesh && s->ts) { // the task stage's bindings follow the fragment stage's (pipe_miss)
uint64_t tw;
@@ -3137,14 +3281,15 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay
}
if (s->fs) {
GfxShaderIo fin[32], vout[32];
// a mesh stage's outputs from its module (spirv_mesh_outputs): the translation reflects none
// a mesh stage's outputs from its module (spirv_mesh_outputs): the translation reflects none; an emulated one
// (mesh_emu) writes what the fragment stage reads
int points = 0;
uint32_t need = gfx_shader_inputs(s->fs, fin), missing = 0,
have = mesh ? (vcode ? spirv_mesh_outputs(vcode, vw, &points) : ~0u) : gfx_shader_outputs(s->vs, vout);
have = mesh ? (vcode ? spirv_mesh_outputs(vcode, vw, &points) : ~0u) : emu ? ~0u : gfx_shader_outputs(s->vs, vout);
for (int i = 0; i < 32; i++) {
if (!(need >> i & 1) || fin[i].packed) continue;
if (!(have >> i & 1)) missing |= 1u << i;
else if (!mesh && vout[i].type != fin[i].type)
else if (!mesh && !emu && vout[i].type != fin[i].type)
LOG_ONCE("vk_pipeline: varying %d is type %d in the vertex stage, %d in the fragment stage", i, vout[i].type, fin[i].type);
}
if (missing && !l->set1 && !mesh) {
@@ -3193,7 +3338,7 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay
VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqv, rqf;
uint64_t sw;
const uint32_t *scode = gfx_shader_spirv(s->vs, &sw); // vcode may be a patched copy that is freed by now
pin_subgroup32(&stages[0], &rqv, scode, sw);
if (!emu) pin_subgroup32(&stages[0], &rqv, scode, sw);
if (s->fs) pin_subgroup32(&stages[1], &rqf, gfx_shader_spirv(s->fs, &sw), sw);
VkVertexInputAttributeDescription va[32];
VkVertexInputBindingDescription vb[17];
@@ -4509,11 +4654,12 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode
// vertex pipelines). They go to the workers like the others: built here, each new one stopped the frame for its
// whole compile (0.26-0.51 s in Kokiri Forest), and they are recorded for the boot prewarm (keys_note).
uint32_t fspan = s->fs ? binding_span(s->fs) : 1;
if (!vk_ctx()->has_mesh_shader) why = "the GPU has no mesh shaders";
int emu = !vk_ctx()->has_mesh_shader && mesh_emu_on();
if (!vk_ctx()->has_mesh_shader && !emu) why = "the GPU has no mesh shaders (mesh_emulation = 1 emulates them)";
else if ((s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS) || (s->ts && gfx_shader_stage(s->ts) != GFX_STAGE_TS)) why = "stage mismatch";
else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1)) ||
(s->ts && !add_stage(&B, s->ts, span - 1 + fspan - 1))) why = "too many bindings";
else if (!(l = layout_get(&B))) why = "descriptor layout creation failed";
else if (!(l = emu ? layout_get_emu(&B) : layout_get(&B))) why = "descriptor layout creation failed";
} else if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) why = "stage mismatch";
else if (k->topo_class == 3) why = "patches need tessellation stages, which are not translated";
else if (P.indep) {