Files
NX2navi/runtime/mesh_fb.c
chrg6 e92f0ae3f1 Clear the whole stage buffer in the mesh fallback too, before the mesh program runs.
On a GPU without mesh shaders the fallback replaces the translated module's entry function with its own, which cleared
only the primitive count and then copied the whole slot (vertex lines, count, index bytes) out of workgroup memory to
be drawn. Lines and index bytes the program did not write kept what an earlier workgroup left there and were drawn as
vertices and primitives. The fallback's entry function now sets every word of the slot to zero in a loop over the
workgroup's threads before the payload copy and the barrier, as the translator's own entry function does.

Checked: --probe-meshfb passes before and after (the fallback's pictures match the mesh pipeline's texel for texel on
an RTX GPU); its module part converts all 520 task and mesh programs of an installed shader cache with 0 failures, all
1,020 resulting modules pass spirv-val (vulkan1.3) and their 520 compute pipelines build on the device.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 19:38:17 -05:00

643 lines
34 KiB
C

// The mesh shader fallback's modules, for a GPU without VK_EXT_mesh_shader (vk_exec_draw.c "mesh fallback" draws with
// them; docs/architecture.md, "Mesh fallback").
//
// The translator turns a task or mesh program into a compute module whose shared memory is the program's stage buffer
// and wraps it in a TaskEXT or MeshEXT entry function (tools/xlate/MeshSpirv.cs). The wrapping is undone here and
// replaced: the translator's main function stays as it is, the entry function that hands the stage buffer to the mesh
// pipeline goes, and a compute entry function copies the stage buffer into a storage buffer instead.
// task: main; the payload area into the task buffer at the workgroup's place.
// mesh: [the payload from the task buffer into the payload area] main; the vertex lines and the primitive area into
// the workgroup's slot of the mesh buffer.
// draw: a vertex module made here reads a slot per instance: vertex index / indices per primitive is the
// primitive, its index byte names the vertex, and the vertex's lines are the outputs.
// The module is made from the translation the shader cache already holds, so the translator and with it every cache
// key stay as they are: a card with mesh shaders never sees these modules, and one without needs no other cache.
//
// Mesh workgroups after a task stage. A task workgroup starts up to `threads` mesh workgroups, and how many is known
// only on the GPU. A pass between the two dispatches (shaders/meshfb_scan.comp) lists them in task order: entry g of
// the list names the task workgroup and the index the mesh program sees as its workgroup id. That id comes from the
// WorkgroupId built-in in the translated code; here the built-in counts list entries, so main reads a private
// variable the entry function fills from the list.
//
// Opcode and enum numbers are the SPIR-V 1.6 specification's and SPV_EXT_mesh_shader's.
#include "mesh_fb.h"
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
enum {
OpName = 5, OpExtension = 10, OpMemoryModel = 14, OpEntryPoint = 15, OpExecutionMode = 16, OpCapability = 17,
OpTypeVoid = 19, OpTypeBool = 20, OpTypeInt = 21, OpTypeFloat = 22, OpTypeVector = 23, OpTypeArray = 28,
OpTypeRuntimeArray = 29, OpTypeStruct = 30, OpTypePointer = 32, OpTypeFunction = 33, OpConstant = 43,
OpConstantComposite = 44, OpConstantNull = 46, OpFunction = 54, OpFunctionEnd = 56, OpFunctionCall = 57,
OpVariable = 59, OpLoad = 61, OpStore = 62, OpCopyMemory = 63, OpAccessChain = 65, OpInBoundsAccessChain = 66,
OpPtrAccessChain = 67, OpArrayLength = 68, OpDecorate = 71, OpMemberDecorate = 72, OpCompositeConstruct = 80,
OpCompositeExtract = 81, OpCopyObject = 83, OpBitcast = 124, OpIAdd = 128, OpISub = 130, OpIMul = 132, OpUDiv = 134,
OpSelect = 169, OpINotEqual = 171, OpULessThan = 176, OpULessThanEqual = 178, OpShiftRightLogical = 194,
OpBitwiseAnd = 199, OpControlBarrier = 224, OpPhi = 245, OpLoopMerge = 246, OpSelectionMerge = 247, OpLabel = 248,
OpBranch = 249, OpBranchConditional = 250, OpReturn = 253, OpDecorateId = 332, OpDecorateString = 5632,
};
enum { ScInput = 1, ScOutput = 3, ScWorkgroup = 4, ScPrivate = 6, ScFunction = 7, ScStorageBuffer = 12, ScTaskPayload = 5402 };
enum { DecBlock = 2, DecArrayStride = 6, DecBuiltIn = 11, DecInvariant = 18, DecNonWritable = 24, DecLocation = 30,
DecBinding = 33, DecDescriptorSet = 34, DecOffset = 35 };
enum { BiPosition = 0, BiPointSize = 1, BiWorkgroupId = 26, BiLocalInvocationIndex = 29, BiVertexIndex = 42,
BiInstanceIndex = 43 };
enum { CapShader = 1, CapMeshShading = 5283, ModelVertex = 0, ModelGLCompute = 5, ModelTask = 5364, ModelMesh = 5365,
ModeLocalSize = 17 };
// ---- the program's layout
static void fail(char *err, size_t cap, const char *fmt, ...) {
if (!err || !cap) return;
va_list ap;
va_start(ap, fmt);
vsnprintf(err, cap, fmt, ap);
va_end(ap);
}
int meshfb_info(const uint8_t *control, const uint8_t *header, int stage, MeshFbInfo *m, char *err, size_t cap) {
uint32_t w58, w5c, w60;
memset(m, 0, sizeof *m);
memcpy(&w58, control + 0x58, 4), memcpy(&w5c, control + 0x5c, 4), memcpy(&w60, control + 0x60, 4);
m->stage = stage;
m->threads = w58 & 0xffff ? w58 & 0xffff : 1;
m->payload_bytes = (w58 >> 24) * 256;
m->maxv = w60 & 0xffff, m->maxp = w60 >> 16;
uint32_t topo = w5c >> 24;
m->ipp = topo == 1 ? 1 : topo == 2 ? 2 : 3;
m->has_task = stage == 7 && (w5c >> 16 & 0xff) != 0;
// The payload as the translator sizes it (tools/xlate/Program.cs Translate): a size of 0 is one 256-byte unit, and
// the header of 0x20 bytes comes on top.
if (stage == 6 || m->has_task) m->payload_bytes = (m->payload_bytes ? m->payload_bytes : 256) + 0x20;
if (stage == 6) {
m->pay = 0, m->shared_bytes = m->payload_bytes;
return 1;
}
if (stage != 7) return fail(err, cap, "stage %d is neither task nor mesh", stage), 0;
if (!m->maxv || !m->maxp || m->maxv > 256) return fail(err, cap, "limits of %u vertices and %u primitives", m->maxv, m->maxp), 0;
// The output map of the program header: system values B at byte 0x35, one bit per component from 0x60; generic
// vectors from byte 0x36, a nibble each. System values A and C never appear in the game's mesh programs.
if (header[0x34] || header[0x48] || header[0x49]) return fail(err, cap, "output map with system values A or C"), 0;
for (int b = 0; b < 8; b++)
if (header[0x35] >> b & 1) m->lines[m->nlines++] = (uint16_t)(0x60 + 4 * b);
for (int g = 0; g < 32; g++)
for (int c = 0; c < 4; c++)
if (header[0x36 + g / 2] >> (4 * (g & 1) + c) & 1) m->lines[m->nlines++] = (uint16_t)(0x80 + 16 * g + 4 * c);
uint32_t groups = (m->maxv + 31) / 32;
m->attr = 0;
m->map = (m->attr + groups * m->nlines * 128 + 15) & ~15u;
m->pay = (m->map + 4 + m->ipp * m->maxp + 15) & ~15u;
m->shared_bytes = m->pay + (m->has_task ? m->payload_bytes : 0);
return 1;
}
// ---- a growing list of SPIR-V words
typedef struct { uint32_t *w; size_t n, cap; int oom; } Wb;
static void wb_put(Wb *b, const uint32_t *w, size_t n) {
if (b->n + n > b->cap) {
size_t cap = (b->n + n) * 2 + 256;
uint32_t *g = realloc(b->w, cap * 4);
if (!g) { b->oom = 1; return; }
b->w = g, b->cap = cap;
}
memcpy(b->w + b->n, w, n * 4);
b->n += n;
}
static void wb_ins(Wb *b, uint32_t op, const uint32_t *operands, size_t n) {
uint32_t head = (uint32_t)((n + 1) << 16) | op;
wb_put(b, &head, 1);
wb_put(b, operands, n);
}
#define INS(b, op, ...) wb_ins(b, op, (const uint32_t[]){__VA_ARGS__}, sizeof((const uint32_t[]){__VA_ARGS__}) / 4)
// What a module under construction adds: decorations, then types, constants and variables, then one function.
typedef struct {
uint32_t bound;
Wb deco, glob, fn;
uint32_t t_void, t_bool, t_uint, t_fnvoid, t_v3uint; // existing or new
uint32_t p_fn_uint, p_sb_uint;
struct { uint32_t v, id; } k[96];
int nk;
} Cx;
static uint32_t nid(Cx *c) { return c->bound++; }
static uint32_t K(Cx *c, uint32_t v) { // a 32-bit unsigned constant
for (int i = 0; i < c->nk; i++)
if (c->k[i].v == v) return c->k[i].id;
uint32_t id = nid(c);
INS(&c->glob, OpConstant, c->t_uint, id, v);
if (c->nk < 96) c->k[c->nk].v = v, c->k[c->nk].id = id, c->nk++;
return id;
}
static uint32_t E_(Cx *c, uint32_t op, uint32_t type, const uint32_t *operands, size_t n) { // an instruction with a result
uint32_t id = nid(c), head[3] = {(uint32_t)((n + 3) << 16) | op, type, id};
wb_put(&c->fn, head, 3);
wb_put(&c->fn, operands, n);
return id;
}
#define E(c, op, type, ...) E_(c, op, type, (const uint32_t[]){__VA_ARGS__}, sizeof((const uint32_t[]){__VA_ARGS__}) / 4)
#define DO(c, op, ...) INS(&(c)->fn, op, __VA_ARGS__)
static uint32_t add(Cx *c, uint32_t a, uint32_t b) { return E(c, OpIAdd, c->t_uint, a, b); }
static uint32_t mul(Cx *c, uint32_t a, uint32_t b) { return E(c, OpIMul, c->t_uint, a, b); }
static uint32_t umin(Cx *c, uint32_t a, uint32_t b) {
return E(c, OpSelect, c->t_uint, E(c, OpULessThan, c->t_bool, a, b), a, b);
}
// if (cond) { ...the instructions emitted until if_end }
static uint32_t if_begin(Cx *c, uint32_t cond) {
uint32_t then = nid(c), merge = nid(c);
DO(c, OpSelectionMerge, merge, 0);
DO(c, OpBranchConditional, cond, then, merge);
DO(c, OpLabel, then);
return merge;
}
static void if_end(Cx *c, uint32_t merge) {
DO(c, OpBranch, merge);
DO(c, OpLabel, merge);
}
// for (i = start; i < limit; i += step) { ...the instructions emitted until loop_end }; counter: a Function variable
typedef struct { uint32_t head, cont, merge, i, counter, step; } Loop;
static Loop loop_begin(Cx *c, uint32_t counter, uint32_t start, uint32_t limit, uint32_t step) {
Loop l = {nid(c), nid(c), nid(c), 0, counter, step};
uint32_t cond = nid(c), body = nid(c);
DO(c, OpStore, counter, start);
DO(c, OpBranch, l.head);
DO(c, OpLabel, l.head);
DO(c, OpLoopMerge, l.merge, l.cont, 0);
DO(c, OpBranch, cond);
DO(c, OpLabel, cond);
l.i = E(c, OpLoad, c->t_uint, counter);
DO(c, OpBranchConditional, E(c, OpULessThan, c->t_bool, l.i, limit), body, l.merge);
DO(c, OpLabel, body);
return l;
}
static void loop_end(Cx *c, const Loop *l) {
DO(c, OpBranch, l->cont);
DO(c, OpLabel, l->cont);
DO(c, OpStore, l->counter, add(c, l->i, l->step));
DO(c, OpBranch, l->head);
DO(c, OpLabel, l->merge);
}
// A storage buffer of 32-bit words at (set 0, binding): its variable. readonly: the module only reads it.
static uint32_t word_buffer(Cx *c, uint32_t binding, int readonly) {
uint32_t arr = nid(c), st = nid(c), ptr = nid(c), var = nid(c);
INS(&c->glob, OpTypeRuntimeArray, arr, c->t_uint);
INS(&c->glob, OpTypeStruct, st, arr);
INS(&c->glob, OpTypePointer, ptr, ScStorageBuffer, st);
INS(&c->glob, OpVariable, ptr, var, ScStorageBuffer);
INS(&c->deco, OpDecorate, arr, DecArrayStride, 4);
INS(&c->deco, OpDecorate, st, DecBlock);
INS(&c->deco, OpMemberDecorate, st, 0, DecOffset, 0);
if (readonly) INS(&c->deco, OpMemberDecorate, st, 0, DecNonWritable);
INS(&c->deco, OpDecorate, var, DecDescriptorSet, 0);
INS(&c->deco, OpDecorate, var, DecBinding, binding);
return var;
}
static uint32_t buf_ptr(Cx *c, uint32_t var, uint32_t index) { return E(c, OpAccessChain, c->p_sb_uint, var, K(c, 0), index); }
static uint32_t buf_load(Cx *c, uint32_t var, uint32_t index) { return E(c, OpLoad, c->t_uint, buf_ptr(c, var, index)); }
// ---- a translated task or mesh module to its compute module
typedef struct { uint32_t at, op, len; } In; // an instruction: its first word's index, opcode, word count
static int id_in(const uint32_t *set, int n, uint32_t id) {
for (int i = 0; i < n; i++)
if (set[i] == id) return 1;
return 0;
}
static int is_decl(uint32_t op) { // a type, constant or global variable declaration
return (op >= OpTypeVoid && op <= 39) || (op >= 41 && op <= 52) || op == OpVariable || op == 1;
}
static uint32_t *convert(const uint32_t *w, uint64_t words, const MeshFbInfo *m, uint32_t binding, uint32_t task_words,
uint64_t *out_words, char *err, size_t cap) {
In *in = NULL;
uint32_t *removed = NULL, *gvars = NULL;
Wb out = {0};
Cx c = {0};
#define FAIL(...) do { fail(err, cap, __VA_ARGS__); goto bad; } while (0)
if (words < 6 || w[0] != 0x07230203) return fail(err, cap, "not a SPIR-V module"), NULL;
if (w[1] < 0x00010400) return fail(err, cap, "SPIR-V %u.%u, 1.4 expected", w[1] >> 16 & 0xff, w[1] >> 8 & 0xff), NULL;
size_t n = 0, ncap = 0;
for (uint64_t p = 5; p < words;) {
uint32_t len = w[p] >> 16, op = w[p] & 0xffff;
if (!len || p + len > words) return fail(err, cap, "malformed SPIR-V"), NULL;
if (n == ncap) {
ncap = ncap ? ncap * 2 : 4096;
In *g = realloc(in, ncap * sizeof *in);
if (!g) FAIL("out of memory");
in = g;
}
in[n++] = (In){(uint32_t)p, op, len};
p += len;
}
c.bound = w[3];
int mesh = m->stage == 7, n_removed = 0, n_gvars = 0, n_entry = 0, n_shared = 0;
removed = malloc((n + 1) * 4), gvars = malloc((n + 16) * 4);
if (!removed || !gvars) FAIL("out of memory");
uint32_t old_entry = 0, model = 0, shared_var = 0, shared_type = 0, wg_var = 0, lii_var = 0, first_fn = 0, first_decl = 0;
// pass 1: what the module has
for (size_t i = 0; i < n; i++) {
const uint32_t *a = w + in[i].at + 1;
uint32_t op = in[i].op, len = in[i].len;
if (op == OpEntryPoint) n_entry++, model = a[0], old_entry = a[1];
else if (op == OpTypeVoid) c.t_void = a[0];
else if (op == OpTypeBool) c.t_bool = a[0];
else if (op == OpTypeInt && a[1] == 32 && a[2] == 0) c.t_uint = a[0];
else if (op == OpTypeVector && a[1] == c.t_uint && c.t_uint && a[2] == 3) c.t_v3uint = a[0];
else if (op == OpTypeFunction && len == 3 && a[1] == c.t_void && c.t_void) c.t_fnvoid = a[0];
else if (op == OpTypePointer && (a[1] == ScOutput || a[1] == ScTaskPayload)) removed[n_removed++] = a[0];
else if (op == OpVariable && !first_fn) {
if (a[2] == ScOutput || a[2] == ScTaskPayload) removed[n_removed++] = a[1];
else gvars[n_gvars++] = a[1];
if (a[2] == ScWorkgroup) n_shared++, shared_var = a[1], shared_type = a[0];
} else if (op == OpFunction && !first_fn) first_fn = (uint32_t)i;
if (!first_decl && !first_fn && is_decl(op)) first_decl = (uint32_t)i;
}
if (n_entry != 1 || model != (uint32_t)(mesh ? ModelMesh : ModelTask)) FAIL("the entry point is not a %s one", mesh ? "mesh" : "task");
if (n_shared != 1) FAIL("%d workgroup variables (one shared memory array expected)", n_shared);
if (!c.t_void || !c.t_bool || !c.t_uint || !c.t_fnvoid || !first_fn || !first_decl) FAIL("a basic type or the functions are missing");
{ // the shared memory array holds the layout this module is asked to copy
uint32_t arr = 0, len_id = 0, len = 0;
for (size_t i = 0; i < n; i++) {
const uint32_t *a = w + in[i].at + 1;
if (in[i].op == OpTypePointer && a[0] == shared_type) arr = a[2];
}
for (size_t i = 0; i < n; i++) {
const uint32_t *a = w + in[i].at + 1;
if (in[i].op == OpTypeArray && a[0] == arr && a[1] == c.t_uint) len_id = a[2];
}
for (size_t i = 0; i < n; i++) {
const uint32_t *a = w + in[i].at + 1;
if (in[i].op == OpConstant && a[1] == len_id && len_id) len = a[2];
}
if (len * 4 < m->shared_bytes) FAIL("shared memory of %u bytes, the program's layout needs %u", len * 4, m->shared_bytes);
}
for (size_t i = 0; i < n; i++) {
const uint32_t *a = w + in[i].at + 1;
if (in[i].op != OpDecorate || in[i].len < 4 || a[1] != DecBuiltIn || id_in(removed, n_removed, a[0])) continue;
if (a[2] == BiWorkgroupId) wg_var = a[0];
if (a[2] == BiLocalInvocationIndex) lii_var = a[0];
}
// the old entry function, and the function it calls: the translator's main
size_t e0 = 0, e1 = 0;
uint32_t main_fn = 0, n_calls = 0;
for (size_t i = first_fn; i < n; i++)
if (in[i].op == OpFunction && w[in[i].at + 2] == old_entry) {
e0 = i;
for (e1 = i; e1 < n && in[e1].op != OpFunctionEnd; e1++)
if (in[e1].op == OpFunctionCall) n_calls++, main_fn = w[in[e1].at + 3];
break;
}
if (!e0 || e1 >= n || n_calls != 1) FAIL("the entry function calls %u functions (one expected)", n_calls);
// ---- new types, variables and the entry function
if (!c.t_v3uint) c.t_v3uint = nid(&c), INS(&c.glob, OpTypeVector, c.t_v3uint, c.t_uint, 3);
c.p_fn_uint = nid(&c), INS(&c.glob, OpTypePointer, c.p_fn_uint, ScFunction, c.t_uint);
c.p_sb_uint = nid(&c), INS(&c.glob, OpTypePointer, c.p_sb_uint, ScStorageBuffer, c.t_uint);
uint32_t p_shared = nid(&c);
INS(&c.glob, OpTypePointer, p_shared, ScWorkgroup, c.t_uint);
if (!lii_var) {
uint32_t pt = nid(&c);
lii_var = nid(&c);
INS(&c.glob, OpTypePointer, pt, ScInput, c.t_uint);
INS(&c.glob, OpVariable, pt, lii_var, ScInput);
INS(&c.deco, OpDecorate, lii_var, DecBuiltIn, BiLocalInvocationIndex);
gvars[n_gvars++] = lii_var;
}
if (!wg_var) {
uint32_t pt = nid(&c);
wg_var = nid(&c);
INS(&c.glob, OpTypePointer, pt, ScInput, c.t_v3uint);
INS(&c.glob, OpVariable, pt, wg_var, ScInput);
INS(&c.deco, OpDecorate, wg_var, DecBuiltIn, BiWorkgroupId);
gvars[n_gvars++] = wg_var;
}
uint32_t fake_wg = 0, p_priv_uint = 0;
if (mesh) { // the workgroup id main sees (see the top)
uint32_t pt = nid(&c);
fake_wg = nid(&c), p_priv_uint = nid(&c);
INS(&c.glob, OpTypePointer, pt, ScPrivate, c.t_v3uint);
INS(&c.glob, OpTypePointer, p_priv_uint, ScPrivate, c.t_uint);
INS(&c.glob, OpVariable, pt, fake_wg, ScPrivate);
gvars[n_gvars++] = fake_wg;
}
uint32_t task_buf = 0, mesh_buf = 0, list_buf = 0;
if (!mesh || m->has_task) task_buf = word_buffer(&c, binding, mesh), gvars[n_gvars++] = task_buf;
if (mesh) mesh_buf = word_buffer(&c, binding + 1, 0), gvars[n_gvars++] = mesh_buf;
if (mesh && m->has_task) list_buf = word_buffer(&c, binding + 2, 1), gvars[n_gvars++] = list_buf;
uint32_t threads = K(&c, m->threads), k0 = K(&c, 0), k1 = K(&c, 1);
uint32_t entry = nid(&c);
DO(&c, OpFunction, c.t_void, entry, 0, c.t_fnvoid);
DO(&c, OpLabel, nid(&c));
uint32_t counter = nid(&c);
DO(&c, OpVariable, c.p_fn_uint, counter, ScFunction);
uint32_t tid = E(&c, OpLoad, c.t_uint, lii_var);
uint32_t g = E(&c, OpCompositeExtract, c.t_uint, E(&c, OpLoad, c.t_v3uint, wg_var), 0);
uint32_t first = E(&c, OpULessThan, c.t_bool, tid, k1); // the workgroup's first thread
#define SHARED(index) E(&c, OpAccessChain, p_shared, shared_var, index)
#define BARRIER() DO(&c, OpControlBarrier, K(&c, 2), K(&c, 2), K(&c, 0x108)) /* Workgroup, AcquireRelease | WorkgroupMemory */
if (!mesh) {
uint32_t pw = m->payload_bytes / 4;
uint32_t at = mul(&c, g, K(&c, pw));
uint32_t room = E(&c, OpULessThanEqual, c.t_bool, add(&c, at, K(&c, pw)), E(&c, OpArrayLength, c.t_uint, task_buf, 0));
// the mesh workgroup count starts at 0: a task that culls everything never writes it
uint32_t mg = if_begin(&c, first);
DO(&c, OpStore, SHARED(K(&c, (m->pay + 4) / 4)), k0);
if_end(&c, mg);
BARRIER();
E(&c, OpFunctionCall, c.t_void, main_fn);
BARRIER();
mg = if_begin(&c, room);
Loop l = loop_begin(&c, counter, tid, K(&c, pw), threads);
DO(&c, OpStore, buf_ptr(&c, task_buf, add(&c, at, l.i)), E(&c, OpLoad, c.t_uint, SHARED(add(&c, l.i, K(&c, m->pay / 4)))));
loop_end(&c, &l);
if_end(&c, mg);
} else {
uint32_t slot_words = meshfb_slot_words(m), count_word = K(&c, m->map / 4);
uint32_t j = g, task_at = 0;
if (m->has_task) {
uint32_t e = buf_load(&c, list_buf, g);
task_at = mul(&c, E(&c, OpBitwiseAnd, c.t_uint, e, K(&c, (1u << MESHFB_LIST_TASK_BITS) - 1)), K(&c, task_words));
j = E(&c, OpShiftRightLogical, c.t_uint, e, K(&c, MESHFB_LIST_TASK_BITS));
}
DO(&c, OpStore, fake_wg, E(&c, OpCompositeConstruct, c.t_v3uint, j, k0, k0));
uint32_t at = mul(&c, g, K(&c, slot_words));
uint32_t room = E(&c, OpULessThanEqual, c.t_bool, add(&c, at, K(&c, slot_words)), E(&c, OpArrayLength, c.t_uint, mesh_buf, 0));
// The whole stage buffer starts at 0: the vertex lines, the primitive count and the vertex indices. Workgroup
// memory is undefined at the start, a mesh program need not write every vertex it has room for or every index
// byte (nor its count when it emits nothing), and the slot below is copied whole and drawn: what an earlier
// workgroup left there became vertices and primitives. The payload area after it is copied in next.
Loop z = loop_begin(&c, counter, tid, K(&c, slot_words), threads);
DO(&c, OpStore, SHARED(z.i), k0);
loop_end(&c, &z);
uint32_t mg;
if (m->has_task) {
uint32_t pw = m->payload_bytes / 4;
if (pw > task_words) pw = task_words;
Loop l = loop_begin(&c, counter, tid, K(&c, pw), threads);
DO(&c, OpStore, SHARED(add(&c, l.i, K(&c, m->pay / 4))), buf_load(&c, task_buf, add(&c, task_at, l.i)));
loop_end(&c, &l);
}
BARRIER();
E(&c, OpFunctionCall, c.t_void, main_fn);
BARRIER();
uint32_t count = E(&c, OpLoad, c.t_uint, SHARED(count_word));
mg = if_begin(&c, room);
{ // a slot without primitives gets its count alone
uint32_t some = E(&c, OpINotEqual, c.t_bool, count, k0), then = nid(&c), other = nid(&c), merge = nid(&c);
DO(&c, OpSelectionMerge, merge, 0);
DO(&c, OpBranchConditional, some, then, other);
DO(&c, OpLabel, then);
Loop l = loop_begin(&c, counter, tid, K(&c, slot_words), threads);
DO(&c, OpStore, buf_ptr(&c, mesh_buf, add(&c, at, l.i)), E(&c, OpLoad, c.t_uint, SHARED(l.i)));
loop_end(&c, &l);
DO(&c, OpBranch, merge);
DO(&c, OpLabel, other);
uint32_t m2 = if_begin(&c, first);
DO(&c, OpStore, buf_ptr(&c, mesh_buf, add(&c, at, count_word)), k0);
if_end(&c, m2);
DO(&c, OpBranch, merge);
DO(&c, OpLabel, merge);
}
if_end(&c, mg);
}
wb_ins(&c.fn, OpReturn, NULL, 0);
wb_ins(&c.fn, OpFunctionEnd, NULL, 0);
// ---- pass 2: the module again
uint32_t head[5] = {w[0], w[1], w[2], 0, 0};
wb_put(&out, head, 5);
for (size_t i = 0; i < n; i++) {
const uint32_t *a = w + in[i].at + 1;
uint32_t op = in[i].op, len = in[i].len;
if (i == first_decl) wb_put(&out, c.deco.w, c.deco.n);
if (i == first_fn) wb_put(&out, c.glob.w, c.glob.n);
if (i >= e0 && i <= e1) continue; // the old entry function
if (op == OpCapability && a[0] == CapMeshShading) continue;
if (op == OpExtension && len >= 6 && !memcmp(a, "SPV_EXT_mesh_shader", 20)) continue;
if (op == OpExecutionMode && a[1] != ModeLocalSize) continue;
if ((op == OpName || op == OpDecorate || op == OpDecorateId || op == OpDecorateString) && id_in(removed, n_removed, a[0])) continue;
if ((op == OpTypePointer && id_in(removed, n_removed, a[0])) || (op == OpVariable && id_in(removed, n_removed, a[1]))) continue;
if (op == OpEntryPoint) { // every global variable is its interface (SPIR-V 1.4)
uint32_t ep[4] = {(uint32_t)((n_gvars + 5) << 16) | OpEntryPoint, ModelGLCompute, entry, 0x6e69616d /* "main" */};
uint32_t zero = 0;
wb_put(&out, ep, 4), wb_put(&out, &zero, 1);
wb_put(&out, gvars, (size_t)n_gvars);
continue;
}
size_t at = out.n;
wb_put(&out, w + in[i].at, len);
if (out.oom) break;
uint32_t *o = out.w + at + 1;
if (op == OpExecutionMode) o[0] = entry;
if (mesh && i > first_fn) { // main and what it calls read the workgroup id from the private variable
if ((op == OpAccessChain || op == OpInBoundsAccessChain) && a[2] == wg_var) {
uint32_t pointee = 0;
for (size_t t = first_decl; t < first_fn; t++)
if (in[t].op == OpTypePointer && w[in[t].at + 1] == a[0]) pointee = w[in[t].at + 3];
if (pointee != c.t_uint || len != 5) FAIL("the workgroup id is read in a way the fallback does not know");
o[0] = p_priv_uint, o[2] = fake_wg;
} else if (op == OpLoad && a[2] == wg_var) {
o[2] = fake_wg;
} else if ((op == OpStore && a[0] == wg_var) || (op == OpCopyMemory && (a[0] == wg_var || a[1] == wg_var)) ||
((op == OpPtrAccessChain || op == OpCopyObject) && a[2] == wg_var)) {
FAIL("the workgroup id is read in a way the fallback does not know");
} else if (op == OpFunctionCall || op == OpSelect || op == OpPhi) {
for (uint32_t k = 2; k + 1 < len; k++)
if (a[k] == wg_var) FAIL("the workgroup id is passed on as a pointer");
}
}
}
wb_put(&out, c.fn.w, c.fn.n);
if (out.oom || c.deco.oom || c.glob.oom || c.fn.oom) FAIL("out of memory");
out.w[3] = c.bound;
*out_words = out.n;
free(in), free(removed), free(gvars), free(c.deco.w), free(c.glob.w), free(c.fn.w);
return out.w;
bad:
free(in), free(removed), free(gvars), free(c.deco.w), free(c.glob.w), free(c.fn.w), free(out.w);
return NULL;
#undef FAIL
#undef SHARED
#undef BARRIER
}
uint32_t *meshfb_task_cs(const uint32_t *spirv, uint64_t words, const MeshFbInfo *m, uint32_t binding, uint64_t *out_words,
char *err, size_t cap) {
if (m->stage != 6) return fail(err, cap, "not a task program"), NULL;
return convert(spirv, words, m, binding, 0, out_words, err, cap);
}
uint32_t *meshfb_mesh_cs(const uint32_t *spirv, uint64_t words, const MeshFbInfo *m, uint32_t binding,
uint32_t task_payload_words, uint64_t *out_words, char *err, size_t cap) {
if (m->stage != 7) return fail(err, cap, "not a mesh program"), NULL;
if (m->has_task && !task_payload_words) return fail(err, cap, "a mesh program after a task stage without its payload size"), NULL;
return convert(spirv, words, m, binding, task_payload_words, out_words, err, cap);
}
// ---- the vertex module
uint32_t *meshfb_vs(const MeshFbInfo *m, uint32_t out_mask, const uint8_t out_type[32], uint64_t *out_words) {
Cx c = {.bound = 1};
Wb out = {0};
uint32_t iface[40], n_iface = 0;
c.t_void = nid(&c), INS(&c.glob, OpTypeVoid, c.t_void);
c.t_bool = nid(&c), INS(&c.glob, OpTypeBool, c.t_bool);
c.t_uint = nid(&c), INS(&c.glob, OpTypeInt, c.t_uint, 32, 0);
uint32_t t_int = nid(&c), t_float = nid(&c), t_v4[3], p_out[3], t_scalar[3]; // by GFX_TYPE_*: float, sint, uint
INS(&c.glob, OpTypeInt, t_int, 32, 1);
INS(&c.glob, OpTypeFloat, t_float, 32);
t_scalar[0] = t_float, t_scalar[1] = t_int, t_scalar[2] = c.t_uint;
for (int t = 0; t < 3; t++) {
t_v4[t] = nid(&c), INS(&c.glob, OpTypeVector, t_v4[t], t_scalar[t], 4);
p_out[t] = nid(&c), INS(&c.glob, OpTypePointer, p_out[t], ScOutput, t_v4[t]);
}
c.t_fnvoid = nid(&c), INS(&c.glob, OpTypeFunction, c.t_fnvoid, c.t_void);
c.p_sb_uint = nid(&c), INS(&c.glob, OpTypePointer, c.p_sb_uint, ScStorageBuffer, c.t_uint);
uint32_t p_in_int = nid(&c), p_out_float = nid(&c);
INS(&c.glob, OpTypePointer, p_in_int, ScInput, t_int);
INS(&c.glob, OpTypePointer, p_out_float, ScOutput, t_float);
uint32_t vi_var = nid(&c), ii_var = nid(&c), pos_var = nid(&c), size_var = 0, buf, loc_var[32] = {0};
INS(&c.glob, OpVariable, p_in_int, vi_var, ScInput);
INS(&c.glob, OpVariable, p_in_int, ii_var, ScInput);
INS(&c.glob, OpVariable, p_out[0], pos_var, ScOutput);
INS(&c.deco, OpDecorate, vi_var, DecBuiltIn, BiVertexIndex);
INS(&c.deco, OpDecorate, ii_var, DecBuiltIn, BiInstanceIndex);
INS(&c.deco, OpDecorate, pos_var, DecBuiltIn, BiPosition);
INS(&c.deco, OpDecorate, pos_var, DecInvariant);
iface[n_iface++] = vi_var, iface[n_iface++] = ii_var, iface[n_iface++] = pos_var;
if (m->ipp == 1) { // a point list needs its size written
size_var = nid(&c);
INS(&c.glob, OpVariable, p_out_float, size_var, ScOutput);
INS(&c.deco, OpDecorate, size_var, DecBuiltIn, BiPointSize);
iface[n_iface++] = size_var;
}
for (int g = 0; g < 32; g++) {
if (!(out_mask >> g & 1)) continue;
int t = out_type[g] < 3 ? out_type[g] : 0;
loc_var[g] = nid(&c);
INS(&c.glob, OpVariable, p_out[t], loc_var[g], ScOutput);
INS(&c.deco, OpDecorate, loc_var[g], DecLocation, (uint32_t)g);
iface[n_iface++] = loc_var[g];
}
buf = word_buffer(&c, 1, 1);
iface[n_iface++] = buf;
uint32_t f0 = nid(&c), f1 = nid(&c), fout = nid(&c), null_v4[3], away = nid(&c);
INS(&c.glob, OpConstant, t_float, f0, 0);
INS(&c.glob, OpConstant, t_float, f1, 0x3f800000);
INS(&c.glob, OpConstant, t_float, fout, 0xc1800000); // -16: outside the clip volume, for a point of no primitive
for (int t = 0; t < 3; t++) null_v4[t] = nid(&c), INS(&c.glob, OpConstantNull, t_v4[t], null_v4[t]);
INS(&c.glob, OpConstantComposite, t_v4[0], away, fout, fout, f0, f1);
uint32_t slot_words = meshfb_slot_words(m), k0 = K(&c, 0), entry = nid(&c);
DO(&c, OpFunction, c.t_void, entry, 0, c.t_fnvoid);
DO(&c, OpLabel, nid(&c));
uint32_t vi = E(&c, OpBitcast, c.t_uint, E(&c, OpLoad, t_int, vi_var));
uint32_t slot = E(&c, OpBitcast, c.t_uint, E(&c, OpLoad, t_int, ii_var));
uint32_t prim = m->ipp == 1 ? vi : E(&c, OpUDiv, c.t_uint, vi, K(&c, m->ipp));
uint32_t corner = m->ipp == 1 ? k0 : E(&c, OpISub, c.t_uint, vi, mul(&c, prim, K(&c, m->ipp)));
uint32_t base = mul(&c, slot, K(&c, slot_words));
// a slot past the buffer (more workgroups than it has room for) draws nothing
uint32_t valid = E(&c, OpULessThanEqual, c.t_bool, add(&c, base, K(&c, slot_words)), E(&c, OpArrayLength, c.t_uint, buf, 0));
uint32_t cw = buf_load(&c, buf, E(&c, OpSelect, c.t_uint, valid, add(&c, base, K(&c, m->map / 4)), k0));
uint32_t count = E(&c, OpSelect, c.t_uint, valid, umin(&c, cw, K(&c, m->maxp)), k0);
uint32_t live = E(&c, OpULessThan, c.t_bool, prim, count), then = nid(&c), other = nid(&c), merge = nid(&c);
DO(&c, OpSelectionMerge, merge, 0);
DO(&c, OpBranchConditional, live, then, other);
DO(&c, OpLabel, then);
{
// byte map + 4 + ipp * prim + corner of the slot is the vertex
uint32_t b = add(&c, add(&c, mul(&c, prim, K(&c, m->ipp)), corner), K(&c, m->map + 4));
uint32_t word = buf_load(&c, buf, add(&c, base, E(&c, OpShiftRightLogical, c.t_uint, b, K(&c, 2))));
uint32_t shift = mul(&c, E(&c, OpBitwiseAnd, c.t_uint, b, K(&c, 3)), K(&c, 8));
uint32_t v = umin(&c, E(&c, OpBitwiseAnd, c.t_uint, E(&c, OpShiftRightLogical, c.t_uint, word, shift), K(&c, 0xff)), K(&c, m->maxv - 1));
// word ((v >> 5) * lines + line) * 32 + (v & 31) of the vertex area
uint32_t at = add(&c, add(&c, mul(&c, E(&c, OpShiftRightLogical, c.t_uint, v, K(&c, 5)), K(&c, m->nlines * 32)),
E(&c, OpBitwiseAnd, c.t_uint, v, K(&c, 31))), add(&c, base, K(&c, m->attr / 4)));
uint32_t pos[4] = {f0, f0, f0, f1}, size = f1, gen[32][4];
memset(gen, 0, sizeof gen);
for (uint32_t l = 0; l < m->nlines; l++) {
uint32_t a = m->lines[l];
int g = a >= 0x80 ? (int)(a - 0x80) / 16 : -1;
if (a < 0x6c || (g >= 0 && !loc_var[g])) continue; // layer and viewport index are not carried; an output nothing reads
uint32_t word_l = buf_load(&c, buf, l ? add(&c, at, K(&c, l * 32)) : at);
if (g >= 0) gen[g][(a - 0x80) % 16 / 4] = word_l;
else if (a == 0x6c) size = E(&c, OpBitcast, t_float, word_l);
else pos[(a - 0x70) / 4] = E(&c, OpBitcast, t_float, word_l);
}
DO(&c, OpStore, pos_var, E(&c, OpCompositeConstruct, t_v4[0], pos[0], pos[1], pos[2], pos[3]));
if (size_var) DO(&c, OpStore, size_var, size);
for (int g = 0; g < 32; g++) {
if (!loc_var[g]) continue;
int t = out_type[g] < 3 ? out_type[g] : 0;
uint32_t comp[4];
for (int k = 0; k < 4; k++) {
uint32_t word_k = gen[g][k] ? gen[g][k] : k0;
comp[k] = t == 2 ? word_k : E(&c, OpBitcast, t_scalar[t], word_k);
}
DO(&c, OpStore, loc_var[g], E(&c, OpCompositeConstruct, t_v4[t], comp[0], comp[1], comp[2], comp[3]));
}
}
DO(&c, OpBranch, merge);
DO(&c, OpLabel, other);
DO(&c, OpStore, pos_var, away);
if (size_var) DO(&c, OpStore, size_var, f1);
for (int g = 0; g < 32; g++)
if (loc_var[g]) DO(&c, OpStore, loc_var[g], null_v4[out_type[g] < 3 ? out_type[g] : 0]);
DO(&c, OpBranch, merge);
DO(&c, OpLabel, merge);
wb_ins(&c.fn, OpReturn, NULL, 0);
wb_ins(&c.fn, OpFunctionEnd, NULL, 0);
uint32_t head[5] = {0x07230203, 0x00010400, 0, c.bound, 0};
wb_put(&out, head, 5);
INS(&out, OpCapability, CapShader);
INS(&out, OpMemoryModel, 0, 1); // Logical, GLSL450
uint32_t ep[5] = {(n_iface + 5) << 16 | OpEntryPoint, ModelVertex, entry, 0x6e69616d /* "main" */, 0};
wb_put(&out, ep, 5);
wb_put(&out, iface, n_iface);
wb_put(&out, c.deco.w, c.deco.n);
wb_put(&out, c.glob.w, c.glob.n);
wb_put(&out, c.fn.w, c.fn.n);
int oom = out.oom || c.deco.oom || c.glob.oom || c.fn.oom;
free(c.deco.w), free(c.glob.w), free(c.fn.w);
if (oom) return free(out.w), NULL;
*out_words = out.n;
return out.w;
}
// ---- reflection text
char *meshfb_res_json(const char *res, uint32_t binding, const int *cb_slots, int n) {
size_t len = strlen(res);
char *out = malloc(len + (size_t)n * 96 + 1), *o = out;
if (!out) return NULL;
const char *sb = strstr(res, "\"sbufs\""), *open = sb ? strchr(sb, '[') : NULL;
if (!open) return free(out), NULL;
memcpy(o, res, (size_t)(open + 1 - res)), o += open + 1 - res;
const char *rest = open + 1;
while (*rest == ' ' || *rest == '\n' || *rest == '\r' || *rest == '\t') rest++;
for (int i = 0; i < n; i++)
o += sprintf(o, "{\"binding\": %u, \"slot\": -1, \"cb_slot\": %d, \"cb_offset\": 0, \"flags\": %d}%s", binding + (uint32_t)i,
cb_slots[i], cb_slots[i] == MESHFB_CB_LIST ? 0 : 1, i + 1 < n || *rest != ']' ? "," : "");
strcpy(o, open + 1);
char *st = strstr(out, "\"stage\"");
st = st ? strchr(st + 7, '"') : NULL;
if (!st || (st[1] != 'm' && st[1] != 't') || st[2] != 's') return free(out), NULL;
st[1] = 'c';
return out;
}