From ff1c1ed87a495d406279cf170e26468cfe8606b4 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:02 +0100 Subject: [PATCH] Mesh emulation: the stages of a pass's emulated draws recorded by stage, with one barrier between stages for all of them. Each draw's indices are now a range of its own and its task records too, reserved by an allocation shader (one invocation, mesh_emu_alloc_spirv) for the run of draw records queued: before the task stages the task records and the index ranges of draws without a task stage, after them the index ranges of the others (mesh workgroups with output, which the task stage now counts in the record, times OutputPrimitives x vertices per primitive). The mesh stage takes its indices from the draw's range with an atomic in its record; vertex slots stay the arena's, as the indices name them. So no draw's stages need another's, and the queue (MEQ) records the fills as the draws come, then at the pre-pass command buffer's end (vk_sync.c pre_end -> vk_exec_mesh_flush): allocation, every task stage, allocation, every mesh stage, a barrier between each. One by one each draw took three barriers and copied the arena's counters after the draw before it: an RX 580 spent 5.5 ms of a frame's GPU time on 69 draws' stages in Kokiri Forest. Co-Authored-By: Claude Opus 5.5 --- runtime/mesh_emu_spirv.c | 142 +++++++++++++++++++++++++++++++++- runtime/mesh_emu_spirv.h | 19 +++++ runtime/vk.h | 3 + runtime/vk_exec_draw.c | 160 +++++++++++++++++++++++++++++++++++++++ runtime/vk_sync.c | 3 + 5 files changed, 326 insertions(+), 1 deletion(-) diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c index 280b1f2e..114d8292 100644 --- a/runtime/mesh_emu_spirv.c +++ b/runtime/mesh_emu_spirv.c @@ -254,8 +254,13 @@ static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int uint32_t mz; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); { + // indices from the draw's own range (reserved by the allocation shader), vertex slots from the arena's + uint32_t drw = pc_u32(e, 1), vcap = pc_u32(e, 5); + uint32_t icap = word_load(e, base, add(e, drw, cu(e, MESH_EMU_R_ICAP))); + uint32_t ib = atomic(e, OpAtomicIAdd, base, add(e, drw, cu(e, MESH_EMU_R_ICUR)), isz); uint32_t vb = atomic(e, OpAtomicIAdd, base, cu(e, 0), vsz); uint32_t iok = E2(e, OpULessThanEqual, m->t_bool, add(e, ib, isz), icap); + ib = add(e, word_load(e, base, add(e, drw, cu(e, 2))), ib); // relative to the area, as firstIndex uint32_t vok = E2(e, OpULessThanEqual, m->t_bool, add(e, vb, vsz), vcap); uint32_t mi; if_begin(e, iok, &mi); @@ -358,6 +363,7 @@ static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint uint32_t m2; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &m2); word_store(e, base, recw, nc); + atomic(e, OpAtomicIAdd, base, add(e, drw, cu(e, MESH_EMU_R_GROUPS)), nc); // for the index range if_end(e, m2); } if_end(e, m1); @@ -954,4 +960,138 @@ uint32_t *mesh_emu_vs(const MeshEmuInfo *mi, uint32_t need, const uint8_t types[ free(t.w), free(an.w), free(c.w); if (bad) { free(b.w); return NULL; } *out_n = b.n; - return b.w; \ No newline at end of file + return b.w; +#undef N +} + +// The allocation shader (mesh_emu_spirv.h): glslangValidator -V -g0 --target-env vulkan1.2, spirv-opt -O --strip-debug, +// spirv-val clean. Its source: +// #version 460 +// #extension GL_EXT_buffer_reference : require +// #extension GL_EXT_buffer_reference_uvec2 : require +// layout(local_size_x = 1) in; +// layout(buffer_reference, std430, buffer_reference_align = 4) buffer Words { uint w[]; }; +// layout(push_constant) uniform PC { uvec2 addr; uint first; uint n; uint mode; uint icap; uint trecs; uint pad; } pc; +// void main() { +// Words a = Words(pc.addr); +// for (uint d = 0u; d < pc.n; d++) { +// uint r = 64u + ((pc.first + d) % 4096u) * 16u; +// bool task = a.w[r + 14u] != 0u; +// if (pc.mode == 0u && task) { +// uint tb = a.w[2], k = a.w[r + 10u]; +// a.w[r + 8u] = tb; +// a.w[2] = k > pc.trecs ? pc.trecs + tb : tb + k; +// } else if ((pc.mode == 0u) != task) { +// uint g = pc.mode == 0u ? a.w[r + 10u] : a.w[r + 9u], per = a.w[r + 13u], ib = a.w[1]; +// uint ni = per != 0u && g > pc.icap / per ? pc.icap + 1u : g * per; +// a.w[r + 2u] = ib; +// a.w[r + 12u] = ib >= pc.icap ? 0u : min(ni, pc.icap - ib); +// a.w[1] = min(ib + ni, 0x7fffffffu); +// } +// } +// } +static const uint32_t alloc_words[] = { + 0x07230203, 0x00010500, 0x0008000b, 0x000000df, 0x00000000, 0x00020011, 0x00000001, 0x00020011, + 0x000014e3, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, + 0x000014e4, 0x00000001, 0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000f, + 0x00060010, 0x00000004, 0x00000011, 0x00000001, 0x00000001, 0x00000001, 0x00040047, 0x00000008, + 0x00000006, 0x00000004, 0x00030047, 0x00000009, 0x00000002, 0x00050048, 0x00000009, 0x00000000, + 0x00000023, 0x00000000, 0x00030047, 0x0000000d, 0x00000002, 0x00050048, 0x0000000d, 0x00000000, + 0x00000023, 0x00000000, 0x00050048, 0x0000000d, 0x00000001, 0x00000023, 0x00000008, 0x00050048, + 0x0000000d, 0x00000002, 0x00000023, 0x0000000c, 0x00050048, 0x0000000d, 0x00000003, 0x00000023, + 0x00000010, 0x00050048, 0x0000000d, 0x00000004, 0x00000023, 0x00000014, 0x00050048, 0x0000000d, + 0x00000005, 0x00000023, 0x00000018, 0x00050048, 0x0000000d, 0x00000006, 0x00000023, 0x0000001c, + 0x00040047, 0x000000cb, 0x0000000b, 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, + 0x00000002, 0x00030027, 0x00000006, 0x000014e5, 0x00040015, 0x00000007, 0x00000020, 0x00000000, + 0x0003001d, 0x00000008, 0x00000007, 0x0003001e, 0x00000009, 0x00000008, 0x00040020, 0x00000006, + 0x000014e5, 0x00000009, 0x00040017, 0x0000000c, 0x00000007, 0x00000002, 0x0009001e, 0x0000000d, + 0x0000000c, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00040020, + 0x0000000e, 0x00000009, 0x0000000d, 0x0004003b, 0x0000000e, 0x0000000f, 0x00000009, 0x00040015, + 0x00000010, 0x00000020, 0x00000001, 0x0004002b, 0x00000010, 0x00000011, 0x00000000, 0x00040020, + 0x00000012, 0x00000009, 0x0000000c, 0x0004002b, 0x00000007, 0x00000018, 0x00000000, 0x0004002b, + 0x00000010, 0x0000001f, 0x00000002, 0x00040020, 0x00000020, 0x00000009, 0x00000007, 0x00020014, + 0x00000023, 0x0004002b, 0x00000010, 0x00000027, 0x00000001, 0x0004002b, 0x00000007, 0x0000002c, + 0x00001000, 0x0004002b, 0x00000007, 0x0000002e, 0x00000010, 0x00040020, 0x00000037, 0x000014e5, + 0x00000007, 0x0004002b, 0x00000010, 0x0000003b, 0x00000003, 0x0004002b, 0x00000010, 0x00000056, + 0x00000005, 0x0004002b, 0x00000010, 0x00000094, 0x00000004, 0x0004002b, 0x00000007, 0x000000a0, + 0x00000001, 0x0004002b, 0x00000007, 0x000000c5, 0x7fffffff, 0x00040017, 0x000000ca, 0x00000007, + 0x00000003, 0x0006002c, 0x000000ca, 0x000000cb, 0x000000a0, 0x000000a0, 0x000000a0, 0x0004002b, + 0x00000007, 0x000000d8, 0x0000004e, 0x0004002b, 0x00000007, 0x000000d9, 0x00000049, 0x0004002b, + 0x00000007, 0x000000da, 0x0000004a, 0x0004002b, 0x00000007, 0x000000db, 0x0000004d, 0x0004002b, + 0x00000007, 0x000000dc, 0x00000042, 0x0004002b, 0x00000007, 0x000000dd, 0x0000004c, 0x0004002b, + 0x00000007, 0x000000de, 0x00000048, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, + 0x000200f8, 0x00000005, 0x00050041, 0x00000012, 0x00000013, 0x0000000f, 0x00000011, 0x0004003d, + 0x0000000c, 0x00000014, 0x00000013, 0x0004007c, 0x00000006, 0x00000015, 0x00000014, 0x000200f9, + 0x00000019, 0x000200f8, 0x00000019, 0x000700f5, 0x00000007, 0x000000cc, 0x00000018, 0x00000005, + 0x000000c9, 0x0000001c, 0x00050041, 0x00000020, 0x00000021, 0x0000000f, 0x0000001f, 0x0004003d, + 0x00000007, 0x00000022, 0x00000021, 0x000500b0, 0x00000023, 0x00000024, 0x000000cc, 0x00000022, + 0x000400f6, 0x0000001b, 0x0000001c, 0x00000000, 0x000400fa, 0x00000024, 0x0000001a, 0x0000001b, + 0x000200f8, 0x0000001a, 0x00050041, 0x00000020, 0x00000028, 0x0000000f, 0x00000027, 0x0004003d, + 0x00000007, 0x00000029, 0x00000028, 0x00050080, 0x00000007, 0x0000002b, 0x00000029, 0x000000cc, + 0x00050089, 0x00000007, 0x0000002d, 0x0000002b, 0x0000002c, 0x00050084, 0x00000007, 0x0000002f, + 0x0000002d, 0x0000002e, 0x00050080, 0x00000007, 0x00000036, 0x0000002f, 0x000000d8, 0x00060041, + 0x00000037, 0x00000038, 0x00000015, 0x00000011, 0x00000036, 0x0006003d, 0x00000007, 0x00000039, + 0x00000038, 0x00000002, 0x00000004, 0x000500ab, 0x00000023, 0x0000003a, 0x00000039, 0x00000018, + 0x00050041, 0x00000020, 0x0000003c, 0x0000000f, 0x0000003b, 0x0004003d, 0x00000007, 0x0000003d, + 0x0000003c, 0x000500aa, 0x00000023, 0x0000003e, 0x0000003d, 0x00000018, 0x000500a7, 0x00000023, + 0x00000040, 0x0000003e, 0x0000003a, 0x000300f7, 0x00000042, 0x00000000, 0x000400fa, 0x00000040, + 0x00000041, 0x00000067, 0x000200f8, 0x00000041, 0x00060041, 0x00000037, 0x00000045, 0x00000015, + 0x00000011, 0x0000001f, 0x0006003d, 0x00000007, 0x00000046, 0x00000045, 0x00000002, 0x00000004, + 0x00050080, 0x00000007, 0x0000004b, 0x0000002f, 0x000000da, 0x00060041, 0x00000037, 0x0000004c, + 0x00000015, 0x00000011, 0x0000004b, 0x0006003d, 0x00000007, 0x0000004d, 0x0000004c, 0x00000002, + 0x00000004, 0x00050080, 0x00000007, 0x00000051, 0x0000002f, 0x000000de, 0x00060041, 0x00000037, + 0x00000053, 0x00000015, 0x00000011, 0x00000051, 0x0005003e, 0x00000053, 0x00000046, 0x00000002, + 0x00000004, 0x00050041, 0x00000020, 0x00000057, 0x0000000f, 0x00000056, 0x0004003d, 0x00000007, + 0x00000058, 0x00000057, 0x000500ac, 0x00000023, 0x00000059, 0x0000004d, 0x00000058, 0x000300f7, + 0x0000005c, 0x00000000, 0x000400fa, 0x00000059, 0x0000005b, 0x00000061, 0x000200f8, 0x0000005b, + 0x00050080, 0x00000007, 0x00000060, 0x00000058, 0x00000046, 0x000200f9, 0x0000005c, 0x000200f8, + 0x00000061, 0x00050080, 0x00000007, 0x00000064, 0x00000046, 0x0000004d, 0x000200f9, 0x0000005c, + 0x000200f8, 0x0000005c, 0x000700f5, 0x00000007, 0x000000d0, 0x00000060, 0x0000005b, 0x00000064, + 0x00000061, 0x0005003e, 0x00000045, 0x000000d0, 0x00000002, 0x00000004, 0x000200f9, 0x00000042, + 0x000200f8, 0x00000067, 0x000500a5, 0x00000023, 0x0000006c, 0x0000003e, 0x0000003a, 0x000300f7, + 0x0000006e, 0x00000000, 0x000400fa, 0x0000006c, 0x0000006d, 0x0000006e, 0x000200f8, 0x0000006d, + 0x000300f7, 0x00000075, 0x00000000, 0x000400fa, 0x0000003e, 0x00000074, 0x0000007b, 0x000200f8, + 0x00000074, 0x00050080, 0x00000007, 0x00000078, 0x0000002f, 0x000000da, 0x00060041, 0x00000037, + 0x00000079, 0x00000015, 0x00000011, 0x00000078, 0x0006003d, 0x00000007, 0x0000007a, 0x00000079, + 0x00000002, 0x00000004, 0x000200f9, 0x00000075, 0x000200f8, 0x0000007b, 0x00050080, 0x00000007, + 0x0000007f, 0x0000002f, 0x000000d9, 0x00060041, 0x00000037, 0x00000080, 0x00000015, 0x00000011, + 0x0000007f, 0x0006003d, 0x00000007, 0x00000081, 0x00000080, 0x00000002, 0x00000004, 0x000200f9, + 0x00000075, 0x000200f8, 0x00000075, 0x000700f5, 0x00000007, 0x000000cd, 0x0000007a, 0x00000074, + 0x00000081, 0x0000007b, 0x00050080, 0x00000007, 0x00000087, 0x0000002f, 0x000000db, 0x00060041, + 0x00000037, 0x00000088, 0x00000015, 0x00000011, 0x00000087, 0x0006003d, 0x00000007, 0x00000089, + 0x00000088, 0x00000002, 0x00000004, 0x00060041, 0x00000037, 0x0000008c, 0x00000015, 0x00000011, + 0x00000027, 0x0006003d, 0x00000007, 0x0000008d, 0x0000008c, 0x00000002, 0x00000004, 0x000500ab, + 0x00000023, 0x00000090, 0x00000089, 0x00000018, 0x000300f7, 0x00000092, 0x00000000, 0x000400fa, + 0x00000090, 0x00000091, 0x00000092, 0x000200f8, 0x00000091, 0x00050041, 0x00000020, 0x00000095, + 0x0000000f, 0x00000094, 0x0004003d, 0x00000007, 0x00000096, 0x00000095, 0x00050086, 0x00000007, + 0x00000098, 0x00000096, 0x00000089, 0x000500ac, 0x00000023, 0x00000099, 0x000000cd, 0x00000098, + 0x000200f9, 0x00000092, 0x000200f8, 0x00000092, 0x000700f5, 0x00000023, 0x0000009a, 0x00000090, + 0x00000075, 0x00000099, 0x00000091, 0x000300f7, 0x0000009d, 0x00000000, 0x000400fa, 0x0000009a, + 0x0000009c, 0x000000a2, 0x000200f8, 0x0000009c, 0x00050041, 0x00000020, 0x0000009e, 0x0000000f, + 0x00000094, 0x0004003d, 0x00000007, 0x0000009f, 0x0000009e, 0x00050080, 0x00000007, 0x000000a1, + 0x0000009f, 0x000000a0, 0x000200f9, 0x0000009d, 0x000200f8, 0x000000a2, 0x00050084, 0x00000007, + 0x000000a5, 0x000000cd, 0x00000089, 0x000200f9, 0x0000009d, 0x000200f8, 0x0000009d, 0x000700f5, + 0x00000007, 0x000000ce, 0x000000a1, 0x0000009c, 0x000000a5, 0x000000a2, 0x00050080, 0x00000007, + 0x000000aa, 0x0000002f, 0x000000dc, 0x00060041, 0x00000037, 0x000000ac, 0x00000015, 0x00000011, + 0x000000aa, 0x0005003e, 0x000000ac, 0x0000008d, 0x00000002, 0x00000004, 0x00050080, 0x00000007, + 0x000000b0, 0x0000002f, 0x000000dd, 0x00050041, 0x00000020, 0x000000b2, 0x0000000f, 0x00000094, + 0x0004003d, 0x00000007, 0x000000b3, 0x000000b2, 0x000500ae, 0x00000023, 0x000000b4, 0x0000008d, + 0x000000b3, 0x000300f7, 0x000000b7, 0x00000000, 0x000400fa, 0x000000b4, 0x000000b6, 0x000000b8, + 0x000200f8, 0x000000b6, 0x000200f9, 0x000000b7, 0x000200f8, 0x000000b8, 0x00050082, 0x00000007, + 0x000000bd, 0x000000b3, 0x0000008d, 0x0007000c, 0x00000007, 0x000000be, 0x00000001, 0x00000026, + 0x000000ce, 0x000000bd, 0x000200f9, 0x000000b7, 0x000200f8, 0x000000b7, 0x000700f5, 0x00000007, + 0x000000cf, 0x00000018, 0x000000b6, 0x000000be, 0x000000b8, 0x00060041, 0x00000037, 0x000000c0, + 0x00000015, 0x00000011, 0x000000b0, 0x0005003e, 0x000000c0, 0x000000cf, 0x00000002, 0x00000004, + 0x00050080, 0x00000007, 0x000000c4, 0x0000008d, 0x000000ce, 0x0007000c, 0x00000007, 0x000000c6, + 0x00000001, 0x00000026, 0x000000c4, 0x000000c5, 0x0005003e, 0x0000008c, 0x000000c6, 0x00000002, + 0x00000004, 0x000200f9, 0x0000006e, 0x000200f8, 0x0000006e, 0x000200f9, 0x00000042, 0x000200f8, + 0x00000042, 0x000200f9, 0x0000001c, 0x000200f8, 0x0000001c, 0x00050080, 0x00000007, 0x000000c9, + 0x000000cc, 0x00000027, 0x000200f9, 0x00000019, 0x000200f8, 0x0000001b, 0x000100fd, 0x00010038, +}; +const uint32_t *mesh_emu_alloc_spirv(size_t *n) { + *n = sizeof alloc_words / sizeof *alloc_words; + return alloc_words; +} +_Static_assert(MESH_EMU_DOFF / 4 == 64 && MESH_EMU_DRECS == 4096 && MESH_EMU_DWORDS == 16 && MESH_EMU_R_GROUPS == 9 && + MESH_EMU_R_FIRST == 10 && MESH_EMU_R_ICAP == 12 && MESH_EMU_R_PER == 13 && MESH_EMU_R_TASK == 14, + "the allocation shader's constants"); diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h index 430ca76a..0584e3ae 100644 --- a/runtime/mesh_emu_spirv.h +++ b/runtime/mesh_emu_spirv.h @@ -7,7 +7,13 @@ #include // The arena: one buffer, its address in the push constants of every emulation stage. +// words 0-3: vertex slots used, indices reserved, task records reserved, overflows (the replay zeroes them per +// submission) // MESH_EMU_DOFF: draw records, MESH_EMU_DWORDS words each: DrawIndexedIndirect {indexCount, instanceCount, +// firstIndex, vertexOffset, firstInstance}, mesh dispatch {x, y, z}, task base, then the MESH_EMU_R_* +// words; the replay fills them, the allocation shader (mesh_emu_alloc_spirv) reserves each draw's +// task records and index range, so that the stages of many draws can run together: a draw's +// indices are one range (its DrawIndexedIndirect) whatever the order its mesh workgroups run in // MESH_EMU_TOFF: task records, MESH_EMU_TWORDS words each: mesh workgroup count, then the payload // index area (uint32 indices), vertex area (16-byte slots: position, then each generic output) at offsets the // replay passes in the push constants. @@ -15,6 +21,12 @@ #define MESH_EMU_DWORDS 16u #define MESH_EMU_DRECS 4096u #define MESH_EMU_TOFF (MESH_EMU_DOFF + MESH_EMU_DRECS * MESH_EMU_DWORDS * 4u) +#define MESH_EMU_R_GROUPS 9u // mesh workgroups with output: the task stage adds the counts it emits +#define MESH_EMU_R_FIRST 10u // the first stage's workgroups (the replay: the count, or a copy of the indirect one) +#define MESH_EMU_R_ICUR 11u // indices taken from the draw's range (the mesh stage's atomics) +#define MESH_EMU_R_ICAP 12u // the draw's range: indices reserved (fit in the area) +#define MESH_EMU_R_PER 13u // indices one mesh workgroup may emit (OutputPrimitives x vertices per primitive) +#define MESH_EMU_R_TASK 14u // 1 with a task stage #define MESH_EMU_TWORDS 780u // 1 + payloads of up to 776 words (3072 bytes and the 32 the // translator adds for the payload header since 0.6.0), padded // task records: as many as the push constants say (the replay sizes them, NX2NAVI_MESH_EMU_TASKS); the index area @@ -26,6 +38,13 @@ // submission); word 3 counts mesh workgroups that did not fit. #define MESH_EMU_PC_BYTES 32u +// The allocation shader (GLCompute, one invocation): for the n draw records from ring index first, mode 0 reserves the +// task records of draws with a task stage (MESH_EMU_R_FIRST of them, from word 2) and the index ranges of draws +// without one (MESH_EMU_R_FIRST x MESH_EMU_R_PER, from word 1); mode 1, after the task stages, the index ranges of +// draws with one (MESH_EMU_R_GROUPS x MESH_EMU_R_PER). Push constants, 32 bytes: u64 arena address; uint first, n, +// mode, index capacity, task record capacity, unused. Static words, *n of them. +const uint32_t *mesh_emu_alloc_spirv(size_t *n); + typedef struct MeshEmuInfo { int task; // 1 TaskEXT, 0 MeshEXT uint32_t threads; // LocalSize x diff --git a/runtime/vk.h b/runtime/vk.h index a4f25dc4..fb7d9a4a 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -764,6 +764,9 @@ void vk_sync_worker_end(VkCommandBuffer cb); // The part's pre-pass command buffer: submitted right before the part being recorded (after its mirror prologue), begun // with a full barrier at the first call of a part; VK_NULL_HANDLE when it cannot be made. Replay thread, sub_lock held. VkCommandBuffer vk_sync_pre(void); +// vk_exec_draw.c: records into the pre-pass command buffer, as it ends, the stages of the emulated mesh draws queued +// for it (NX2NAVI_MESH_EMU_BATCH). A weak vk_sync.c version does nothing. +void vk_exec_mesh_flush(VkCommandBuffer cb); // Ends the part being recorded without handing it to the queue (it goes with the next part that does, in order) and // returns the command buffer the replay continues in; VK_NULL_HANDLE when no more parts can wait. Replay thread. VkCommandBuffer vk_sync_cut(void); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index e5b6509f..e6423227 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5829,6 +5829,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { // NX2NAVI_MESH_EMU_TASKS its task records per submission (default 16384, 3 KiB each). Before the counters start over // they are copied to a readback ring; the perf log reports the most a submission asked for against the capacities. #define ME_RB 64 +static struct { VkBuffer buf, rb; VmaAllocation alloc, rb_alloc; uint32_t *rb_map; @@ -5837,8 +5838,29 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { int failed; uint64_t draws, skipped; uint32_t max[4]; + VkPipelineLayout alayout; // the allocation shader's (push constants only) + VkPipeline apipe; } ME; +// The allocation shader's pipeline (mesh_emu_spirv.h mesh_emu_alloc_spirv); 0 when it cannot be made. +static int me_alloc_pipe(void) { + VkDevice dev = vk_ctx()->device; + VkPushConstantRange pcr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 32}; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .pushConstantRangeCount = 1, + .pPushConstantRanges = &pcr}; + if (vkCreatePipelineLayout(dev, &li, NULL, &ME.alayout) != VK_SUCCESS) return ME.alayout = VK_NULL_HANDLE, 0; + size_t n = 0; + const uint32_t *w = mesh_emu_alloc_spirv(&n); + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = n * 4, .pCode = w}; + VkShaderModule m; + if (vkCreateShaderModule(dev, &mi, NULL, &m) != VK_SUCCESS) return 0; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = ME.alayout}; + VkResult r = vkCreateComputePipelines(dev, VK_NULL_HANDLE, 1, &ci, NULL, &ME.apipe); + vkDestroyShaderModule(dev, m, NULL); + if (r != VK_SUCCESS) return ME.apipe = VK_NULL_HANDLE, 0; return 1; } @@ -5858,6 +5880,9 @@ static int me_arena(void) { VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (!me_alloc_pipe()) { + ME.failed = 1; + rt_log("vk: mesh emulation: the allocation shader's pipeline could not be created: mesh draws skipped"); return 0; } if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ME.buf, &ME.alloc, NULL) != VK_SUCCESS) { @@ -5880,6 +5905,8 @@ static int me_arena(void) { if (vmaCreateBuffer(vk_ctx()->vma, &rbi, &rai, &ME.rb, &ME.rb_alloc, &rinfo) == VK_SUCCESS) ME.rb_map = rinfo.pMappedData; else ME.rb = VK_NULL_HANDLE; return 1; +} + // The access masks also carry the generic MEMORY_WRITE / MEMORY_READ bits. SHADER_STORAGE_READ/WRITE exist only in // synchronization2 (bits above 31), and AMD's driver for Polaris (23.10.01, RX 580) did not act on them: the mesh // dispatch that reads the {groups, task workgroups, 1} the task stage had just written read zeros and launched no @@ -5892,9 +5919,101 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla .dstAccessMask = da | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); +} + +// The stages of the emulated draws whose records were filled, waiting to be recorded (me_flush): the draws of a pass +// with NX2NAVI_MESH_EMU_BATCH, which run in the part's pre-pass command buffer when it ends (vk_exec_mesh_flush, from +// vk_sync.c), or the one draw that runs in the part. Recorded by stage, with one barrier between stages for all the +// draws: the fills, the allocation (task records; index ranges of draws without a task stage), every task stage, the +// allocation of the other index ranges, every mesh stage. Each draw's indices are a range of its own and its task +// records too (the allocation shader), so the draws' stages need nothing of each other's. One by one, as before +// 2026-11-11, each draw took three barriers, and its task base and first index were copies of the arena's counters +// after the draw before it: an RX 580 spent 5.5 ms of a frame's GPU time on 69 draws' stages (Kokiri Forest), +// most of it waiting at the barriers with the GPU nearly idle. +typedef struct { + VkPipeline tp, mp; + VkPipelineLayout layout; + VkDescriptorUpdateTemplate tpl_c; + VkDescriptorSet set0, pset; + int push; + uint32_t count, rec, pc[8]; + VkBuffer ib; // the first stage's indirect dispatch (VK_NULL_HANDLE: count) + VkDeviceSize io; + size_t doff; + uint32_t dbytes; +} MeQ; +#define MEQ_MAX 256 +static struct { + MeQ q[MEQ_MAX]; + int n, tasks; + uint8_t *d; // the push descriptors' data of the queued draws + size_t dn, dcap; +} MEQ; + +static void me_bind(VkCommandBuffer cb, const MeQ *q) { + if (q->push) rec_push_tpl(vk_pipeline_push_tpl_fn(), cb, q->tpl_c, q->layout, 0, MEQ.d + q->doff, q->dbytes); + else vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->layout, 0, 1, &q->set0, 0, NULL); + if (q->pset) vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->layout, 2, 1, &q->pset, 0, NULL); + vkCmdPushConstants(cb, q->layout, VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES, q->pc); +} + +static void me_first(VkCommandBuffer cb, const MeQ *q) { + if (q->ib) vkCmdDispatchIndirect(cb, q->ib, q->io); + else vkCmdDispatch(cb, q->count, 1, 1); +} + +static void me_alloc(VkCommandBuffer cb, uint32_t mode) { + uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), MEQ.q[0].rec, (uint32_t)MEQ.n, mode, ME.icap, ME.trecs, 0}; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, ME.apipe); + vkCmdPushConstants(cb, ME.alayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); + vkCmdDispatch(cb, 1, 1, 1); +} + +// The queued draws' stages into cb (the pre-pass command buffer, or the part for a draw that runs there); returns the +// commands recorded. +static int me_flush(VkCommandBuffer cb) { + if (!MEQ.n || !cb) return 0; + const VkAccessFlags2 rw = VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT; + const VkPipelineStageFlags2 cs = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT; + int k = 0; + me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + me_alloc(cb, 0), k += 4; + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + if (MEQ.tasks) { // the task stages wrote the mesh dispatch {max mesh groups, task workgroups, 1} into the draw record + for (int i = 0; i < MEQ.n; i++) { + const MeQ *q = &MEQ.q[i]; + if (!q->tp) continue; + me_bind(cb, q); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->tp); + me_first(cb, q), k += 4; } + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + me_alloc(cb, 1), k += 4; + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); } + for (int i = 0; i < MEQ.n; i++) { + const MeQ *q = &MEQ.q[i]; + me_bind(cb, q); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->mp); + if (q->tp) vkCmdDispatchIndirect(cb, ME.buf, MESH_EMU_DOFF + (VkDeviceSize)q->rec * MESH_EMU_DWORDS * 4 + 20); + else me_first(cb, q); + k += 4; } + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, + VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_2_INDEX_INPUT_BIT | VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT | + cs | VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_2_INDEX_READ_BIT | rw | VK_ACCESS_2_TRANSFER_READ_BIT); + MEQ.n = MEQ.tasks = 0, MEQ.dn = 0; + return k + 4; +} + +// vk.h: the pre-pass command buffer ends (vk_sync.c pre_end): the stages of the draws queued for it go in first. +void vk_exec_mesh_flush(VkCommandBuffer cb) { me_flush(cb); } + // count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from // args: an indirect draw's arguments in game memory (count, 1, 1 copied into a ring entry, which ind then is) static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args) { @@ -5939,12 +6058,16 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const if (batch && ex->in_pass) { if (ex->pre_pass != ex->pass_serial && vk_exec_split_now(ex)) { if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { // the pass again, in the new part + ME.skipped++; + return 0; + } ex->pre_pass = ex->pass_serial; set0 = ex->dc->B.ds_bound[0]; } if (ex->pre_pass == ex->pass_serial) pre = vk_sync_pre(); } if (!pre) vk_exec_barrier(ex); // ends the pass; what the game wrote before (its culling) is visible to the stages + if (!pre && MEQ.n) me_flush(vk_sync_pre()); // the draws queued for the pre-pass command buffer go before this one VkCommandBuffer cb = pre ? pre : ex->cb; VkMemSpan ring = {0}; if (args) { @@ -5952,8 +6075,10 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); ME.skipped++; return 0; + } ring.buffer = mesh_ring, ind = ˚ cb = pre ? pre : ex->cb; + } uint32_t ep = vk_texture_epoch(); if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over ME.epoch = ep; @@ -5963,20 +6088,55 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const const uint32_t *h = ME.rb_map + 4 * ((ME.resets - 32) % ME_RB); for (int k = 0; k < 4; k++) if (h[k] > ME.max[k]) ME.max[k] = h[k]; + } VkBufferCopy rc = {0, 16 * (ME.resets % ME_RB), 16}; vkCmdCopyBuffer(cb, ME.buf, ME.rb, 1, &rc); ME.resets++; me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); + } vkCmdFillBuffer(cb, ME.buf, 0, 32, 0); // the counters, and the diagnosis words 4-7 (mesh_emu_spirv.c) me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + } uint32_t rec = ME.rec++ % MESH_EMU_DRECS; VkDeviceSize ro = MESH_EMU_DOFF + (VkDeviceSize)rec * MESH_EMU_DWORDS * 4; + if (MEQ.n && (MEQ.n == MEQ_MAX || (MEQ.q[0].rec + (uint32_t)MEQ.n) % MESH_EMU_DRECS != rec)) + me_flush(vk_sync_pre()); // full, or not the next record (the allocation shader takes a run of them) + // the record: DrawIndexedIndirect {0, 1, 0, 0, 0}, mesh dispatch, task base, then the words the allocation shader + // reads (mesh_emu_spirv.h MESH_EMU_R_*); an indirect first stage's count is copied from its arguments vkCmdFillBuffer(cb, ME.buf, ro, 4, 0); vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1); + vkCmdFillBuffer(cb, ME.buf, ro + 8, (MESH_EMU_R_FIRST - 2) * 4, 0); + if (ind) { + const VkMemSpan *src = args ? args : ind; + VkBufferCopy cp = {src->offset, ro + MESH_EMU_R_FIRST * 4, 4}; + vkCmdCopyBuffer(cb, src->buffer, ME.buf, 1, &cp); + } else vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_FIRST * 4, 4, count); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_ICUR * 4, 8, 0); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_PER * 4, 4, mi.max_p * (mi.prim ? mi.prim : 3)); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_TASK * 4, 8, tp ? 1 : 0); // and the unused last word uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), (uint32_t)(ro / 4), ME.iw, ME.icap, ME.vw, ME.vcap, ME.trecs}; + MeQ *q = &MEQ.q[MEQ.n]; + *q = (MeQ){.tp = tp, .mp = mp, .layout = layout, .tpl_c = tpl_c, .set0 = set0, .pset = pool ? PL.set : VK_NULL_HANDLE, + .push = push, .count = count, .rec = rec, .ib = ind ? ind->buffer : VK_NULL_HANDLE, .io = ind ? ind->offset : 0}; + memcpy(q->pc, pc, sizeof pc); + if (push) { + if (MEQ.dn + emu_d_bytes > MEQ.dcap) { + size_t c = MEQ.dcap ? MEQ.dcap * 2 : 65536; + while (c < MEQ.dn + emu_d_bytes) c *= 2; + uint8_t *d = realloc(MEQ.d, c); + if (!d) { + ME.skipped++; + return 0; + } + MEQ.d = d, MEQ.dcap = c; + } + memcpy(MEQ.d + MEQ.dn, emu_d, emu_d_bytes); + q->doff = MEQ.dn, q->dbytes = emu_d_bytes, MEQ.dn += emu_d_bytes; } + MEQ.n++, MEQ.tasks += !!tp; + if (!pre) ex->recorded += 3 + me_flush(cb); // the one draw's stages, here if (!pre) { // the draw: prepared again (the pass reopens, every binding is recorded again) vk_exec_draw_new_cb(); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 80953490..7c2c1e5e 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -812,6 +812,8 @@ static void gpu_stats(PerfGpu *g) { g->mirror_ns = (uint64_t)plat_xchg64(&ts_mirror_ns, 0); } +WEAK void vk_exec_mesh_flush(VkCommandBuffer cb) { (void)cb; } // without vk_exec_draw.c: nothing is queued + // Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted. WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { (void)cb, (void)fence; @@ -1037,6 +1039,7 @@ static int nchain; static VkCommandBuffer pre_end(uint64_t fence) { VkCommandBuffer cb = pre_cb; if (!cb) return VK_NULL_HANDLE; + vk_exec_mesh_flush(cb); // the stages of the pass's emulated draws, by stage ts_close(cb, pre_pair, fence, GTL_PRE); pre_pair = -1; vkEndCommandBuffer(cb);