diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c index 280b1f2e..114d8292 100644 --- a/runtime/mesh_emu_spirv.c +++ b/runtime/mesh_emu_spirv.c @@ -254,8 +254,13 @@ static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int uint32_t mz; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); { + // indices from the draw's own range (reserved by the allocation shader), vertex slots from the arena's + uint32_t drw = pc_u32(e, 1), vcap = pc_u32(e, 5); + uint32_t icap = word_load(e, base, add(e, drw, cu(e, MESH_EMU_R_ICAP))); + uint32_t ib = atomic(e, OpAtomicIAdd, base, add(e, drw, cu(e, MESH_EMU_R_ICUR)), isz); uint32_t vb = atomic(e, OpAtomicIAdd, base, cu(e, 0), vsz); uint32_t iok = E2(e, OpULessThanEqual, m->t_bool, add(e, ib, isz), icap); + ib = add(e, word_load(e, base, add(e, drw, cu(e, 2))), ib); // relative to the area, as firstIndex uint32_t vok = E2(e, OpULessThanEqual, m->t_bool, add(e, vb, vsz), vcap); uint32_t mi; if_begin(e, iok, &mi); @@ -358,6 +363,7 @@ static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint uint32_t m2; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &m2); word_store(e, base, recw, nc); + atomic(e, OpAtomicIAdd, base, add(e, drw, cu(e, MESH_EMU_R_GROUPS)), nc); // for the index range if_end(e, m2); } if_end(e, m1); @@ -954,4 +960,138 @@ uint32_t *mesh_emu_vs(const MeshEmuInfo *mi, uint32_t need, const uint8_t types[ free(t.w), free(an.w), free(c.w); if (bad) { free(b.w); return NULL; } *out_n = b.n; - return b.w; \ No newline at end of file + return b.w; +#undef N +} + +// The allocation shader (mesh_emu_spirv.h): glslangValidator -V -g0 --target-env vulkan1.2, spirv-opt -O --strip-debug, +// spirv-val clean. Its source: +// #version 460 +// #extension GL_EXT_buffer_reference : require +// #extension GL_EXT_buffer_reference_uvec2 : require +// layout(local_size_x = 1) in; +// layout(buffer_reference, std430, buffer_reference_align = 4) buffer Words { uint w[]; }; +// layout(push_constant) uniform PC { uvec2 addr; uint first; uint n; uint mode; uint icap; uint trecs; uint pad; } pc; +// void main() { +// Words a = Words(pc.addr); +// for (uint d = 0u; d < pc.n; d++) { +// uint r = 64u + ((pc.first + d) % 4096u) * 16u; +// bool task = a.w[r + 14u] != 0u; +// if (pc.mode == 0u && task) { +// uint tb = a.w[2], k = a.w[r + 10u]; +// a.w[r + 8u] = tb; +// a.w[2] = k > pc.trecs ? pc.trecs + tb : tb + k; +// } else if ((pc.mode == 0u) != task) { +// uint g = pc.mode == 0u ? a.w[r + 10u] : a.w[r + 9u], per = a.w[r + 13u], ib = a.w[1]; +// uint ni = per != 0u && g > pc.icap / per ? pc.icap + 1u : g * per; +// a.w[r + 2u] = ib; +// a.w[r + 12u] = ib >= pc.icap ? 0u : min(ni, pc.icap - ib); +// a.w[1] = min(ib + ni, 0x7fffffffu); +// } +// } +// } +static const uint32_t alloc_words[] = { + 0x07230203, 0x00010500, 0x0008000b, 0x000000df, 0x00000000, 0x00020011, 0x00000001, 0x00020011, + 0x000014e3, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, + 0x000014e4, 0x00000001, 0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000f, + 0x00060010, 0x00000004, 0x00000011, 0x00000001, 0x00000001, 0x00000001, 0x00040047, 0x00000008, + 0x00000006, 0x00000004, 0x00030047, 0x00000009, 0x00000002, 0x00050048, 0x00000009, 0x00000000, + 0x00000023, 0x00000000, 0x00030047, 0x0000000d, 0x00000002, 0x00050048, 0x0000000d, 0x00000000, + 0x00000023, 0x00000000, 0x00050048, 0x0000000d, 0x00000001, 0x00000023, 0x00000008, 0x00050048, + 0x0000000d, 0x00000002, 0x00000023, 0x0000000c, 0x00050048, 0x0000000d, 0x00000003, 0x00000023, + 0x00000010, 0x00050048, 0x0000000d, 0x00000004, 0x00000023, 0x00000014, 0x00050048, 0x0000000d, + 0x00000005, 0x00000023, 0x00000018, 0x00050048, 0x0000000d, 0x00000006, 0x00000023, 0x0000001c, + 0x00040047, 0x000000cb, 0x0000000b, 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, + 0x00000002, 0x00030027, 0x00000006, 0x000014e5, 0x00040015, 0x00000007, 0x00000020, 0x00000000, + 0x0003001d, 0x00000008, 0x00000007, 0x0003001e, 0x00000009, 0x00000008, 0x00040020, 0x00000006, + 0x000014e5, 0x00000009, 0x00040017, 0x0000000c, 0x00000007, 0x00000002, 0x0009001e, 0x0000000d, + 0x0000000c, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00040020, + 0x0000000e, 0x00000009, 0x0000000d, 0x0004003b, 0x0000000e, 0x0000000f, 0x00000009, 0x00040015, + 0x00000010, 0x00000020, 0x00000001, 0x0004002b, 0x00000010, 0x00000011, 0x00000000, 0x00040020, + 0x00000012, 0x00000009, 0x0000000c, 0x0004002b, 0x00000007, 0x00000018, 0x00000000, 0x0004002b, + 0x00000010, 0x0000001f, 0x00000002, 0x00040020, 0x00000020, 0x00000009, 0x00000007, 0x00020014, + 0x00000023, 0x0004002b, 0x00000010, 0x00000027, 0x00000001, 0x0004002b, 0x00000007, 0x0000002c, + 0x00001000, 0x0004002b, 0x00000007, 0x0000002e, 0x00000010, 0x00040020, 0x00000037, 0x000014e5, + 0x00000007, 0x0004002b, 0x00000010, 0x0000003b, 0x00000003, 0x0004002b, 0x00000010, 0x00000056, + 0x00000005, 0x0004002b, 0x00000010, 0x00000094, 0x00000004, 0x0004002b, 0x00000007, 0x000000a0, + 0x00000001, 0x0004002b, 0x00000007, 0x000000c5, 0x7fffffff, 0x00040017, 0x000000ca, 0x00000007, + 0x00000003, 0x0006002c, 0x000000ca, 0x000000cb, 0x000000a0, 0x000000a0, 0x000000a0, 0x0004002b, + 0x00000007, 0x000000d8, 0x0000004e, 0x0004002b, 0x00000007, 0x000000d9, 0x00000049, 0x0004002b, + 0x00000007, 0x000000da, 0x0000004a, 0x0004002b, 0x00000007, 0x000000db, 0x0000004d, 0x0004002b, + 0x00000007, 0x000000dc, 0x00000042, 0x0004002b, 0x00000007, 0x000000dd, 0x0000004c, 0x0004002b, + 0x00000007, 0x000000de, 0x00000048, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, + 0x000200f8, 0x00000005, 0x00050041, 0x00000012, 0x00000013, 0x0000000f, 0x00000011, 0x0004003d, + 0x0000000c, 0x00000014, 0x00000013, 0x0004007c, 0x00000006, 0x00000015, 0x00000014, 0x000200f9, + 0x00000019, 0x000200f8, 0x00000019, 0x000700f5, 0x00000007, 0x000000cc, 0x00000018, 0x00000005, + 0x000000c9, 0x0000001c, 0x00050041, 0x00000020, 0x00000021, 0x0000000f, 0x0000001f, 0x0004003d, + 0x00000007, 0x00000022, 0x00000021, 0x000500b0, 0x00000023, 0x00000024, 0x000000cc, 0x00000022, + 0x000400f6, 0x0000001b, 0x0000001c, 0x00000000, 0x000400fa, 0x00000024, 0x0000001a, 0x0000001b, + 0x000200f8, 0x0000001a, 0x00050041, 0x00000020, 0x00000028, 0x0000000f, 0x00000027, 0x0004003d, + 0x00000007, 0x00000029, 0x00000028, 0x00050080, 0x00000007, 0x0000002b, 0x00000029, 0x000000cc, + 0x00050089, 0x00000007, 0x0000002d, 0x0000002b, 0x0000002c, 0x00050084, 0x00000007, 0x0000002f, + 0x0000002d, 0x0000002e, 0x00050080, 0x00000007, 0x00000036, 0x0000002f, 0x000000d8, 0x00060041, + 0x00000037, 0x00000038, 0x00000015, 0x00000011, 0x00000036, 0x0006003d, 0x00000007, 0x00000039, + 0x00000038, 0x00000002, 0x00000004, 0x000500ab, 0x00000023, 0x0000003a, 0x00000039, 0x00000018, + 0x00050041, 0x00000020, 0x0000003c, 0x0000000f, 0x0000003b, 0x0004003d, 0x00000007, 0x0000003d, + 0x0000003c, 0x000500aa, 0x00000023, 0x0000003e, 0x0000003d, 0x00000018, 0x000500a7, 0x00000023, + 0x00000040, 0x0000003e, 0x0000003a, 0x000300f7, 0x00000042, 0x00000000, 0x000400fa, 0x00000040, + 0x00000041, 0x00000067, 0x000200f8, 0x00000041, 0x00060041, 0x00000037, 0x00000045, 0x00000015, + 0x00000011, 0x0000001f, 0x0006003d, 0x00000007, 0x00000046, 0x00000045, 0x00000002, 0x00000004, + 0x00050080, 0x00000007, 0x0000004b, 0x0000002f, 0x000000da, 0x00060041, 0x00000037, 0x0000004c, + 0x00000015, 0x00000011, 0x0000004b, 0x0006003d, 0x00000007, 0x0000004d, 0x0000004c, 0x00000002, + 0x00000004, 0x00050080, 0x00000007, 0x00000051, 0x0000002f, 0x000000de, 0x00060041, 0x00000037, + 0x00000053, 0x00000015, 0x00000011, 0x00000051, 0x0005003e, 0x00000053, 0x00000046, 0x00000002, + 0x00000004, 0x00050041, 0x00000020, 0x00000057, 0x0000000f, 0x00000056, 0x0004003d, 0x00000007, + 0x00000058, 0x00000057, 0x000500ac, 0x00000023, 0x00000059, 0x0000004d, 0x00000058, 0x000300f7, + 0x0000005c, 0x00000000, 0x000400fa, 0x00000059, 0x0000005b, 0x00000061, 0x000200f8, 0x0000005b, + 0x00050080, 0x00000007, 0x00000060, 0x00000058, 0x00000046, 0x000200f9, 0x0000005c, 0x000200f8, + 0x00000061, 0x00050080, 0x00000007, 0x00000064, 0x00000046, 0x0000004d, 0x000200f9, 0x0000005c, + 0x000200f8, 0x0000005c, 0x000700f5, 0x00000007, 0x000000d0, 0x00000060, 0x0000005b, 0x00000064, + 0x00000061, 0x0005003e, 0x00000045, 0x000000d0, 0x00000002, 0x00000004, 0x000200f9, 0x00000042, + 0x000200f8, 0x00000067, 0x000500a5, 0x00000023, 0x0000006c, 0x0000003e, 0x0000003a, 0x000300f7, + 0x0000006e, 0x00000000, 0x000400fa, 0x0000006c, 0x0000006d, 0x0000006e, 0x000200f8, 0x0000006d, + 0x000300f7, 0x00000075, 0x00000000, 0x000400fa, 0x0000003e, 0x00000074, 0x0000007b, 0x000200f8, + 0x00000074, 0x00050080, 0x00000007, 0x00000078, 0x0000002f, 0x000000da, 0x00060041, 0x00000037, + 0x00000079, 0x00000015, 0x00000011, 0x00000078, 0x0006003d, 0x00000007, 0x0000007a, 0x00000079, + 0x00000002, 0x00000004, 0x000200f9, 0x00000075, 0x000200f8, 0x0000007b, 0x00050080, 0x00000007, + 0x0000007f, 0x0000002f, 0x000000d9, 0x00060041, 0x00000037, 0x00000080, 0x00000015, 0x00000011, + 0x0000007f, 0x0006003d, 0x00000007, 0x00000081, 0x00000080, 0x00000002, 0x00000004, 0x000200f9, + 0x00000075, 0x000200f8, 0x00000075, 0x000700f5, 0x00000007, 0x000000cd, 0x0000007a, 0x00000074, + 0x00000081, 0x0000007b, 0x00050080, 0x00000007, 0x00000087, 0x0000002f, 0x000000db, 0x00060041, + 0x00000037, 0x00000088, 0x00000015, 0x00000011, 0x00000087, 0x0006003d, 0x00000007, 0x00000089, + 0x00000088, 0x00000002, 0x00000004, 0x00060041, 0x00000037, 0x0000008c, 0x00000015, 0x00000011, + 0x00000027, 0x0006003d, 0x00000007, 0x0000008d, 0x0000008c, 0x00000002, 0x00000004, 0x000500ab, + 0x00000023, 0x00000090, 0x00000089, 0x00000018, 0x000300f7, 0x00000092, 0x00000000, 0x000400fa, + 0x00000090, 0x00000091, 0x00000092, 0x000200f8, 0x00000091, 0x00050041, 0x00000020, 0x00000095, + 0x0000000f, 0x00000094, 0x0004003d, 0x00000007, 0x00000096, 0x00000095, 0x00050086, 0x00000007, + 0x00000098, 0x00000096, 0x00000089, 0x000500ac, 0x00000023, 0x00000099, 0x000000cd, 0x00000098, + 0x000200f9, 0x00000092, 0x000200f8, 0x00000092, 0x000700f5, 0x00000023, 0x0000009a, 0x00000090, + 0x00000075, 0x00000099, 0x00000091, 0x000300f7, 0x0000009d, 0x00000000, 0x000400fa, 0x0000009a, + 0x0000009c, 0x000000a2, 0x000200f8, 0x0000009c, 0x00050041, 0x00000020, 0x0000009e, 0x0000000f, + 0x00000094, 0x0004003d, 0x00000007, 0x0000009f, 0x0000009e, 0x00050080, 0x00000007, 0x000000a1, + 0x0000009f, 0x000000a0, 0x000200f9, 0x0000009d, 0x000200f8, 0x000000a2, 0x00050084, 0x00000007, + 0x000000a5, 0x000000cd, 0x00000089, 0x000200f9, 0x0000009d, 0x000200f8, 0x0000009d, 0x000700f5, + 0x00000007, 0x000000ce, 0x000000a1, 0x0000009c, 0x000000a5, 0x000000a2, 0x00050080, 0x00000007, + 0x000000aa, 0x0000002f, 0x000000dc, 0x00060041, 0x00000037, 0x000000ac, 0x00000015, 0x00000011, + 0x000000aa, 0x0005003e, 0x000000ac, 0x0000008d, 0x00000002, 0x00000004, 0x00050080, 0x00000007, + 0x000000b0, 0x0000002f, 0x000000dd, 0x00050041, 0x00000020, 0x000000b2, 0x0000000f, 0x00000094, + 0x0004003d, 0x00000007, 0x000000b3, 0x000000b2, 0x000500ae, 0x00000023, 0x000000b4, 0x0000008d, + 0x000000b3, 0x000300f7, 0x000000b7, 0x00000000, 0x000400fa, 0x000000b4, 0x000000b6, 0x000000b8, + 0x000200f8, 0x000000b6, 0x000200f9, 0x000000b7, 0x000200f8, 0x000000b8, 0x00050082, 0x00000007, + 0x000000bd, 0x000000b3, 0x0000008d, 0x0007000c, 0x00000007, 0x000000be, 0x00000001, 0x00000026, + 0x000000ce, 0x000000bd, 0x000200f9, 0x000000b7, 0x000200f8, 0x000000b7, 0x000700f5, 0x00000007, + 0x000000cf, 0x00000018, 0x000000b6, 0x000000be, 0x000000b8, 0x00060041, 0x00000037, 0x000000c0, + 0x00000015, 0x00000011, 0x000000b0, 0x0005003e, 0x000000c0, 0x000000cf, 0x00000002, 0x00000004, + 0x00050080, 0x00000007, 0x000000c4, 0x0000008d, 0x000000ce, 0x0007000c, 0x00000007, 0x000000c6, + 0x00000001, 0x00000026, 0x000000c4, 0x000000c5, 0x0005003e, 0x0000008c, 0x000000c6, 0x00000002, + 0x00000004, 0x000200f9, 0x0000006e, 0x000200f8, 0x0000006e, 0x000200f9, 0x00000042, 0x000200f8, + 0x00000042, 0x000200f9, 0x0000001c, 0x000200f8, 0x0000001c, 0x00050080, 0x00000007, 0x000000c9, + 0x000000cc, 0x00000027, 0x000200f9, 0x00000019, 0x000200f8, 0x0000001b, 0x000100fd, 0x00010038, +}; +const uint32_t *mesh_emu_alloc_spirv(size_t *n) { + *n = sizeof alloc_words / sizeof *alloc_words; + return alloc_words; +} +_Static_assert(MESH_EMU_DOFF / 4 == 64 && MESH_EMU_DRECS == 4096 && MESH_EMU_DWORDS == 16 && MESH_EMU_R_GROUPS == 9 && + MESH_EMU_R_FIRST == 10 && MESH_EMU_R_ICAP == 12 && MESH_EMU_R_PER == 13 && MESH_EMU_R_TASK == 14, + "the allocation shader's constants"); diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h index 430ca76a..0584e3ae 100644 --- a/runtime/mesh_emu_spirv.h +++ b/runtime/mesh_emu_spirv.h @@ -7,7 +7,13 @@ #include // The arena: one buffer, its address in the push constants of every emulation stage. +// words 0-3: vertex slots used, indices reserved, task records reserved, overflows (the replay zeroes them per +// submission) // MESH_EMU_DOFF: draw records, MESH_EMU_DWORDS words each: DrawIndexedIndirect {indexCount, instanceCount, +// firstIndex, vertexOffset, firstInstance}, mesh dispatch {x, y, z}, task base, then the MESH_EMU_R_* +// words; the replay fills them, the allocation shader (mesh_emu_alloc_spirv) reserves each draw's +// task records and index range, so that the stages of many draws can run together: a draw's +// indices are one range (its DrawIndexedIndirect) whatever the order its mesh workgroups run in // MESH_EMU_TOFF: task records, MESH_EMU_TWORDS words each: mesh workgroup count, then the payload // index area (uint32 indices), vertex area (16-byte slots: position, then each generic output) at offsets the // replay passes in the push constants. @@ -15,6 +21,12 @@ #define MESH_EMU_DWORDS 16u #define MESH_EMU_DRECS 4096u #define MESH_EMU_TOFF (MESH_EMU_DOFF + MESH_EMU_DRECS * MESH_EMU_DWORDS * 4u) +#define MESH_EMU_R_GROUPS 9u // mesh workgroups with output: the task stage adds the counts it emits +#define MESH_EMU_R_FIRST 10u // the first stage's workgroups (the replay: the count, or a copy of the indirect one) +#define MESH_EMU_R_ICUR 11u // indices taken from the draw's range (the mesh stage's atomics) +#define MESH_EMU_R_ICAP 12u // the draw's range: indices reserved (fit in the area) +#define MESH_EMU_R_PER 13u // indices one mesh workgroup may emit (OutputPrimitives x vertices per primitive) +#define MESH_EMU_R_TASK 14u // 1 with a task stage #define MESH_EMU_TWORDS 780u // 1 + payloads of up to 776 words (3072 bytes and the 32 the // translator adds for the payload header since 0.6.0), padded // task records: as many as the push constants say (the replay sizes them, NX2NAVI_MESH_EMU_TASKS); the index area @@ -26,6 +38,13 @@ // submission); word 3 counts mesh workgroups that did not fit. #define MESH_EMU_PC_BYTES 32u +// The allocation shader (GLCompute, one invocation): for the n draw records from ring index first, mode 0 reserves the +// task records of draws with a task stage (MESH_EMU_R_FIRST of them, from word 2) and the index ranges of draws +// without one (MESH_EMU_R_FIRST x MESH_EMU_R_PER, from word 1); mode 1, after the task stages, the index ranges of +// draws with one (MESH_EMU_R_GROUPS x MESH_EMU_R_PER). Push constants, 32 bytes: u64 arena address; uint first, n, +// mode, index capacity, task record capacity, unused. Static words, *n of them. +const uint32_t *mesh_emu_alloc_spirv(size_t *n); + typedef struct MeshEmuInfo { int task; // 1 TaskEXT, 0 MeshEXT uint32_t threads; // LocalSize x diff --git a/runtime/vk.h b/runtime/vk.h index a4f25dc4..fb7d9a4a 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -764,6 +764,9 @@ void vk_sync_worker_end(VkCommandBuffer cb); // The part's pre-pass command buffer: submitted right before the part being recorded (after its mirror prologue), begun // with a full barrier at the first call of a part; VK_NULL_HANDLE when it cannot be made. Replay thread, sub_lock held. VkCommandBuffer vk_sync_pre(void); +// vk_exec_draw.c: records into the pre-pass command buffer, as it ends, the stages of the emulated mesh draws queued +// for it (NX2NAVI_MESH_EMU_BATCH). A weak vk_sync.c version does nothing. +void vk_exec_mesh_flush(VkCommandBuffer cb); // Ends the part being recorded without handing it to the queue (it goes with the next part that does, in order) and // returns the command buffer the replay continues in; VK_NULL_HANDLE when no more parts can wait. Replay thread. VkCommandBuffer vk_sync_cut(void); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index e5b6509f..e6423227 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5829,6 +5829,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { // NX2NAVI_MESH_EMU_TASKS its task records per submission (default 16384, 3 KiB each). Before the counters start over // they are copied to a readback ring; the perf log reports the most a submission asked for against the capacities. #define ME_RB 64 +static struct { VkBuffer buf, rb; VmaAllocation alloc, rb_alloc; uint32_t *rb_map; @@ -5837,8 +5838,29 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { int failed; uint64_t draws, skipped; uint32_t max[4]; + VkPipelineLayout alayout; // the allocation shader's (push constants only) + VkPipeline apipe; } ME; +// The allocation shader's pipeline (mesh_emu_spirv.h mesh_emu_alloc_spirv); 0 when it cannot be made. +static int me_alloc_pipe(void) { + VkDevice dev = vk_ctx()->device; + VkPushConstantRange pcr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 32}; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .pushConstantRangeCount = 1, + .pPushConstantRanges = &pcr}; + if (vkCreatePipelineLayout(dev, &li, NULL, &ME.alayout) != VK_SUCCESS) return ME.alayout = VK_NULL_HANDLE, 0; + size_t n = 0; + const uint32_t *w = mesh_emu_alloc_spirv(&n); + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = n * 4, .pCode = w}; + VkShaderModule m; + if (vkCreateShaderModule(dev, &mi, NULL, &m) != VK_SUCCESS) return 0; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = ME.alayout}; + VkResult r = vkCreateComputePipelines(dev, VK_NULL_HANDLE, 1, &ci, NULL, &ME.apipe); + vkDestroyShaderModule(dev, m, NULL); + if (r != VK_SUCCESS) return ME.apipe = VK_NULL_HANDLE, 0; return 1; } @@ -5858,6 +5880,9 @@ static int me_arena(void) { VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (!me_alloc_pipe()) { + ME.failed = 1; + rt_log("vk: mesh emulation: the allocation shader's pipeline could not be created: mesh draws skipped"); return 0; } if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ME.buf, &ME.alloc, NULL) != VK_SUCCESS) { @@ -5880,6 +5905,8 @@ static int me_arena(void) { if (vmaCreateBuffer(vk_ctx()->vma, &rbi, &rai, &ME.rb, &ME.rb_alloc, &rinfo) == VK_SUCCESS) ME.rb_map = rinfo.pMappedData; else ME.rb = VK_NULL_HANDLE; return 1; +} + // The access masks also carry the generic MEMORY_WRITE / MEMORY_READ bits. SHADER_STORAGE_READ/WRITE exist only in // synchronization2 (bits above 31), and AMD's driver for Polaris (23.10.01, RX 580) did not act on them: the mesh // dispatch that reads the {groups, task workgroups, 1} the task stage had just written read zeros and launched no @@ -5892,9 +5919,101 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla .dstAccessMask = da | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); +} + +// The stages of the emulated draws whose records were filled, waiting to be recorded (me_flush): the draws of a pass +// with NX2NAVI_MESH_EMU_BATCH, which run in the part's pre-pass command buffer when it ends (vk_exec_mesh_flush, from +// vk_sync.c), or the one draw that runs in the part. Recorded by stage, with one barrier between stages for all the +// draws: the fills, the allocation (task records; index ranges of draws without a task stage), every task stage, the +// allocation of the other index ranges, every mesh stage. Each draw's indices are a range of its own and its task +// records too (the allocation shader), so the draws' stages need nothing of each other's. One by one, as before +// 2026-11-11, each draw took three barriers, and its task base and first index were copies of the arena's counters +// after the draw before it: an RX 580 spent 5.5 ms of a frame's GPU time on 69 draws' stages (Kokiri Forest), +// most of it waiting at the barriers with the GPU nearly idle. +typedef struct { + VkPipeline tp, mp; + VkPipelineLayout layout; + VkDescriptorUpdateTemplate tpl_c; + VkDescriptorSet set0, pset; + int push; + uint32_t count, rec, pc[8]; + VkBuffer ib; // the first stage's indirect dispatch (VK_NULL_HANDLE: count) + VkDeviceSize io; + size_t doff; + uint32_t dbytes; +} MeQ; +#define MEQ_MAX 256 +static struct { + MeQ q[MEQ_MAX]; + int n, tasks; + uint8_t *d; // the push descriptors' data of the queued draws + size_t dn, dcap; +} MEQ; + +static void me_bind(VkCommandBuffer cb, const MeQ *q) { + if (q->push) rec_push_tpl(vk_pipeline_push_tpl_fn(), cb, q->tpl_c, q->layout, 0, MEQ.d + q->doff, q->dbytes); + else vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->layout, 0, 1, &q->set0, 0, NULL); + if (q->pset) vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->layout, 2, 1, &q->pset, 0, NULL); + vkCmdPushConstants(cb, q->layout, VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES, q->pc); +} + +static void me_first(VkCommandBuffer cb, const MeQ *q) { + if (q->ib) vkCmdDispatchIndirect(cb, q->ib, q->io); + else vkCmdDispatch(cb, q->count, 1, 1); +} + +static void me_alloc(VkCommandBuffer cb, uint32_t mode) { + uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), MEQ.q[0].rec, (uint32_t)MEQ.n, mode, ME.icap, ME.trecs, 0}; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, ME.apipe); + vkCmdPushConstants(cb, ME.alayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); + vkCmdDispatch(cb, 1, 1, 1); +} + +// The queued draws' stages into cb (the pre-pass command buffer, or the part for a draw that runs there); returns the +// commands recorded. +static int me_flush(VkCommandBuffer cb) { + if (!MEQ.n || !cb) return 0; + const VkAccessFlags2 rw = VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT; + const VkPipelineStageFlags2 cs = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT; + int k = 0; + me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + me_alloc(cb, 0), k += 4; + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + if (MEQ.tasks) { // the task stages wrote the mesh dispatch {max mesh groups, task workgroups, 1} into the draw record + for (int i = 0; i < MEQ.n; i++) { + const MeQ *q = &MEQ.q[i]; + if (!q->tp) continue; + me_bind(cb, q); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->tp); + me_first(cb, q), k += 4; } + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + me_alloc(cb, 1), k += 4; + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); } + for (int i = 0; i < MEQ.n; i++) { + const MeQ *q = &MEQ.q[i]; + me_bind(cb, q); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->mp); + if (q->tp) vkCmdDispatchIndirect(cb, ME.buf, MESH_EMU_DOFF + (VkDeviceSize)q->rec * MESH_EMU_DWORDS * 4 + 20); + else me_first(cb, q); + k += 4; } + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, + VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_2_INDEX_INPUT_BIT | VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT | + cs | VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_2_INDEX_READ_BIT | rw | VK_ACCESS_2_TRANSFER_READ_BIT); + MEQ.n = MEQ.tasks = 0, MEQ.dn = 0; + return k + 4; +} + +// vk.h: the pre-pass command buffer ends (vk_sync.c pre_end): the stages of the draws queued for it go in first. +void vk_exec_mesh_flush(VkCommandBuffer cb) { me_flush(cb); } + // count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from // args: an indirect draw's arguments in game memory (count, 1, 1 copied into a ring entry, which ind then is) static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args) { @@ -5939,12 +6058,16 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const if (batch && ex->in_pass) { if (ex->pre_pass != ex->pass_serial && vk_exec_split_now(ex)) { if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { // the pass again, in the new part + ME.skipped++; + return 0; + } ex->pre_pass = ex->pass_serial; set0 = ex->dc->B.ds_bound[0]; } if (ex->pre_pass == ex->pass_serial) pre = vk_sync_pre(); } if (!pre) vk_exec_barrier(ex); // ends the pass; what the game wrote before (its culling) is visible to the stages + if (!pre && MEQ.n) me_flush(vk_sync_pre()); // the draws queued for the pre-pass command buffer go before this one VkCommandBuffer cb = pre ? pre : ex->cb; VkMemSpan ring = {0}; if (args) { @@ -5952,8 +6075,10 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); ME.skipped++; return 0; + } ring.buffer = mesh_ring, ind = ˚ cb = pre ? pre : ex->cb; + } uint32_t ep = vk_texture_epoch(); if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over ME.epoch = ep; @@ -5963,20 +6088,55 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const const uint32_t *h = ME.rb_map + 4 * ((ME.resets - 32) % ME_RB); for (int k = 0; k < 4; k++) if (h[k] > ME.max[k]) ME.max[k] = h[k]; + } VkBufferCopy rc = {0, 16 * (ME.resets % ME_RB), 16}; vkCmdCopyBuffer(cb, ME.buf, ME.rb, 1, &rc); ME.resets++; me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); + } vkCmdFillBuffer(cb, ME.buf, 0, 32, 0); // the counters, and the diagnosis words 4-7 (mesh_emu_spirv.c) me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + } uint32_t rec = ME.rec++ % MESH_EMU_DRECS; VkDeviceSize ro = MESH_EMU_DOFF + (VkDeviceSize)rec * MESH_EMU_DWORDS * 4; + if (MEQ.n && (MEQ.n == MEQ_MAX || (MEQ.q[0].rec + (uint32_t)MEQ.n) % MESH_EMU_DRECS != rec)) + me_flush(vk_sync_pre()); // full, or not the next record (the allocation shader takes a run of them) + // the record: DrawIndexedIndirect {0, 1, 0, 0, 0}, mesh dispatch, task base, then the words the allocation shader + // reads (mesh_emu_spirv.h MESH_EMU_R_*); an indirect first stage's count is copied from its arguments vkCmdFillBuffer(cb, ME.buf, ro, 4, 0); vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1); + vkCmdFillBuffer(cb, ME.buf, ro + 8, (MESH_EMU_R_FIRST - 2) * 4, 0); + if (ind) { + const VkMemSpan *src = args ? args : ind; + VkBufferCopy cp = {src->offset, ro + MESH_EMU_R_FIRST * 4, 4}; + vkCmdCopyBuffer(cb, src->buffer, ME.buf, 1, &cp); + } else vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_FIRST * 4, 4, count); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_ICUR * 4, 8, 0); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_PER * 4, 4, mi.max_p * (mi.prim ? mi.prim : 3)); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_TASK * 4, 8, tp ? 1 : 0); // and the unused last word uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), (uint32_t)(ro / 4), ME.iw, ME.icap, ME.vw, ME.vcap, ME.trecs}; + MeQ *q = &MEQ.q[MEQ.n]; + *q = (MeQ){.tp = tp, .mp = mp, .layout = layout, .tpl_c = tpl_c, .set0 = set0, .pset = pool ? PL.set : VK_NULL_HANDLE, + .push = push, .count = count, .rec = rec, .ib = ind ? ind->buffer : VK_NULL_HANDLE, .io = ind ? ind->offset : 0}; + memcpy(q->pc, pc, sizeof pc); + if (push) { + if (MEQ.dn + emu_d_bytes > MEQ.dcap) { + size_t c = MEQ.dcap ? MEQ.dcap * 2 : 65536; + while (c < MEQ.dn + emu_d_bytes) c *= 2; + uint8_t *d = realloc(MEQ.d, c); + if (!d) { + ME.skipped++; + return 0; + } + MEQ.d = d, MEQ.dcap = c; + } + memcpy(MEQ.d + MEQ.dn, emu_d, emu_d_bytes); + q->doff = MEQ.dn, q->dbytes = emu_d_bytes, MEQ.dn += emu_d_bytes; } + MEQ.n++, MEQ.tasks += !!tp; + if (!pre) ex->recorded += 3 + me_flush(cb); // the one draw's stages, here if (!pre) { // the draw: prepared again (the pass reopens, every binding is recorded again) vk_exec_draw_new_cb(); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 80953490..7c2c1e5e 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -812,6 +812,8 @@ static void gpu_stats(PerfGpu *g) { g->mirror_ns = (uint64_t)plat_xchg64(&ts_mirror_ns, 0); } +WEAK void vk_exec_mesh_flush(VkCommandBuffer cb) { (void)cb; } // without vk_exec_draw.c: nothing is queued + // Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted. WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { (void)cb, (void)fence; @@ -1037,6 +1039,7 @@ static int nchain; static VkCommandBuffer pre_end(uint64_t fence) { VkCommandBuffer cb = pre_cb; if (!cb) return VK_NULL_HANDLE; + vk_exec_mesh_flush(cb); // the stages of the pass's emulated draws, by stage ts_close(cb, pre_pair, fence, GTL_PRE); pre_pair = -1; vkEndCommandBuffer(cb);