diff --git a/runtime/gfx.h b/runtime/gfx.h index bcff7188..21250f95 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -100,6 +100,7 @@ size_t gfx_shader_warm_list(GfxWarmEnt **out); // malloc'd; recorde const GfxShader *gfx_shader_load_private(const uint8_t iid[32], int *owned); void gfx_shader_release_private(const GfxShader *s); const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words); +const char *gfx_shader_res_json(const GfxShader *s); // the translator's .res.json text int gfx_shader_write_code(const GfxShader *s, const char *path); // the guest program, from the stored inputs // A shader of the runtime's own (SPIR-V and a .res.json text as the translator writes them), outside the cache and // never freed; its key is the sha256 of both. NULL when the SPIR-V is malformed. diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 132c395e..1653b3b7 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -2424,6 +2424,8 @@ const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words) { return s->spirv; } +const char *gfx_shader_res_json(const GfxShader *s) { return s->reflection; } + // The guest program a shader was translated from, as its stored inputs hold it (the whole program: NVN2 code from its // first byte), written to path for a disassembler (NX2NAVI_DUMP's sh_.code), and next to it .job, the // translator request that translates it (one line for nx2navi_xlate --serve; its "out" names the shader cache, and diff --git a/runtime/shaders/meshfb_scan.comp b/runtime/shaders/meshfb_scan.comp new file mode 100644 index 00000000..eac0e3cd --- /dev/null +++ b/runtime/shaders/meshfb_scan.comp @@ -0,0 +1,47 @@ +#version 450 +// The mesh fallback's pass between a draw's task dispatch and its mesh dispatch (vk_mesh_fb.c vk_meshfb_scan, +// mesh_fb.c): every task workgroup wrote how many mesh workgroups it starts into its payload (word 1). This lists them +// in task order, so that mesh workgroup g finds its task and its index among that task's in list entry g, and writes +// the draw's record: how many mesh workgroups to dispatch and how many instances to draw. +// +// One thread walks the tasks: the order of the list is the order the primitives are drawn in, which has to be the +// same in every frame and in the depth and colour passes of one frame. +layout(local_size_x = 1) in; + +layout(push_constant) uniform Params { + uint rec; // the record's first word in the record buffer + uint task_words; // words of a task workgroup's payload in the task buffer + uint fan; // mesh workgroups a task workgroup starts at most + uint slot_words; // words of a mesh workgroup's output in the mesh buffer + uint verts; // vertices drawn per mesh workgroup + uint max_groups; // workgroups one dispatch may have (maxComputeWorkGroupCount) +} pc; + +// The record: words 0-2 the task dispatch (x written before this pass), 4-6 the mesh dispatch, 8-11 the draw (vertices, +// instances, first vertex, first instance), 12 the mesh workgroups the tasks asked for, 13 the task workgroups. +layout(set = 0, binding = 0) buffer Rec { uint w[]; } rec; +layout(set = 0, binding = 1) readonly buffer Task { uint w[]; } task; +layout(set = 0, binding = 2) writeonly buffer List { uint e[]; } list; +layout(set = 0, binding = 3) readonly buffer Mesh { uint w[]; } mesh; // read for its size alone + +void main() { + uint asked = rec.w[pc.rec]; + uint tasks = min(min(asked, uint(task.w.length()) / pc.task_words), 1u << 24); + uint room = min(min(uint(mesh.w.length()) / pc.slot_words, uint(list.e.length())), pc.max_groups); + uint fan = min(pc.fan, 255u); + uint m = 0u, want = 0u; + for (uint t = 0u; t < tasks; t++) { + uint c = min(task.w[t * pc.task_words + 1u], fan); + want += c; + for (uint j = 0u; j < c && m < room; j++) list.e[m++] = t | j << 24; + } + rec.w[pc.rec + 4u] = m; + rec.w[pc.rec + 5u] = 1u; + rec.w[pc.rec + 6u] = 1u; + rec.w[pc.rec + 8u] = pc.verts; + rec.w[pc.rec + 9u] = m; + rec.w[pc.rec + 10u] = 0u; + rec.w[pc.rec + 11u] = 0u; + rec.w[pc.rec + 12u] = want; + rec.w[pc.rec + 13u] = asked; +} diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 584c1d54..102e4b9d 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -29,6 +29,8 @@ #endif #include "vk_exec.h" #include "vk_gpa.h" +#include "mesh_fb.h" +#include "vk_mesh_fb.h" #include "platform/platform.h" #include "vk_dlaa.h" @@ -446,7 +448,12 @@ static void twins_release(VkCtx *c); // Releases the stand-ins and the depth twins. The device must be idle; the game path leaves them to process exit like // the pipelines. +static void mesh_ring_release(VkCtx *c); +static void pool_marks_release(VkCtx *c); + void vk_exec_draw_shutdown(void) { + vk_meshfb_shutdown(); + if (vk_ctx()) mesh_ring_release(vk_ctx()), pool_marks_release(vk_ctx()); // made without the stand-ins below too if (!Z.ready) return; VkCtx *c = vk_ctx(); twins_release(c); @@ -948,6 +955,13 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G // and *pending = 1 says the wait ended first. static int sf_mvprev; // shader_for asks for the vertex shader's last-frame variant (per-object motion) +// Probes only (mesh_fb_probe.c): the shader of the program stage whose control section is ctl, in place of a +// translation. The probes build their programs from modules made here, which no translator has an input for. +static struct { const uint8_t *ctl; const GfxShader *s; } probe_sh[16]; +static int n_probe_sh; +void vk_exec_probe_shader(const uint8_t *ctl, const GfxShader *s) { + if (n_probe_sh < 16) probe_sh[n_probe_sh].ctl = ctl, probe_sh[n_probe_sh].s = s, n_probe_sh++; +} static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pending, uint32_t await_ms) { const uint8_t *ctl, *code; uint32_t len; @@ -955,6 +969,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (pending) *pending = 0; if (!ex->prog || !find_stage(ex->prog, st, &ctl, &code, &len)) return NULL; *present = 1; + for (int i = 0; i < n_probe_sh; i++) + if (probe_sh[i].ctl == ctl) return probe_sh[i].s; GfxShaderAnswers a; uint64_t serial = ex->prog->serial; answers_for(ex, st, ctl, slot_mask_of(serial, st), &a); @@ -994,7 +1010,12 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin // ---- descriptors +// The mesh fallback (below, "mesh fallback") runs a program's task and mesh stages as compute modules: while it binds +// one's resources, a compute module's bindings are those of this NVN stage (6 task, 7 mesh), not of the compute stage. +static int mfb_stage; + static int nvn_stage_of(int gfx_stage) { + if (gfx_stage == GFX_STAGE_CS && mfb_stage) return mfb_stage; return gfx_stage == GFX_STAGE_VS ? NVN_STAGE_VS : gfx_stage == GFX_STAGE_FS ? NVN_STAGE_FS : gfx_stage == GFX_STAGE_TS ? NVN_STAGE_TASK : gfx_stage == GFX_STAGE_MS ? NVN_STAGE_MESH : NVN_STAGE_CS; } @@ -1373,6 +1394,8 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * VkDescriptorBufferInfo t = vk_gpa_info(); return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE}; } + // The mesh fallback's own buffers (mesh_fb.c): no guest memory behind them + if (r->cbuf_slot >= MESHFB_CB_TASK && r->cbuf_slot <= MESHFB_CB_LIST) return vk_meshfb_buffer(r->cbuf_slot); if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); uint64_t max = LIM.sbo_max, align = LIM.sbo_align; VkDescriptorBufferInfo bi; @@ -1908,6 +1931,15 @@ static void pool_marks_make(void) { __atomic_store_n(&PL.marks_ready, 1, __ATOMIC_RELEASE); // the recording thread reads the rest after this } +static void pool_marks_release(VkCtx *c) { // vk_exec_draw_shutdown: a probe ends with the device idle + if (!PL.marks) return; + __atomic_store_n(&PL.marks_ready, 0, __ATOMIC_RELEASE); + vmaDestroyBuffer(c->vma, PL.marks, PL.marks_alloc); + for (int i = 0; i < MARKS_RB; i++) + if (PL.rb[i].buf) vmaDestroyBuffer(c->vma, PL.rb[i].buf, PL.rb[i].alloc), PL.rb[i].buf = VK_NULL_HANDLE; + PL.marks = VK_NULL_HANDLE; +} + void vk_exec_pool_marks(VkExec *ex) { if (!__atomic_load_n(&PL.marks_ready, __ATOMIC_ACQUIRE)) return; VkCtx *c = vk_ctx(); @@ -3121,7 +3153,7 @@ static int pb_key(const VkExec *ex, const VkPipeBinding *b, uint32_t k, uint64_t return 1; // slot 1: the program's constants (pb_globals); others: the zero buffer } if (b->kind == GFX_RES_SSBO) { - if ((r->flags & 1) || r->ptr || r->cbuf_slot == VK_GPA_CB_SLOT) return 0; + if ((r->flags & 1) || r->ptr || r->cbuf_slot >= VK_GPA_CB_SLOT) return 0; // the table, the mesh fallback's buffers int ubo, slot = ssbo_slot(r, st, &ubo); if (slot < 0 || slot >= 16) return 0; const p_bind_buf *p = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot]; @@ -3860,8 +3892,13 @@ static void prog_lead(const prog_t *p, int mesh) { } // mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing -// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. +// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. 2: a mesh draw of +// the mesh fallback (mfb_draw): the vertex stage is mfb_cur_vs, in an ordinary pipeline without vertex input. +static const GfxShader *mfb_cur_vs; + static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) { + int fb = mesh == 2; // from here on mesh says what the draw is, and !fb that its pipeline is a mesh pipeline + mesh = mesh != 0; if (ex->render_disabled) return vk_cond_draws_skip += ex->has_render_cond, draw_skipped(ex, mesh, SK_DISABLED); cb_res(ex); dummies_init(); @@ -3892,8 +3929,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int } } else { SM.valid = 0; - vs = shader_for(ex, vst, &has_vs, &vs_wait, 0); - if (mesh) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0); + if (fb) vs = mfb_cur_vs, has_vs = 1; + else vs = shader_for(ex, vst, &has_vs, &vs_wait, 0); + if (mesh && !fb) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0); fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, 0); if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs)) memo_key(ex, gen, &SM.k), SM.vs = vs, SM.fs = fs, SM.has_fs = has_fs, SM.valid = 1; @@ -3927,6 +3965,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int VkPipeState s = ex->pipe; s.vs = vs, s.fs = fs, s.ts = ts; if (mesh) foliage_shadow_cull(ex, &s); + if (fb) memset(s.vattr, 0, sizeof s.vattr), memset(s.vstream, 0, sizeof s.vstream); // whatever the last vertex draw left s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0)); s.prim = (uint8_t)prim; s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0); @@ -3946,7 +3985,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int int fullscreen = 0; // the wait is a full-screen draw's if (oneshot > 0 && oneshot_budget_on()) { budget = frame_wait(); - wait_ms = oneshot_ms(budget, mesh, &s); + wait_ms = oneshot_ms(budget, mesh && !fb, &s); if (wait_ms) t0 = kern_now_ns(); } else if (oneshot > 0) { wait_ms = ONESHOT_WAIT_MS; @@ -3984,7 +4023,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int vk_exec_ahead(ex); // A mesh pipeline that is known to be compiling gets no fast link, so the few milliseconds a draw waits // for one are lost in every frame that meets it until the compile ends, 0.2 to 1.1 s later (MESH_WAIT_MS). - if (wait_ms && linking && mesh && vk_pipeline_building_ms(&s) > MESH_WAIT_MS) wait_ms = 0; + if (wait_ms && linking && mesh && !fb && vk_pipeline_building_ms(&s) > MESH_WAIT_MS) wait_ms = 0; if (wait_ms) p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); } if (t0 && budget) { @@ -3999,7 +4038,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int if (!w0) w0 = kern_now_ns(); if (oneshot_budget_on()) { uint64_t *pool = frame_wait(), b0 = kern_now_ns(); - p = vk_pipeline_graphics_async(&s, &info, &pending, oneshot_ms(pool, mesh, &s)); + p = vk_pipeline_graphics_async(&s, &info, &pending, oneshot_ms(pool, mesh && !fb, &s)); pool_charge(pool, b0); } else { p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS); @@ -4045,7 +4084,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int delta = cfg_bool("NX2NAVI_DYN_DELTA"); if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)"); } - if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh, + if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh && !fb, &vk_exec_frame.dyn_left); else vk_pipeline_set_dynamic(ex->cb, &s); ex->recorded++; @@ -4068,7 +4107,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int if (fbl & ex->fb_dirty) feedback_barrier(ex); feedback_written(ex, &s, fbl); if (!mesh) bind_vertex_buffers(ex); - if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh); + if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh && !fb); if (vk_hdr_track) vk_hdr_draw(ex, info); vk_exec_oq_draw(ex); DRS.issued[mesh]++; @@ -4951,12 +4990,266 @@ static void mesh_shading_rate(VkExec *ex) { ex->recorded++; } +// ---- mesh fallback: a mesh draw on a device without VK_EXT_mesh_shader (mesh_fallback, vk_device.c), or on any +// device with mesh_fallback = 1. The program's task and mesh stages run as compute dispatches that write the +// fallback's buffers, and an ordinary draw reads them (mesh_fb.c makes the modules, vk_mesh_fb.c holds the buffers): +// the draw's record its workgroup count, copied on the GPU when the game's own record holds it +// [task dispatch the task module, with the task stage's bindings +// scan lists the mesh workgroups the tasks started and counts them into the record] +// mesh dispatch the mesh module, with the mesh stage's bindings +// draw one instance per mesh workgroup, with the generated vertex stage and the program's +// fragment stage, the instance count from the record +// Every step reads what the one before wrote, so a full barrier stands between them, and the dispatches cannot be +// recorded inside a pass: each such draw ends the open pass, and the next draw begins it again. The buffers are +// reused by every draw (vk_mesh_fb.c), which the barrier before a draw's first dispatch makes safe: the previous +// draw's vertex stage has read them by then. +// +// The modules of a program are made once, on a thread of their own, with their compute pipelines (a mesh program of +// the game is up to 900,000 SPIR-V words, and its pipeline compiles for as long as the mesh pipeline it replaces): +// until they are ready the program's draws are skipped, like draws whose pipeline is being built. Without background +// pipeline builds (the probes) they are made at the draw. +typedef struct MfbCs { + const GfxShader *ms, *ts; // the key: the translated mesh and task shaders (ts NULL: none) + MeshFbInfo mi, ti; + volatile int32_t state; // 0 queued or being made, 1 ready, -1 failed + const GfxShader *cs_mesh, *cs_task; // the compute modules (gfx_shader_own) + struct MfbCs *next; // the worker's queue +} MfbCs; +typedef struct { const GfxShader *ms; uint32_t mask; uint8_t type[32]; const GfxShader *vs; } MfbVs; +#define MFB_TAB 4096 +static MfbCs *mfb_cs_tab[MFB_TAB]; +static MfbVs mfb_vs_tab[MFB_TAB]; +static struct { + PlatMutex lock; + PlatCond wake; + PlatThread *th; + MfbCs *head, *tail; + uint64_t made, failed, pending_skips; +} MFB = {.lock = PLAT_MUTEX_INIT}; + +static uint32_t mfb_binding_base(const GfxShader *s) { // one past the highest binding the module declares + uint64_t n; + const uint32_t *w = gfx_shader_spirv(s, &n); + uint32_t top = 0; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (!len) break; + if (op == 71 && len == 4 && w[i + 2] == 33 && w[i + 3] > top) top = w[i + 3]; // OpDecorate Binding + i += len; + } + return top + 1; +} + +// The compute module of stage (6 task, 7 mesh) of e, as a shader of the runtime's own with the stage's resources and +// the fallback's buffers. +static const GfxShader *mfb_module(const MfbCs *e, int stage, char *err, size_t cap) { + const GfxShader *src = stage == 6 ? e->ts : e->ms; + uint64_t n, on = 0; + const uint32_t *w = gfx_shader_spirv(src, &n); + uint32_t base = mfb_binding_base(src); + uint32_t *out = stage == 6 ? meshfb_task_cs(w, n, &e->ti, base, &on, err, cap) + : meshfb_mesh_cs(w, n, &e->mi, base, e->ti.payload_bytes / 4, &on, err, cap); + if (!out) return NULL; + static const int task_slots[1] = {MESHFB_CB_TASK}, mesh_slots[3] = {MESHFB_CB_TASK, MESHFB_CB_MESH, MESHFB_CB_LIST}; + char *res = stage == 6 ? meshfb_res_json(gfx_shader_res_json(src), base, task_slots, 1) + : e->mi.has_task ? meshfb_res_json(gfx_shader_res_json(src), base, mesh_slots, 3) + : meshfb_res_json(gfx_shader_res_json(src), base + 1, mesh_slots + 1, 1); + const GfxShader *s = res ? gfx_shader_own(out, on, res) : NULL; + if (!s) snprintf(err, cap, "the converted module or its reflection is malformed"); + free(out), free(res); + return s; +} + +static void mfb_make(MfbCs *e) { + char err[200] = ""; + const VkPipeInfo *info; + int ok = (e->cs_mesh = mfb_module(e, 7, err, sizeof err)) != NULL; + if (ok && e->ts) ok = (e->cs_task = mfb_module(e, 6, err, sizeof err)) != NULL; + if (ok && !vk_pipeline_compute(e->cs_mesh, &info)) ok = 0, snprintf(err, sizeof err, "the mesh stage's compute pipeline failed"); + if (ok && e->ts && !vk_pipeline_compute(e->cs_task, &info)) ok = 0, snprintf(err, sizeof err, "the task stage's compute pipeline failed"); + plat_mutex_lock(&MFB.lock); + uint64_t nth = ok ? ++MFB.made : ++MFB.failed; + plat_mutex_unlock(&MFB.lock); + if (!ok && (nth <= 8 || !(nth & (nth - 1)))) + rt_log("vk: mesh fallback: a program's modules could not be made (%s): its draws are skipped (%llu such programs)", err, + (unsigned long long)nth); + __atomic_store_n(&e->state, ok ? 1 : -1, __ATOMIC_RELEASE); +} + +static void mfb_worker(void *arg) { + (void)arg; + plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); // as the pipeline workers' optimised builds: the game's threads go first + for (;;) { + plat_mutex_lock(&MFB.lock); + while (!MFB.head) plat_cond_wait(&MFB.wake, &MFB.lock); + MfbCs *e = MFB.head; + if (!(MFB.head = e->next)) MFB.tail = NULL; + plat_mutex_unlock(&MFB.lock); + mfb_make(e); + } +} + +// The entry of (ms, ts), made and queued at its first draw; NULL (logged) when the program's layout is not carried. +static MfbCs *mfb_cs_get(VkExec *ex, const GfxShader *ms, const GfxShader *ts) { + uint32_t i = (uint32_t)((((uintptr_t)ms >> 4) * 0x9e3779b1u) ^ (((uintptr_t)ts >> 4) * 0x85ebca6bu)) & (MFB_TAB - 1); + for (uint32_t k = 0; k < MFB_TAB; k++, i = (i + 1) & (MFB_TAB - 1)) { + MfbCs *e = mfb_cs_tab[i]; + if (e && (e->ms != ms || e->ts != ts)) continue; + if (e) return e; + const uint8_t *ctl, *code; + uint32_t len; + char err[160] = ""; + e = calloc(1, sizeof *e); + if (!e) rt_fatal("vk: out of memory"); + e->ms = ms, e->ts = ts, e->state = -1; + mfb_cs_tab[i] = e; // a program that is refused is not looked at again + int ok = find_stage_scan(ex->prog, NVN_STAGE_MESH, &ctl, &code, &len) && len >= 0x80 && + meshfb_info(ctl, code, 7, &e->mi, err, sizeof err); + if (ok && ts) ok = find_stage_scan(ex->prog, NVN_STAGE_TASK, &ctl, &code, &len) && meshfb_info(ctl, code, 6, &e->ti, err, sizeof err); + if (ok && e->mi.has_task != (ts != NULL)) ok = 0, snprintf(err, sizeof err, "the mesh stage %s a task stage and the program has %s", + e->mi.has_task ? "expects" : "does not expect", ts ? "one" : "none"); + if (!ok) { + warn_once(W_MESH, "mesh fallback: a program is not carried (%s): its draws are skipped", *err ? err : "no stage data"); + return e; + } + e->state = 0; + if (!vk_pipeline_async_on()) { + mfb_make(e); + return e; + } + plat_mutex_lock(&MFB.lock); + if (!MFB.th) plat_cond_init(&MFB.wake), MFB.th = plat_thread_new(mfb_worker, NULL, "mesh fallback"); + if (MFB.tail) MFB.tail->next = e; + else MFB.head = e; + MFB.tail = e; + plat_cond_signal(&MFB.wake); + plat_mutex_unlock(&MFB.lock); + return e; + } + rt_fatal("vk: mesh fallback: more than %d task and mesh program pairs", MFB_TAB); +} + +// The vertex stage that draws e's mesh buffer slots for fragment shader fs: an output for every input fs reads, in +// its type. +static const GfxShader *mfb_vs_get(const MfbCs *e, const GfxShader *fs) { + GfxShaderIo fin[32]; + uint8_t type[32] = {0}; + uint32_t mask = fs ? gfx_shader_inputs(fs, fin) : 0; + for (int g = 0; g < 32; g++) + if (mask >> g & 1) type[g] = fin[g].type; + uint32_t i = (uint32_t)((((uintptr_t)e->ms >> 4) * 0x9e3779b1u) ^ (mask * 0x85ebca6bu)) & (MFB_TAB - 1); + for (uint32_t k = 0; k < MFB_TAB; k++, i = (i + 1) & (MFB_TAB - 1)) { + MfbVs *v = &mfb_vs_tab[i]; + if (v->ms && (v->ms != e->ms || v->mask != mask || memcmp(v->type, type, 32))) continue; + if (v->ms) return v->vs; + static const char res[] = "{\"key\":\"meshfb\",\"stage\":\"vs\",\"notes\":[],\"cbufs\":[],\"sbufs\":[{\"binding\": 1, " + "\"slot\": -1, \"cb_slot\": 66, \"cb_offset\": 0, \"flags\": 0}],\"textures\":[],\"images\":[]}"; + _Static_assert(MESHFB_CB_MESH == 66, "the vertex stage's reflection names the mesh buffer"); + uint64_t n = 0; + uint32_t *w = meshfb_vs(&e->mi, mask, type, &n); + v->ms = e->ms, v->mask = mask, memcpy(v->type, type, 32); + v->vs = w ? gfx_shader_own(w, n, res) : NULL; + free(w); + if (!v->vs) warn_once(W_MESH, "mesh fallback: a vertex stage could not be made: its draws are skipped"); + return v->vs; + } + rt_fatal("vk: mesh fallback: more than %d vertex stages", MFB_TAB); +} + +// One of the draw's compute dispatches: module cs with the bindings of NVN stage `stage`, its group counts read from +// the record at `at`. 0 when its pipeline is not there. +static int mfb_dispatch(VkExec *ex, const GfxShader *cs, int stage, VkBuffer rec, VkDeviceSize at) { + const VkPipeInfo *info = NULL; + VkPipeline p = vk_pipeline_compute(cs, &info); + if (!p || !info || (info->pool && !pool_on())) return 0; + bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); + mfb_stage = stage; + bind_resources(ex, info); + mfb_stage = 0; + VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0); + vkCmdDispatchIndirect(ex->cb, rec, at); + vk_pass_mark_end(ex->cb, m); + ex->recorded++; + return 1; +} + +// A mesh draw of `count` workgroups, or of the count in the game's record at ind (4 bytes, GPU-written). +static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { + if (ex->render_disabled) { + vk_cond_draws_skip += ex->has_render_cond; + draw_skipped(ex, 1, SK_DISABLED); + return; + } + if (!ex->prog) { draw_skipped(ex, 1, SK_NO_PROGRAM); return; } + cb_res(ex); + dummies_init(); + int has_ms, has_ts, has_fs, w_ms = 0, w_ts = 0, w_fs = 0; + const GfxShader *ms = shader_for(ex, NVN_STAGE_MESH, &has_ms, &w_ms, 0); + const GfxShader *ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &w_ts, 0); + const GfxShader *fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &w_fs, 0); + if (!has_ms) { draw_skipped(ex, 1, SK_NO_VERTEX_STAGE); return; } + if (w_ms || w_ts || w_fs) { + vk_exec_ahead(ex); + perf_count(PERF_CNT_SHADER_SKIP); + draw_skipped(ex, 1, SK_SHADER_PENDING); + return; + } + if (!ms || (has_ts && !ts) || (has_fs && !fs)) { draw_skipped(ex, 1, SK_SHADER_FAILED); return; } + MfbCs *c = mfb_cs_get(ex, ms, ts); + int state = __atomic_load_n(&c->state, __ATOMIC_ACQUIRE); + if (!state && !vk_exec_targets_recent(ex, pipe_skip_mode() >= 1)) { + // a target the game does not render every frame: the draw may not come again, so it waits for its modules + // from the frame's wait pool, as such a draw waits for a pipeline (prepare_draw_ex) + uint64_t *pool = frame_wait(), t0 = kern_now_ns(); + while (!state && pool_ms(pool, t0)) plat_sleep_ms(1), state = __atomic_load_n(&c->state, __ATOMIC_ACQUIRE); + pool_charge(pool, t0); + } + if (!state) { MFB.pending_skips++; draw_skipped(ex, 1, SK_PIPE_PENDING); return; } + const GfxShader *vs = state > 0 ? mfb_vs_get(c, fs) : NULL; + if (!vs || !vk_meshfb_init()) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; } + uint32_t verts = c->mi.maxp * c->mi.ipp, slot_words = meshfb_slot_words(&c->mi); + uint32_t most = vk_ctx()->props.limits.maxComputeWorkGroupCount[0]; + if (count > most) count = most; // a count the GPU wrote is not looked at here: the dispatch reads it as it is + VkBuffer rec; + int cmds; + vk_exec_barrier(ex); // ends the pass; the culling's record and the last draw's reads come before what follows + VkDeviceSize at = vk_meshfb_record(ex->cb, ind ? ind->buffer : VK_NULL_HANDLE, ind ? ind->offset : 0, count, ts != NULL, + verts, slot_words, &rec, &cmds); + ex->recorded += (uint32_t)cmds; + vk_exec_barrier(ex); + if (ts) { + if (!mfb_dispatch(ex, c->cs_task, NVN_STAGE_TASK, rec, at)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; } + vk_exec_barrier(ex); + vk_meshfb_scan(ex->cb, at, c->ti.payload_bytes / 4, c->ti.threads, slot_words, verts); + B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own + ex->recorded += 4; + vk_exec_barrier(ex); + } + if (!mfb_dispatch(ex, c->cs_mesh, NVN_STAGE_MESH, rec, at + 16)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; } + vk_exec_barrier(ex); + vk_meshfb_note(ex->cb, at); + ex->recorded++; + vk_exec_barrier(ex); + c0_binst = 0; + mfb_cur_vs = vs; + int prim = c->mi.ipp == 1 ? NVN_DRAW_PRIMITIVE_POINTS : c->mi.ipp == 2 ? NVN_DRAW_PRIMITIVE_LINES : NVN_DRAW_PRIMITIVE_TRIANGLES; + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 2)) return; + vk_exec_frame.draws++; + vkCmdDrawIndirect(ex->cb, rec, at + 32, 1, sizeof(VkDrawIndirectCommand)); + ex->recorded++; +} + static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { - PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); - if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis) if (!d->count || !on) return; + if (vk_ctx()->mesh_fallback) { + if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first); + mfb_draw(ex, d->count, NULL); + return; + } + PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); + if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); c0_binst = 0; if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first); @@ -4980,13 +5273,21 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { // MESH_RING entries outlast the frames in flight: an entry is rewritten only after a barrier behind its last draw. enum { MESH_RING = 4096 }; static VkBuffer mesh_ring; +static VmaAllocation mesh_ring_alloc; static uint32_t mesh_ring_next; // NX2NAVI_GPU_PASSES: the workgroup counts of the indirect mesh draws of the timed frames, copied to host memory. int vk_gpu_passes_capturing(void); enum { MESH_COUNTS = 4096 }; static VkBuffer mesh_count_buf; +static VmaAllocation mesh_count_alloc; static uint32_t *mesh_count_map, mesh_count_n, mesh_count_was; + +static void mesh_ring_release(VkCtx *c) { // vk_exec_draw_shutdown: a probe ends with the device idle + if (mesh_ring) vmaDestroyBuffer(c->vma, mesh_ring, mesh_ring_alloc); + if (mesh_count_buf) vmaDestroyBuffer(c->vma, mesh_count_buf, mesh_count_alloc); + mesh_ring = mesh_count_buf = VK_NULL_HANDLE, mesh_count_map = NULL; +} uint32_t vk_exec_mesh_counts(const uint32_t **out) { *out = mesh_count_map; return mesh_count_map ? mesh_count_n : 0; @@ -5001,8 +5302,7 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; VmaAllocationInfo info; - VmaAllocation a; - if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_count_buf, &a, &info) != VK_SUCCESS) return; + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_count_buf, &mesh_count_alloc, &info) != VK_SUCCESS) return; mesh_count_map = info.pMappedData; } if (mesh_count_n >= MESH_COUNTS) return; @@ -5053,8 +5353,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING, .usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; - VmaAllocation a; - if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &a, NULL) != VK_SUCCESS) { + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &mesh_ring_alloc, NULL) != VK_SUCCESS) { mesh_ring = VK_NULL_HANDLE; return 0; } @@ -5107,7 +5406,8 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { if (vk_ctx()->has_mesh_shader) fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT"); } - if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); + int fallback = vk_ctx()->mesh_fallback; + if (!fn && !fallback) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); if (!on) return; @@ -5117,6 +5417,12 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { (unsigned long long)d->ind); c0_binst = 0; VK_MEM_USE(ip); + if (fallback) { + mfb_draw(ex, 0, &ip); + vk_mem_count(ip.cls, VK_MUSE_IND, 4); + VK_MEM_BOUND(ip); + return; + } VkDeviceSize at; if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; diff --git a/runtime/vk_mesh_fb.c b/runtime/vk_mesh_fb.c new file mode 100644 index 00000000..e0939f54 --- /dev/null +++ b/runtime/vk_mesh_fb.c @@ -0,0 +1,215 @@ +// The mesh fallback's buffers and records. A task or mesh program runs as a compute dispatch that writes a storage +// buffer (mesh_fb.c), and the draw reads it: +// task buffer a task workgroup's payload at its index, payload-size apart +// list per mesh workgroup after a task stage: its task and its index among that task's (meshfb_scan.comp) +// mesh buffer a mesh workgroup's vertex lines and primitive area at its slot +// records per draw 64 bytes: the indirect arguments of its task dispatch, its mesh dispatch and its draw +// The three buffers are reused by every draw: a draw's dispatches write them and its vertex stage has read them +// before the next draw's dispatches run, with a barrier in between (vk_exec_draw.c mfb_draw), so their size is what +// the largest draw needs, not what a frame's draws need together. mesh_fallback_mb sets it: an eighth for the task +// buffer, the rest for the mesh buffer. Workgroups past either buffer's end write nothing and draw nothing (the +// modules check the buffer's length); the draws that happened to are counted and the log names the size they needed. +// A record is used by one draw and written again only after MESHFB_RECS further draws. +#include +#include +#include + +#include "cfg.h" +#include "host.h" +#include "mesh_fb.h" +#include "vk.h" +#include "vk_mesh_fb.h" +#include "meshfb_scan_spv.h" + +enum { MESHFB_RECS = 4096, REC_BYTES = 64 }; + +static struct { + int tried, ok; + VkBuffer task, mesh, list, rec, stat; + VmaAllocation task_a, mesh_a, list_a, rec_a, stat_a; + VkDeviceSize task_bytes, mesh_bytes, list_bytes; // as one storage binding spans them + const uint32_t *stat_map; // per record: mesh workgroups and task workgroups it asked for + uint32_t cap_slots[MESHFB_RECS], slot_words[MESHFB_RECS], cap_tasks[MESHFB_RECS]; // what the record's draw had room for + uint8_t noted[MESHFB_RECS]; + uint32_t next; + uint64_t draws, short_draws, worst_bytes; + PFN_vkCmdPushDescriptorSetKHR push; + VkDescriptorSetLayout dsl; + VkPipelineLayout layout; + VkPipeline scan; +} M; + +static int make_buffer(VkCtx *c, VkDeviceSize size, VkBufferUsageFlags usage, int host, VkBuffer *b, VmaAllocation *a, void **map) { + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size, .usage = usage}; + VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, + .flags = host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT : 0}; + VmaAllocationInfo info; + if (vmaCreateBuffer(c->vma, &bi, &ai, b, a, &info) != VK_SUCCESS) return *b = VK_NULL_HANDLE, 0; + if (map) *map = info.pMappedData; + return !host || info.pMappedData; +} + +int vk_meshfb_init(void) { + if (M.tried) return M.ok; + M.tried = 1; + VkCtx *c = vk_ctx(); + if (!c) return 0; + uint64_t mb = (uint64_t)cfg_int("NX2NAVI_MESH_FALLBACK_MB"); + if (mb < 16) mb = 16; + if (mb > 4096) mb = 4096; + VkDeviceSize range = c->props.limits.maxStorageBufferRange & ~(VkDeviceSize)3, task = (mb << 20) / 8, mesh = (mb << 20) - task; + VkDeviceSize list = mesh / 128; // an entry per slot of 512 bytes, and no mesh program's slot is smaller + void *map = NULL; + const VkBufferUsageFlags st = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + if (!make_buffer(c, task, st, 0, &M.task, &M.task_a, NULL) || !make_buffer(c, mesh, st, 0, &M.mesh, &M.mesh_a, NULL) || + !make_buffer(c, list, st, 0, &M.list, &M.list_a, NULL) || + !make_buffer(c, (VkDeviceSize)MESHFB_RECS * REC_BYTES, + st | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT, 0, + &M.rec, &M.rec_a, NULL) || + !make_buffer(c, (VkDeviceSize)MESHFB_RECS * 8, VK_BUFFER_USAGE_TRANSFER_DST_BIT, 1, &M.stat, &M.stat_a, &map)) { + rt_log("vk: mesh fallback: its buffers (%llu MiB, mesh_fallback_mb) could not be allocated: task and mesh draws are skipped", + (unsigned long long)mb); + return 0; + } + M.stat_map = map; + memset(map, 0, (size_t)MESHFB_RECS * 8); + M.task_bytes = task < range ? task : range, M.mesh_bytes = mesh < range ? mesh : range, M.list_bytes = list < range ? list : range; + + M.push = (PFN_vkCmdPushDescriptorSetKHR)vk_dpa(c->device, "vkCmdPushDescriptorSetKHR"); + VkDescriptorSetLayoutBinding b[4]; + for (int i = 0; i < 4; i++) + b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}; + VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, + .bindingCount = 4, .pBindings = b}; + VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 24}; + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof meshfb_scan_spv, + .pCode = meshfb_scan_spv}; + VkShaderModule m = VK_NULL_HANDLE; + if (!M.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &M.dsl) != VK_SUCCESS) goto fail; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, + .pSetLayouts = &M.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr}; + if (vkCreatePipelineLayout(c->device, &li, NULL, &M.layout) != VK_SUCCESS || + vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS) + goto fail; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = M.layout}; + VkResult res = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &M.scan); + vkDestroyShaderModule(c->device, m, NULL); + if (res != VK_SUCCESS) goto fail; + rt_log("vk: mesh fallback: %llu MiB for task payloads, %llu MiB for mesh output (mesh_fallback_mb = %llu)", + (unsigned long long)(M.task_bytes >> 20), (unsigned long long)(M.mesh_bytes >> 20), (unsigned long long)mb); + return M.ok = 1; +fail: + rt_log("vk: mesh fallback: the scan pipeline could not be created: task and mesh draws are skipped"); + return 0; +} + +VkDescriptorBufferInfo vk_meshfb_buffer(int cb_slot) { + if (cb_slot == MESHFB_CB_TASK) return (VkDescriptorBufferInfo){M.task, 0, M.task_bytes}; + if (cb_slot == MESHFB_CB_LIST) return (VkDescriptorBufferInfo){M.list, 0, M.list_bytes}; + return (VkDescriptorBufferInfo){M.mesh, 0, M.mesh_bytes}; +} + +uint64_t vk_meshfb_mesh_words(void) { return M.mesh_bytes / 4; } + +// The record about to be written again: what its last draw asked for against what it had room for. +static void record_check(uint32_t i) { + if (!M.noted[i]) return; + M.noted[i] = 0; + uint32_t want = M.stat_map[2 * i], tasks = M.stat_map[2 * i + 1]; + if (want <= M.cap_slots[i] && tasks <= M.cap_tasks[i]) return; + uint64_t need = (uint64_t)want * M.slot_words[i] * 4; + if (need > M.worst_bytes) M.worst_bytes = need; + uint64_t n = ++M.short_draws; + if (n <= 4 || !(n & (n - 1))) { + char from[96] = ""; + if (tasks) snprintf(from, sizeof from, " from %u task workgroups (room for %u)", tasks, M.cap_tasks[i]); + rt_log("vk: mesh fallback: a draw's programs emitted %u mesh workgroups of %u bytes%s, and the mesh buffer holds %u: " + "what did not fit was not drawn (%llu such draws of %llu; mesh_fallback_mb of %llu or more would hold the " + "largest so far)", want, M.slot_words[i] * 4, from, M.cap_slots[i], (unsigned long long)n, + (unsigned long long)M.draws, (unsigned long long)((M.worst_bytes * 8 / 7 >> 20) + 1)); + } +} + +VkDeviceSize vk_meshfb_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, uint32_t count, int has_task, + uint32_t verts, uint32_t slot_words, VkBuffer *rec, int *cmds) { + uint32_t i = M.next++ % MESHFB_RECS; + VkDeviceSize at = (VkDeviceSize)i * REC_BYTES; + record_check(i); + M.draws++; + M.slot_words[i] = slot_words, M.cap_slots[i] = (uint32_t)(M.mesh_bytes / 4 / (slot_words ? slot_words : 1)); + M.cap_tasks[i] = 0xffffffffu; // vk_meshfb_scan knows the payload size + if (has_task && M.cap_slots[i] > M.list_bytes / 4) M.cap_slots[i] = (uint32_t)(M.list_bytes / 4); + *rec = M.rec; + int n = 0; + vkCmdFillBuffer(cb, M.rec, at, REC_BYTES, 1), n++; + // where the workgroup count goes: the task dispatch, or the mesh dispatch, the draw's instances and the report + VkDeviceSize to[3] = {has_task ? at : at + 16, at + 36, at + 48}; + int nto = has_task ? 1 : 3; + if (!has_task) { + vkCmdFillBuffer(cb, M.rec, at + 32, 4, verts), n++; + vkCmdFillBuffer(cb, M.rec, at + 40, 8, 0), n++; + vkCmdFillBuffer(cb, M.rec, at + 52, 4, 0), n++; + } + if (src) { + VkBufferCopy cp[3]; + for (int k = 0; k < nto; k++) cp[k] = (VkBufferCopy){src_off, to[k], 4}; + vkCmdCopyBuffer(cb, src, M.rec, (uint32_t)nto, cp), n++; + } else { + for (int k = 0; k < nto; k++) vkCmdFillBuffer(cb, M.rec, to[k], 4, count), n++; + } + *cmds = n; + return at; +} + +void vk_meshfb_scan(VkCommandBuffer cb, VkDeviceSize at, uint32_t task_words, uint32_t fan, uint32_t slot_words, uint32_t verts) { + uint32_t i = (uint32_t)(at / REC_BYTES); + M.cap_tasks[i] = (uint32_t)(M.task_bytes / 4 / (task_words ? task_words : 1)); + VkDescriptorBufferInfo bi[4] = {{M.rec, 0, VK_WHOLE_SIZE}, {M.task, 0, M.task_bytes}, {M.list, 0, M.list_bytes}, + {M.mesh, 0, M.mesh_bytes}}; + VkWriteDescriptorSet w[4]; + for (int k = 0; k < 4; k++) + w[k] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)k, + .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .pBufferInfo = &bi[k]}; + uint32_t pc[6] = {(uint32_t)(at / 4), task_words, fan, slot_words, verts, vk_ctx()->props.limits.maxComputeWorkGroupCount[0]}; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, M.scan); + rec_push(M.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, M.layout, 0, 4, w); + vkCmdPushConstants(cb, M.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); + vkCmdDispatch(cb, 1, 1, 1); +} + +void vk_meshfb_note(VkCommandBuffer cb, VkDeviceSize at) { + uint32_t i = (uint32_t)(at / REC_BYTES); + VkBufferCopy cp = {at + 48, (VkDeviceSize)i * 8, 8}; + vkCmdCopyBuffer(cb, M.rec, M.stat, 1, &cp); + M.noted[i] = 1; +} + +uint64_t vk_meshfb_short(void) { + if (!M.ok) return 0; + for (uint32_t i = 0; i < MESHFB_RECS; i++) record_check(i); + return M.short_draws; +} + +void vk_meshfb_shutdown(void) { + VkCtx *c = vk_ctx(); + if (!c || !M.tried) return; + if (vk_meshfb_short()) + rt_log("vk: mesh fallback: %llu of %llu draws emitted more than the buffers hold (mesh_fallback_mb of %llu or more would " + "hold the largest)", (unsigned long long)M.short_draws, (unsigned long long)M.draws, + (unsigned long long)((M.worst_bytes * 8 / 7 >> 20) + 1)); + if (M.scan) vkDestroyPipeline(c->device, M.scan, NULL); + if (M.layout) vkDestroyPipelineLayout(c->device, M.layout, NULL); + if (M.dsl) vkDestroyDescriptorSetLayout(c->device, M.dsl, NULL); + if (M.task) vmaDestroyBuffer(c->vma, M.task, M.task_a); + if (M.mesh) vmaDestroyBuffer(c->vma, M.mesh, M.mesh_a); + if (M.list) vmaDestroyBuffer(c->vma, M.list, M.list_a); + if (M.rec) vmaDestroyBuffer(c->vma, M.rec, M.rec_a); + if (M.stat) vmaDestroyBuffer(c->vma, M.stat, M.stat_a); + memset(&M, 0, sizeof M); +} diff --git a/runtime/vk_mesh_fb.h b/runtime/vk_mesh_fb.h new file mode 100644 index 00000000..8012c510 --- /dev/null +++ b/runtime/vk_mesh_fb.h @@ -0,0 +1,29 @@ +// The mesh fallback's buffers and its records (vk_mesh_fb.c); the modules are mesh_fb.c's, the draws +// vk_exec_draw.c's ("mesh fallback"). +#pragma once +#include "vk.h" + +// Creates the buffers and the scan pipeline on first use. 0 when that failed (logged once): the draws are skipped. +int vk_meshfb_init(void); +// The buffer a module's extra storage binding reads or writes (mesh_fb.h MESHFB_CB_*), whole or as much of it as one +// storage binding may span. +VkDescriptorBufferInfo vk_meshfb_buffer(int cb_slot); +uint64_t vk_meshfb_mesh_words(void); // the mesh buffer's size as a binding sees it, in words + +// A draw's record, 64 bytes of the record buffer (*rec) at the offset returned: the task dispatch at +0, the mesh +// dispatch at +16, the draw (VkDrawIndirectCommand) at +32. Records the fills and copies that start it, outside a +// pass: the workgroup count from 4 bytes at src_off of src, or `count` when src is VK_NULL_HANDLE. Without a task +// stage that count is the mesh dispatch and the draw's instance count; with one it is the task dispatch, and +// vk_meshfb_scan writes the rest. slot_words: of the draw's mesh program, for the report of a draw the mesh buffer +// was too small for. Returns the number of commands recorded in *cmds. +VkDeviceSize vk_meshfb_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, uint32_t count, int has_task, + uint32_t verts, uint32_t slot_words, VkBuffer *rec, int *cmds); +// The pass between the task and the mesh dispatch (shaders/meshfb_scan.comp): binds its own compute pipeline and set. +void vk_meshfb_scan(VkCommandBuffer cb, VkDeviceSize at, uint32_t task_words, uint32_t fan, uint32_t slot_words, uint32_t verts); +// After the record is complete (the copies or the scan, and a barrier): keeps what the draw asked for where the host +// reads it later. One copy command. +void vk_meshfb_note(VkCommandBuffer cb, VkDeviceSize at); +// Draws so far whose programs emitted more than the buffers hold, the records not looked at yet included: call it +// with the device idle (a probe, the shutdown). +uint64_t vk_meshfb_short(void); +void vk_meshfb_shutdown(void); // the device must be idle diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index cce67e01..d05cc29b 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -3427,6 +3427,7 @@ static void keys_note(const VkPipeState *s) { static const uint8_t none[32]; if (s->fs && !memcmp(gfx_shader_input_id(s->fs), none, 32)) return; // the runtime's own shader (gfx_shader_own) if (s->ts && !memcmp(gfx_shader_input_id(s->ts), none, 32)) return; + if (!memcmp(gfx_shader_input_id(s->vs), none, 32)) return; // the mesh fallback's vertex stage (vk_exec_draw.c mfb_vs_get) KeyRec r; rec_fill(&r, s); if (!seen_add(rec_hash(&r))) return; diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index a06f0e97..97098330 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -49,7 +49,7 @@ enum { R_SetRasterizerDiscardEnable, R_SetScissorWithCount, R_SetStencilCompareMask, R_SetStencilOp, R_SetStencilReference, R_SetStencilTestEnable, R_SetStencilWriteMask, R_SetViewportWithCount, R_WriteTimestamp, R_WriteTimestamp2, R_PushTpl, R_Push, R_VertexInput, R_Bool, R_PolygonMode, R_BlendEnable, R_BlendEquation, R_WriteMask, R_EndCb, - R_PassMark, R_DrawMesh, R_DrawMeshInd, R_ShadingRate, + R_PassMark, R_DrawMesh, R_DrawMeshInd, R_ShadingRate, R_DrawIndirect, }; static uint64_t now_ns(void) { return plat_now_ns(); } @@ -347,6 +347,9 @@ FIXED(DrawIndexed, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, int3 #define PACK_DrawIndexedIndirect *a = (A_DrawIndexedIndirect){b, o, n, s} FIXED(DrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s), VkBuffer b; VkDeviceSize o; uint32_t n, s) +#define PACK_DrawIndirect *a = (A_DrawIndirect){b, o, n, s} +FIXED(DrawIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s), + VkBuffer b; VkDeviceSize o; uint32_t n, s) #define PACK_EndQuery *a = (A_EndQuery){p, q} FIXED(EndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), (cb, p, q), VkQueryPool p; uint32_t q) #define PACK_EndRendering (void)a @@ -705,6 +708,7 @@ static void exec_packet(uint32_t op, void *p) { F(Draw, a->n, a->i, a->f, a->fi) F(DrawIndexed, a->n, a->i, a->f, a->v, a->fi) F(DrawIndexedIndirect, a->b, a->o, a->n, a->s) + F(DrawIndirect, a->b, a->o, a->n, a->s) F(EndQuery, a->p, a->q) F(FillBuffer, a->b, a->o, a->n, a->v) F(ResetQueryPool, a->p, a->f, a->n) diff --git a/runtime/vk_rec.h b/runtime/vk_rec.h index 9261ee16..62f8c440 100644 --- a/runtime/vk_rec.h +++ b/runtime/vk_rec.h @@ -80,6 +80,7 @@ void rec_pass_mark(VkCommandBuffer cb, uint32_t id, int end); X(Draw, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, uint32_t fi), (cb, n, i, f, fi)) \ X(DrawIndexed, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, int32_t v, uint32_t fi), (cb, n, i, f, v, fi)) \ X(DrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s)) \ + X(DrawIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s)) \ X(EndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), (cb, p, q)) \ X(EndRendering, (VkCommandBuffer cb), (cb)) \ X(FillBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkDeviceSize n, uint32_t v), (cb, b, o, n, v)) \ @@ -137,6 +138,7 @@ REC_CMDS(REC_DECL) #define vkCmdDraw rc_vkCmdDraw #define vkCmdDrawIndexed rc_vkCmdDrawIndexed #define vkCmdDrawIndexedIndirect rc_vkCmdDrawIndexedIndirect +#define vkCmdDrawIndirect rc_vkCmdDrawIndirect #define vkCmdEndQuery rc_vkCmdEndQuery #define vkCmdEndRendering rc_vkCmdEndRendering #define vkCmdFillBuffer rc_vkCmdFillBuffer