Draw task and mesh programs through compute passes when the mesh fallback is on.

A mesh draw becomes its record (the workgroup count, copied on the GPU for an indirect draw), the task dispatch, a
scan that lists the mesh workgroups the tasks started, the mesh dispatch, and an instanced indirect draw with a
generated vertex stage and the program's fragment stage. The compute modules bind the resources of the task and mesh
stages, and their pipelines are made on a thread of their own while the program's draws are skipped.

The indirect mesh ring and the texture pool marks are released at a probe's shutdown.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 05:58:42 -05:00
parent a3da8569c8
commit a8c6e992ee
9 changed files with 624 additions and 17 deletions

View File

@@ -100,6 +100,7 @@ size_t gfx_shader_warm_list(GfxWarmEnt **out); // malloc'd; recorde
const GfxShader *gfx_shader_load_private(const uint8_t iid[32], int *owned);
void gfx_shader_release_private(const GfxShader *s);
const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words);
const char *gfx_shader_res_json(const GfxShader *s); // the translator's .res.json text
int gfx_shader_write_code(const GfxShader *s, const char *path); // the guest program, from the stored inputs
// A shader of the runtime's own (SPIR-V and a .res.json text as the translator writes them), outside the cache and
// never freed; its key is the sha256 of both. NULL when the SPIR-V is malformed.

View File

@@ -2424,6 +2424,8 @@ const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words) {
return s->spirv;
}
const char *gfx_shader_res_json(const GfxShader *s) { return s->reflection; }
// The guest program a shader was translated from, as its stored inputs hold it (the whole program: NVN2 code from its
// first byte), written to path for a disassembler (NX2NAVI_DUMP's sh_<hash>.code), and next to it <path>.job, the
// translator request that translates it (one line for nx2navi_xlate --serve; its "out" names the shader cache, and

View File

@@ -0,0 +1,47 @@
#version 450
// The mesh fallback's pass between a draw's task dispatch and its mesh dispatch (vk_mesh_fb.c vk_meshfb_scan,
// mesh_fb.c): every task workgroup wrote how many mesh workgroups it starts into its payload (word 1). This lists them
// in task order, so that mesh workgroup g finds its task and its index among that task's in list entry g, and writes
// the draw's record: how many mesh workgroups to dispatch and how many instances to draw.
//
// One thread walks the tasks: the order of the list is the order the primitives are drawn in, which has to be the
// same in every frame and in the depth and colour passes of one frame.
layout(local_size_x = 1) in;
layout(push_constant) uniform Params {
uint rec; // the record's first word in the record buffer
uint task_words; // words of a task workgroup's payload in the task buffer
uint fan; // mesh workgroups a task workgroup starts at most
uint slot_words; // words of a mesh workgroup's output in the mesh buffer
uint verts; // vertices drawn per mesh workgroup
uint max_groups; // workgroups one dispatch may have (maxComputeWorkGroupCount)
} pc;
// The record: words 0-2 the task dispatch (x written before this pass), 4-6 the mesh dispatch, 8-11 the draw (vertices,
// instances, first vertex, first instance), 12 the mesh workgroups the tasks asked for, 13 the task workgroups.
layout(set = 0, binding = 0) buffer Rec { uint w[]; } rec;
layout(set = 0, binding = 1) readonly buffer Task { uint w[]; } task;
layout(set = 0, binding = 2) writeonly buffer List { uint e[]; } list;
layout(set = 0, binding = 3) readonly buffer Mesh { uint w[]; } mesh; // read for its size alone
void main() {
uint asked = rec.w[pc.rec];
uint tasks = min(min(asked, uint(task.w.length()) / pc.task_words), 1u << 24);
uint room = min(min(uint(mesh.w.length()) / pc.slot_words, uint(list.e.length())), pc.max_groups);
uint fan = min(pc.fan, 255u);
uint m = 0u, want = 0u;
for (uint t = 0u; t < tasks; t++) {
uint c = min(task.w[t * pc.task_words + 1u], fan);
want += c;
for (uint j = 0u; j < c && m < room; j++) list.e[m++] = t | j << 24;
}
rec.w[pc.rec + 4u] = m;
rec.w[pc.rec + 5u] = 1u;
rec.w[pc.rec + 6u] = 1u;
rec.w[pc.rec + 8u] = pc.verts;
rec.w[pc.rec + 9u] = m;
rec.w[pc.rec + 10u] = 0u;
rec.w[pc.rec + 11u] = 0u;
rec.w[pc.rec + 12u] = want;
rec.w[pc.rec + 13u] = asked;
}

View File

@@ -29,6 +29,8 @@
#endif
#include "vk_exec.h"
#include "vk_gpa.h"
#include "mesh_fb.h"
#include "vk_mesh_fb.h"
#include "platform/platform.h"
#include "vk_dlaa.h"
@@ -446,7 +448,12 @@ static void twins_release(VkCtx *c);
// Releases the stand-ins and the depth twins. The device must be idle; the game path leaves them to process exit like
// the pipelines.
static void mesh_ring_release(VkCtx *c);
static void pool_marks_release(VkCtx *c);
void vk_exec_draw_shutdown(void) {
vk_meshfb_shutdown();
if (vk_ctx()) mesh_ring_release(vk_ctx()), pool_marks_release(vk_ctx()); // made without the stand-ins below too
if (!Z.ready) return;
VkCtx *c = vk_ctx();
twins_release(c);
@@ -948,6 +955,13 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G
// and *pending = 1 says the wait ended first.
static int sf_mvprev; // shader_for asks for the vertex shader's last-frame variant (per-object motion)
// Probes only (mesh_fb_probe.c): the shader of the program stage whose control section is ctl, in place of a
// translation. The probes build their programs from modules made here, which no translator has an input for.
static struct { const uint8_t *ctl; const GfxShader *s; } probe_sh[16];
static int n_probe_sh;
void vk_exec_probe_shader(const uint8_t *ctl, const GfxShader *s) {
if (n_probe_sh < 16) probe_sh[n_probe_sh].ctl = ctl, probe_sh[n_probe_sh].s = s, n_probe_sh++;
}
static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pending, uint32_t await_ms) {
const uint8_t *ctl, *code;
uint32_t len;
@@ -955,6 +969,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
if (pending) *pending = 0;
if (!ex->prog || !find_stage(ex->prog, st, &ctl, &code, &len)) return NULL;
*present = 1;
for (int i = 0; i < n_probe_sh; i++)
if (probe_sh[i].ctl == ctl) return probe_sh[i].s;
GfxShaderAnswers a;
uint64_t serial = ex->prog->serial;
answers_for(ex, st, ctl, slot_mask_of(serial, st), &a);
@@ -994,7 +1010,12 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
// ---- descriptors
// The mesh fallback (below, "mesh fallback") runs a program's task and mesh stages as compute modules: while it binds
// one's resources, a compute module's bindings are those of this NVN stage (6 task, 7 mesh), not of the compute stage.
static int mfb_stage;
static int nvn_stage_of(int gfx_stage) {
if (gfx_stage == GFX_STAGE_CS && mfb_stage) return mfb_stage;
return gfx_stage == GFX_STAGE_VS ? NVN_STAGE_VS : gfx_stage == GFX_STAGE_FS ? NVN_STAGE_FS
: gfx_stage == GFX_STAGE_TS ? NVN_STAGE_TASK : gfx_stage == GFX_STAGE_MS ? NVN_STAGE_MESH : NVN_STAGE_CS;
}
@@ -1373,6 +1394,8 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *
VkDescriptorBufferInfo t = vk_gpa_info();
return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE};
}
// The mesh fallback's own buffers (mesh_fb.c): no guest memory behind them
if (r->cbuf_slot >= MESHFB_CB_TASK && r->cbuf_slot <= MESHFB_CB_LIST) return vk_meshfb_buffer(r->cbuf_slot);
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
uint64_t max = LIM.sbo_max, align = LIM.sbo_align;
VkDescriptorBufferInfo bi;
@@ -1908,6 +1931,15 @@ static void pool_marks_make(void) {
__atomic_store_n(&PL.marks_ready, 1, __ATOMIC_RELEASE); // the recording thread reads the rest after this
}
static void pool_marks_release(VkCtx *c) { // vk_exec_draw_shutdown: a probe ends with the device idle
if (!PL.marks) return;
__atomic_store_n(&PL.marks_ready, 0, __ATOMIC_RELEASE);
vmaDestroyBuffer(c->vma, PL.marks, PL.marks_alloc);
for (int i = 0; i < MARKS_RB; i++)
if (PL.rb[i].buf) vmaDestroyBuffer(c->vma, PL.rb[i].buf, PL.rb[i].alloc), PL.rb[i].buf = VK_NULL_HANDLE;
PL.marks = VK_NULL_HANDLE;
}
void vk_exec_pool_marks(VkExec *ex) {
if (!__atomic_load_n(&PL.marks_ready, __ATOMIC_ACQUIRE)) return;
VkCtx *c = vk_ctx();
@@ -3121,7 +3153,7 @@ static int pb_key(const VkExec *ex, const VkPipeBinding *b, uint32_t k, uint64_t
return 1; // slot 1: the program's constants (pb_globals); others: the zero buffer
}
if (b->kind == GFX_RES_SSBO) {
if ((r->flags & 1) || r->ptr || r->cbuf_slot == VK_GPA_CB_SLOT) return 0;
if ((r->flags & 1) || r->ptr || r->cbuf_slot >= VK_GPA_CB_SLOT) return 0; // the table, the mesh fallback's buffers
int ubo, slot = ssbo_slot(r, st, &ubo);
if (slot < 0 || slot >= 16) return 0;
const p_bind_buf *p = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot];
@@ -3860,8 +3892,13 @@ static void prog_lead(const prog_t *p, int mesh) {
}
// mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing
// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted.
// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. 2: a mesh draw of
// the mesh fallback (mfb_draw): the vertex stage is mfb_cur_vs, in an ordinary pipeline without vertex input.
static const GfxShader *mfb_cur_vs;
static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) {
int fb = mesh == 2; // from here on mesh says what the draw is, and !fb that its pipeline is a mesh pipeline
mesh = mesh != 0;
if (ex->render_disabled) return vk_cond_draws_skip += ex->has_render_cond, draw_skipped(ex, mesh, SK_DISABLED);
cb_res(ex);
dummies_init();
@@ -3892,8 +3929,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
}
} else {
SM.valid = 0;
vs = shader_for(ex, vst, &has_vs, &vs_wait, 0);
if (mesh) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0);
if (fb) vs = mfb_cur_vs, has_vs = 1;
else vs = shader_for(ex, vst, &has_vs, &vs_wait, 0);
if (mesh && !fb) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0);
fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, 0);
if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs))
memo_key(ex, gen, &SM.k), SM.vs = vs, SM.fs = fs, SM.has_fs = has_fs, SM.valid = 1;
@@ -3927,6 +3965,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
VkPipeState s = ex->pipe;
s.vs = vs, s.fs = fs, s.ts = ts;
if (mesh) foliage_shadow_cull(ex, &s);
if (fb) memset(s.vattr, 0, sizeof s.vattr), memset(s.vstream, 0, sizeof s.vstream); // whatever the last vertex draw left
s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0));
s.prim = (uint8_t)prim;
s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0);
@@ -3946,7 +3985,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
int fullscreen = 0; // the wait is a full-screen draw's
if (oneshot > 0 && oneshot_budget_on()) {
budget = frame_wait();
wait_ms = oneshot_ms(budget, mesh, &s);
wait_ms = oneshot_ms(budget, mesh && !fb, &s);
if (wait_ms) t0 = kern_now_ns();
} else if (oneshot > 0) {
wait_ms = ONESHOT_WAIT_MS;
@@ -3984,7 +4023,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
vk_exec_ahead(ex);
// A mesh pipeline that is known to be compiling gets no fast link, so the few milliseconds a draw waits
// for one are lost in every frame that meets it until the compile ends, 0.2 to 1.1 s later (MESH_WAIT_MS).
if (wait_ms && linking && mesh && vk_pipeline_building_ms(&s) > MESH_WAIT_MS) wait_ms = 0;
if (wait_ms && linking && mesh && !fb && vk_pipeline_building_ms(&s) > MESH_WAIT_MS) wait_ms = 0;
if (wait_ms) p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
}
if (t0 && budget) {
@@ -3999,7 +4038,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
if (!w0) w0 = kern_now_ns();
if (oneshot_budget_on()) {
uint64_t *pool = frame_wait(), b0 = kern_now_ns();
p = vk_pipeline_graphics_async(&s, &info, &pending, oneshot_ms(pool, mesh, &s));
p = vk_pipeline_graphics_async(&s, &info, &pending, oneshot_ms(pool, mesh && !fb, &s));
pool_charge(pool, b0);
} else {
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
@@ -4045,7 +4084,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
delta = cfg_bool("NX2NAVI_DYN_DELTA");
if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)");
}
if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh,
if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh && !fb,
&vk_exec_frame.dyn_left);
else vk_pipeline_set_dynamic(ex->cb, &s);
ex->recorded++;
@@ -4068,7 +4107,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
if (fbl & ex->fb_dirty) feedback_barrier(ex);
feedback_written(ex, &s, fbl);
if (!mesh) bind_vertex_buffers(ex);
if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh);
if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh && !fb);
if (vk_hdr_track) vk_hdr_draw(ex, info);
vk_exec_oq_draw(ex);
DRS.issued[mesh]++;
@@ -4951,12 +4990,266 @@ static void mesh_shading_rate(VkExec *ex) {
ex->recorded++;
}
// ---- mesh fallback: a mesh draw on a device without VK_EXT_mesh_shader (mesh_fallback, vk_device.c), or on any
// device with mesh_fallback = 1. The program's task and mesh stages run as compute dispatches that write the
// fallback's buffers, and an ordinary draw reads them (mesh_fb.c makes the modules, vk_mesh_fb.c holds the buffers):
// the draw's record its workgroup count, copied on the GPU when the game's own record holds it
// [task dispatch the task module, with the task stage's bindings
// scan lists the mesh workgroups the tasks started and counts them into the record]
// mesh dispatch the mesh module, with the mesh stage's bindings
// draw one instance per mesh workgroup, with the generated vertex stage and the program's
// fragment stage, the instance count from the record
// Every step reads what the one before wrote, so a full barrier stands between them, and the dispatches cannot be
// recorded inside a pass: each such draw ends the open pass, and the next draw begins it again. The buffers are
// reused by every draw (vk_mesh_fb.c), which the barrier before a draw's first dispatch makes safe: the previous
// draw's vertex stage has read them by then.
//
// The modules of a program are made once, on a thread of their own, with their compute pipelines (a mesh program of
// the game is up to 900,000 SPIR-V words, and its pipeline compiles for as long as the mesh pipeline it replaces):
// until they are ready the program's draws are skipped, like draws whose pipeline is being built. Without background
// pipeline builds (the probes) they are made at the draw.
typedef struct MfbCs {
const GfxShader *ms, *ts; // the key: the translated mesh and task shaders (ts NULL: none)
MeshFbInfo mi, ti;
volatile int32_t state; // 0 queued or being made, 1 ready, -1 failed
const GfxShader *cs_mesh, *cs_task; // the compute modules (gfx_shader_own)
struct MfbCs *next; // the worker's queue
} MfbCs;
typedef struct { const GfxShader *ms; uint32_t mask; uint8_t type[32]; const GfxShader *vs; } MfbVs;
#define MFB_TAB 4096
static MfbCs *mfb_cs_tab[MFB_TAB];
static MfbVs mfb_vs_tab[MFB_TAB];
static struct {
PlatMutex lock;
PlatCond wake;
PlatThread *th;
MfbCs *head, *tail;
uint64_t made, failed, pending_skips;
} MFB = {.lock = PLAT_MUTEX_INIT};
static uint32_t mfb_binding_base(const GfxShader *s) { // one past the highest binding the module declares
uint64_t n;
const uint32_t *w = gfx_shader_spirv(s, &n);
uint32_t top = 0;
for (uint64_t i = 5; i < n;) {
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
if (!len) break;
if (op == 71 && len == 4 && w[i + 2] == 33 && w[i + 3] > top) top = w[i + 3]; // OpDecorate Binding
i += len;
}
return top + 1;
}
// The compute module of stage (6 task, 7 mesh) of e, as a shader of the runtime's own with the stage's resources and
// the fallback's buffers.
static const GfxShader *mfb_module(const MfbCs *e, int stage, char *err, size_t cap) {
const GfxShader *src = stage == 6 ? e->ts : e->ms;
uint64_t n, on = 0;
const uint32_t *w = gfx_shader_spirv(src, &n);
uint32_t base = mfb_binding_base(src);
uint32_t *out = stage == 6 ? meshfb_task_cs(w, n, &e->ti, base, &on, err, cap)
: meshfb_mesh_cs(w, n, &e->mi, base, e->ti.payload_bytes / 4, &on, err, cap);
if (!out) return NULL;
static const int task_slots[1] = {MESHFB_CB_TASK}, mesh_slots[3] = {MESHFB_CB_TASK, MESHFB_CB_MESH, MESHFB_CB_LIST};
char *res = stage == 6 ? meshfb_res_json(gfx_shader_res_json(src), base, task_slots, 1)
: e->mi.has_task ? meshfb_res_json(gfx_shader_res_json(src), base, mesh_slots, 3)
: meshfb_res_json(gfx_shader_res_json(src), base + 1, mesh_slots + 1, 1);
const GfxShader *s = res ? gfx_shader_own(out, on, res) : NULL;
if (!s) snprintf(err, cap, "the converted module or its reflection is malformed");
free(out), free(res);
return s;
}
static void mfb_make(MfbCs *e) {
char err[200] = "";
const VkPipeInfo *info;
int ok = (e->cs_mesh = mfb_module(e, 7, err, sizeof err)) != NULL;
if (ok && e->ts) ok = (e->cs_task = mfb_module(e, 6, err, sizeof err)) != NULL;
if (ok && !vk_pipeline_compute(e->cs_mesh, &info)) ok = 0, snprintf(err, sizeof err, "the mesh stage's compute pipeline failed");
if (ok && e->ts && !vk_pipeline_compute(e->cs_task, &info)) ok = 0, snprintf(err, sizeof err, "the task stage's compute pipeline failed");
plat_mutex_lock(&MFB.lock);
uint64_t nth = ok ? ++MFB.made : ++MFB.failed;
plat_mutex_unlock(&MFB.lock);
if (!ok && (nth <= 8 || !(nth & (nth - 1))))
rt_log("vk: mesh fallback: a program's modules could not be made (%s): its draws are skipped (%llu such programs)", err,
(unsigned long long)nth);
__atomic_store_n(&e->state, ok ? 1 : -1, __ATOMIC_RELEASE);
}
static void mfb_worker(void *arg) {
(void)arg;
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); // as the pipeline workers' optimised builds: the game's threads go first
for (;;) {
plat_mutex_lock(&MFB.lock);
while (!MFB.head) plat_cond_wait(&MFB.wake, &MFB.lock);
MfbCs *e = MFB.head;
if (!(MFB.head = e->next)) MFB.tail = NULL;
plat_mutex_unlock(&MFB.lock);
mfb_make(e);
}
}
// The entry of (ms, ts), made and queued at its first draw; NULL (logged) when the program's layout is not carried.
static MfbCs *mfb_cs_get(VkExec *ex, const GfxShader *ms, const GfxShader *ts) {
uint32_t i = (uint32_t)((((uintptr_t)ms >> 4) * 0x9e3779b1u) ^ (((uintptr_t)ts >> 4) * 0x85ebca6bu)) & (MFB_TAB - 1);
for (uint32_t k = 0; k < MFB_TAB; k++, i = (i + 1) & (MFB_TAB - 1)) {
MfbCs *e = mfb_cs_tab[i];
if (e && (e->ms != ms || e->ts != ts)) continue;
if (e) return e;
const uint8_t *ctl, *code;
uint32_t len;
char err[160] = "";
e = calloc(1, sizeof *e);
if (!e) rt_fatal("vk: out of memory");
e->ms = ms, e->ts = ts, e->state = -1;
mfb_cs_tab[i] = e; // a program that is refused is not looked at again
int ok = find_stage_scan(ex->prog, NVN_STAGE_MESH, &ctl, &code, &len) && len >= 0x80 &&
meshfb_info(ctl, code, 7, &e->mi, err, sizeof err);
if (ok && ts) ok = find_stage_scan(ex->prog, NVN_STAGE_TASK, &ctl, &code, &len) && meshfb_info(ctl, code, 6, &e->ti, err, sizeof err);
if (ok && e->mi.has_task != (ts != NULL)) ok = 0, snprintf(err, sizeof err, "the mesh stage %s a task stage and the program has %s",
e->mi.has_task ? "expects" : "does not expect", ts ? "one" : "none");
if (!ok) {
warn_once(W_MESH, "mesh fallback: a program is not carried (%s): its draws are skipped", *err ? err : "no stage data");
return e;
}
e->state = 0;
if (!vk_pipeline_async_on()) {
mfb_make(e);
return e;
}
plat_mutex_lock(&MFB.lock);
if (!MFB.th) plat_cond_init(&MFB.wake), MFB.th = plat_thread_new(mfb_worker, NULL, "mesh fallback");
if (MFB.tail) MFB.tail->next = e;
else MFB.head = e;
MFB.tail = e;
plat_cond_signal(&MFB.wake);
plat_mutex_unlock(&MFB.lock);
return e;
}
rt_fatal("vk: mesh fallback: more than %d task and mesh program pairs", MFB_TAB);
}
// The vertex stage that draws e's mesh buffer slots for fragment shader fs: an output for every input fs reads, in
// its type.
static const GfxShader *mfb_vs_get(const MfbCs *e, const GfxShader *fs) {
GfxShaderIo fin[32];
uint8_t type[32] = {0};
uint32_t mask = fs ? gfx_shader_inputs(fs, fin) : 0;
for (int g = 0; g < 32; g++)
if (mask >> g & 1) type[g] = fin[g].type;
uint32_t i = (uint32_t)((((uintptr_t)e->ms >> 4) * 0x9e3779b1u) ^ (mask * 0x85ebca6bu)) & (MFB_TAB - 1);
for (uint32_t k = 0; k < MFB_TAB; k++, i = (i + 1) & (MFB_TAB - 1)) {
MfbVs *v = &mfb_vs_tab[i];
if (v->ms && (v->ms != e->ms || v->mask != mask || memcmp(v->type, type, 32))) continue;
if (v->ms) return v->vs;
static const char res[] = "{\"key\":\"meshfb\",\"stage\":\"vs\",\"notes\":[],\"cbufs\":[],\"sbufs\":[{\"binding\": 1, "
"\"slot\": -1, \"cb_slot\": 66, \"cb_offset\": 0, \"flags\": 0}],\"textures\":[],\"images\":[]}";
_Static_assert(MESHFB_CB_MESH == 66, "the vertex stage's reflection names the mesh buffer");
uint64_t n = 0;
uint32_t *w = meshfb_vs(&e->mi, mask, type, &n);
v->ms = e->ms, v->mask = mask, memcpy(v->type, type, 32);
v->vs = w ? gfx_shader_own(w, n, res) : NULL;
free(w);
if (!v->vs) warn_once(W_MESH, "mesh fallback: a vertex stage could not be made: its draws are skipped");
return v->vs;
}
rt_fatal("vk: mesh fallback: more than %d vertex stages", MFB_TAB);
}
// One of the draw's compute dispatches: module cs with the bindings of NVN stage `stage`, its group counts read from
// the record at `at`. 0 when its pipeline is not there.
static int mfb_dispatch(VkExec *ex, const GfxShader *cs, int stage, VkBuffer rec, VkDeviceSize at) {
const VkPipeInfo *info = NULL;
VkPipeline p = vk_pipeline_compute(cs, &info);
if (!p || !info || (info->pool && !pool_on())) return 0;
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
mfb_stage = stage;
bind_resources(ex, info);
mfb_stage = 0;
VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0);
vkCmdDispatchIndirect(ex->cb, rec, at);
vk_pass_mark_end(ex->cb, m);
ex->recorded++;
return 1;
}
// A mesh draw of `count` workgroups, or of the count in the game's record at ind (4 bytes, GPU-written).
static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) {
if (ex->render_disabled) {
vk_cond_draws_skip += ex->has_render_cond;
draw_skipped(ex, 1, SK_DISABLED);
return;
}
if (!ex->prog) { draw_skipped(ex, 1, SK_NO_PROGRAM); return; }
cb_res(ex);
dummies_init();
int has_ms, has_ts, has_fs, w_ms = 0, w_ts = 0, w_fs = 0;
const GfxShader *ms = shader_for(ex, NVN_STAGE_MESH, &has_ms, &w_ms, 0);
const GfxShader *ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &w_ts, 0);
const GfxShader *fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &w_fs, 0);
if (!has_ms) { draw_skipped(ex, 1, SK_NO_VERTEX_STAGE); return; }
if (w_ms || w_ts || w_fs) {
vk_exec_ahead(ex);
perf_count(PERF_CNT_SHADER_SKIP);
draw_skipped(ex, 1, SK_SHADER_PENDING);
return;
}
if (!ms || (has_ts && !ts) || (has_fs && !fs)) { draw_skipped(ex, 1, SK_SHADER_FAILED); return; }
MfbCs *c = mfb_cs_get(ex, ms, ts);
int state = __atomic_load_n(&c->state, __ATOMIC_ACQUIRE);
if (!state && !vk_exec_targets_recent(ex, pipe_skip_mode() >= 1)) {
// a target the game does not render every frame: the draw may not come again, so it waits for its modules
// from the frame's wait pool, as such a draw waits for a pipeline (prepare_draw_ex)
uint64_t *pool = frame_wait(), t0 = kern_now_ns();
while (!state && pool_ms(pool, t0)) plat_sleep_ms(1), state = __atomic_load_n(&c->state, __ATOMIC_ACQUIRE);
pool_charge(pool, t0);
}
if (!state) { MFB.pending_skips++; draw_skipped(ex, 1, SK_PIPE_PENDING); return; }
const GfxShader *vs = state > 0 ? mfb_vs_get(c, fs) : NULL;
if (!vs || !vk_meshfb_init()) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; }
uint32_t verts = c->mi.maxp * c->mi.ipp, slot_words = meshfb_slot_words(&c->mi);
uint32_t most = vk_ctx()->props.limits.maxComputeWorkGroupCount[0];
if (count > most) count = most; // a count the GPU wrote is not looked at here: the dispatch reads it as it is
VkBuffer rec;
int cmds;
vk_exec_barrier(ex); // ends the pass; the culling's record and the last draw's reads come before what follows
VkDeviceSize at = vk_meshfb_record(ex->cb, ind ? ind->buffer : VK_NULL_HANDLE, ind ? ind->offset : 0, count, ts != NULL,
verts, slot_words, &rec, &cmds);
ex->recorded += (uint32_t)cmds;
vk_exec_barrier(ex);
if (ts) {
if (!mfb_dispatch(ex, c->cs_task, NVN_STAGE_TASK, rec, at)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; }
vk_exec_barrier(ex);
vk_meshfb_scan(ex->cb, at, c->ti.payload_bytes / 4, c->ti.threads, slot_words, verts);
B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own
ex->recorded += 4;
vk_exec_barrier(ex);
}
if (!mfb_dispatch(ex, c->cs_mesh, NVN_STAGE_MESH, rec, at + 16)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; }
vk_exec_barrier(ex);
vk_meshfb_note(ex->cb, at);
ex->recorded++;
vk_exec_barrier(ex);
c0_binst = 0;
mfb_cur_vs = vs;
int prim = c->mi.ipp == 1 ? NVN_DRAW_PRIMITIVE_POINTS : c->mi.ipp == 2 ? NVN_DRAW_PRIMITIVE_LINES : NVN_DRAW_PRIMITIVE_TRIANGLES;
if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 2)) return;
vk_exec_frame.draws++;
vkCmdDrawIndirect(ex->cb, rec, at + 32, 1, sizeof(VkDrawIndirectCommand));
ex->recorded++;
}
static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) {
PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn();
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
static int on = -1;
if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis)
if (!d->count || !on) return;
if (vk_ctx()->mesh_fallback) {
if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first);
mfb_draw(ex, d->count, NULL);
return;
}
PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn();
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
c0_binst = 0;
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;
if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first);
@@ -4980,13 +5273,21 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) {
// MESH_RING entries outlast the frames in flight: an entry is rewritten only after a barrier behind its last draw.
enum { MESH_RING = 4096 };
static VkBuffer mesh_ring;
static VmaAllocation mesh_ring_alloc;
static uint32_t mesh_ring_next;
// NX2NAVI_GPU_PASSES: the workgroup counts of the indirect mesh draws of the timed frames, copied to host memory.
int vk_gpu_passes_capturing(void);
enum { MESH_COUNTS = 4096 };
static VkBuffer mesh_count_buf;
static VmaAllocation mesh_count_alloc;
static uint32_t *mesh_count_map, mesh_count_n, mesh_count_was;
static void mesh_ring_release(VkCtx *c) { // vk_exec_draw_shutdown: a probe ends with the device idle
if (mesh_ring) vmaDestroyBuffer(c->vma, mesh_ring, mesh_ring_alloc);
if (mesh_count_buf) vmaDestroyBuffer(c->vma, mesh_count_buf, mesh_count_alloc);
mesh_ring = mesh_count_buf = VK_NULL_HANDLE, mesh_count_map = NULL;
}
uint32_t vk_exec_mesh_counts(const uint32_t **out) {
*out = mesh_count_map;
return mesh_count_map ? mesh_count_n : 0;
@@ -5001,8 +5302,7 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) {
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
VmaAllocationInfo info;
VmaAllocation a;
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_count_buf, &a, &info) != VK_SUCCESS) return;
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_count_buf, &mesh_count_alloc, &info) != VK_SUCCESS) return;
mesh_count_map = info.pMappedData;
}
if (mesh_count_n >= MESH_COUNTS) return;
@@ -5053,8 +5353,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING,
.usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
VmaAllocation a;
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &a, NULL) != VK_SUCCESS) {
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &mesh_ring_alloc, NULL) != VK_SUCCESS) {
mesh_ring = VK_NULL_HANDLE;
return 0;
}
@@ -5107,7 +5406,8 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) {
if (vk_ctx()->has_mesh_shader)
fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT");
}
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
int fallback = vk_ctx()->mesh_fallback;
if (!fn && !fallback) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
static int on = -1;
if (on < 0) on = rt_knob("MESH", "1");
if (!on) return;
@@ -5117,6 +5417,12 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) {
(unsigned long long)d->ind);
c0_binst = 0;
VK_MEM_USE(ip);
if (fallback) {
mfb_draw(ex, 0, &ip);
vk_mem_count(ip.cls, VK_MUSE_IND, 4);
VK_MEM_BOUND(ip);
return;
}
VkDeviceSize at;
if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped");
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;

215
runtime/vk_mesh_fb.c Normal file
View File

@@ -0,0 +1,215 @@
// The mesh fallback's buffers and records. A task or mesh program runs as a compute dispatch that writes a storage
// buffer (mesh_fb.c), and the draw reads it:
// task buffer a task workgroup's payload at its index, payload-size apart
// list per mesh workgroup after a task stage: its task and its index among that task's (meshfb_scan.comp)
// mesh buffer a mesh workgroup's vertex lines and primitive area at its slot
// records per draw 64 bytes: the indirect arguments of its task dispatch, its mesh dispatch and its draw
// The three buffers are reused by every draw: a draw's dispatches write them and its vertex stage has read them
// before the next draw's dispatches run, with a barrier in between (vk_exec_draw.c mfb_draw), so their size is what
// the largest draw needs, not what a frame's draws need together. mesh_fallback_mb sets it: an eighth for the task
// buffer, the rest for the mesh buffer. Workgroups past either buffer's end write nothing and draw nothing (the
// modules check the buffer's length); the draws that happened to are counted and the log names the size they needed.
// A record is used by one draw and written again only after MESHFB_RECS further draws.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cfg.h"
#include "host.h"
#include "mesh_fb.h"
#include "vk.h"
#include "vk_mesh_fb.h"
#include "meshfb_scan_spv.h"
enum { MESHFB_RECS = 4096, REC_BYTES = 64 };
static struct {
int tried, ok;
VkBuffer task, mesh, list, rec, stat;
VmaAllocation task_a, mesh_a, list_a, rec_a, stat_a;
VkDeviceSize task_bytes, mesh_bytes, list_bytes; // as one storage binding spans them
const uint32_t *stat_map; // per record: mesh workgroups and task workgroups it asked for
uint32_t cap_slots[MESHFB_RECS], slot_words[MESHFB_RECS], cap_tasks[MESHFB_RECS]; // what the record's draw had room for
uint8_t noted[MESHFB_RECS];
uint32_t next;
uint64_t draws, short_draws, worst_bytes;
PFN_vkCmdPushDescriptorSetKHR push;
VkDescriptorSetLayout dsl;
VkPipelineLayout layout;
VkPipeline scan;
} M;
static int make_buffer(VkCtx *c, VkDeviceSize size, VkBufferUsageFlags usage, int host, VkBuffer *b, VmaAllocation *a, void **map) {
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size, .usage = usage};
VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.flags = host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT : 0};
VmaAllocationInfo info;
if (vmaCreateBuffer(c->vma, &bi, &ai, b, a, &info) != VK_SUCCESS) return *b = VK_NULL_HANDLE, 0;
if (map) *map = info.pMappedData;
return !host || info.pMappedData;
}
int vk_meshfb_init(void) {
if (M.tried) return M.ok;
M.tried = 1;
VkCtx *c = vk_ctx();
if (!c) return 0;
uint64_t mb = (uint64_t)cfg_int("NX2NAVI_MESH_FALLBACK_MB");
if (mb < 16) mb = 16;
if (mb > 4096) mb = 4096;
VkDeviceSize range = c->props.limits.maxStorageBufferRange & ~(VkDeviceSize)3, task = (mb << 20) / 8, mesh = (mb << 20) - task;
VkDeviceSize list = mesh / 128; // an entry per slot of 512 bytes, and no mesh program's slot is smaller
void *map = NULL;
const VkBufferUsageFlags st = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
if (!make_buffer(c, task, st, 0, &M.task, &M.task_a, NULL) || !make_buffer(c, mesh, st, 0, &M.mesh, &M.mesh_a, NULL) ||
!make_buffer(c, list, st, 0, &M.list, &M.list_a, NULL) ||
!make_buffer(c, (VkDeviceSize)MESHFB_RECS * REC_BYTES,
st | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT, 0,
&M.rec, &M.rec_a, NULL) ||
!make_buffer(c, (VkDeviceSize)MESHFB_RECS * 8, VK_BUFFER_USAGE_TRANSFER_DST_BIT, 1, &M.stat, &M.stat_a, &map)) {
rt_log("vk: mesh fallback: its buffers (%llu MiB, mesh_fallback_mb) could not be allocated: task and mesh draws are skipped",
(unsigned long long)mb);
return 0;
}
M.stat_map = map;
memset(map, 0, (size_t)MESHFB_RECS * 8);
M.task_bytes = task < range ? task : range, M.mesh_bytes = mesh < range ? mesh : range, M.list_bytes = list < range ? list : range;
M.push = (PFN_vkCmdPushDescriptorSetKHR)vk_dpa(c->device, "vkCmdPushDescriptorSetKHR");
VkDescriptorSetLayoutBinding b[4];
for (int i = 0; i < 4; i++)
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
.bindingCount = 4, .pBindings = b};
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 24};
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof meshfb_scan_spv,
.pCode = meshfb_scan_spv};
VkShaderModule m = VK_NULL_HANDLE;
if (!M.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &M.dsl) != VK_SUCCESS) goto fail;
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
.pSetLayouts = &M.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
if (vkCreatePipelineLayout(c->device, &li, NULL, &M.layout) != VK_SUCCESS ||
vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS)
goto fail;
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
.layout = M.layout};
VkResult res = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &M.scan);
vkDestroyShaderModule(c->device, m, NULL);
if (res != VK_SUCCESS) goto fail;
rt_log("vk: mesh fallback: %llu MiB for task payloads, %llu MiB for mesh output (mesh_fallback_mb = %llu)",
(unsigned long long)(M.task_bytes >> 20), (unsigned long long)(M.mesh_bytes >> 20), (unsigned long long)mb);
return M.ok = 1;
fail:
rt_log("vk: mesh fallback: the scan pipeline could not be created: task and mesh draws are skipped");
return 0;
}
VkDescriptorBufferInfo vk_meshfb_buffer(int cb_slot) {
if (cb_slot == MESHFB_CB_TASK) return (VkDescriptorBufferInfo){M.task, 0, M.task_bytes};
if (cb_slot == MESHFB_CB_LIST) return (VkDescriptorBufferInfo){M.list, 0, M.list_bytes};
return (VkDescriptorBufferInfo){M.mesh, 0, M.mesh_bytes};
}
uint64_t vk_meshfb_mesh_words(void) { return M.mesh_bytes / 4; }
// The record about to be written again: what its last draw asked for against what it had room for.
static void record_check(uint32_t i) {
if (!M.noted[i]) return;
M.noted[i] = 0;
uint32_t want = M.stat_map[2 * i], tasks = M.stat_map[2 * i + 1];
if (want <= M.cap_slots[i] && tasks <= M.cap_tasks[i]) return;
uint64_t need = (uint64_t)want * M.slot_words[i] * 4;
if (need > M.worst_bytes) M.worst_bytes = need;
uint64_t n = ++M.short_draws;
if (n <= 4 || !(n & (n - 1))) {
char from[96] = "";
if (tasks) snprintf(from, sizeof from, " from %u task workgroups (room for %u)", tasks, M.cap_tasks[i]);
rt_log("vk: mesh fallback: a draw's programs emitted %u mesh workgroups of %u bytes%s, and the mesh buffer holds %u: "
"what did not fit was not drawn (%llu such draws of %llu; mesh_fallback_mb of %llu or more would hold the "
"largest so far)", want, M.slot_words[i] * 4, from, M.cap_slots[i], (unsigned long long)n,
(unsigned long long)M.draws, (unsigned long long)((M.worst_bytes * 8 / 7 >> 20) + 1));
}
}
VkDeviceSize vk_meshfb_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, uint32_t count, int has_task,
uint32_t verts, uint32_t slot_words, VkBuffer *rec, int *cmds) {
uint32_t i = M.next++ % MESHFB_RECS;
VkDeviceSize at = (VkDeviceSize)i * REC_BYTES;
record_check(i);
M.draws++;
M.slot_words[i] = slot_words, M.cap_slots[i] = (uint32_t)(M.mesh_bytes / 4 / (slot_words ? slot_words : 1));
M.cap_tasks[i] = 0xffffffffu; // vk_meshfb_scan knows the payload size
if (has_task && M.cap_slots[i] > M.list_bytes / 4) M.cap_slots[i] = (uint32_t)(M.list_bytes / 4);
*rec = M.rec;
int n = 0;
vkCmdFillBuffer(cb, M.rec, at, REC_BYTES, 1), n++;
// where the workgroup count goes: the task dispatch, or the mesh dispatch, the draw's instances and the report
VkDeviceSize to[3] = {has_task ? at : at + 16, at + 36, at + 48};
int nto = has_task ? 1 : 3;
if (!has_task) {
vkCmdFillBuffer(cb, M.rec, at + 32, 4, verts), n++;
vkCmdFillBuffer(cb, M.rec, at + 40, 8, 0), n++;
vkCmdFillBuffer(cb, M.rec, at + 52, 4, 0), n++;
}
if (src) {
VkBufferCopy cp[3];
for (int k = 0; k < nto; k++) cp[k] = (VkBufferCopy){src_off, to[k], 4};
vkCmdCopyBuffer(cb, src, M.rec, (uint32_t)nto, cp), n++;
} else {
for (int k = 0; k < nto; k++) vkCmdFillBuffer(cb, M.rec, to[k], 4, count), n++;
}
*cmds = n;
return at;
}
void vk_meshfb_scan(VkCommandBuffer cb, VkDeviceSize at, uint32_t task_words, uint32_t fan, uint32_t slot_words, uint32_t verts) {
uint32_t i = (uint32_t)(at / REC_BYTES);
M.cap_tasks[i] = (uint32_t)(M.task_bytes / 4 / (task_words ? task_words : 1));
VkDescriptorBufferInfo bi[4] = {{M.rec, 0, VK_WHOLE_SIZE}, {M.task, 0, M.task_bytes}, {M.list, 0, M.list_bytes},
{M.mesh, 0, M.mesh_bytes}};
VkWriteDescriptorSet w[4];
for (int k = 0; k < 4; k++)
w[k] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)k,
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &bi[k]};
uint32_t pc[6] = {(uint32_t)(at / 4), task_words, fan, slot_words, verts, vk_ctx()->props.limits.maxComputeWorkGroupCount[0]};
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, M.scan);
rec_push(M.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, M.layout, 0, 4, w);
vkCmdPushConstants(cb, M.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc);
vkCmdDispatch(cb, 1, 1, 1);
}
void vk_meshfb_note(VkCommandBuffer cb, VkDeviceSize at) {
uint32_t i = (uint32_t)(at / REC_BYTES);
VkBufferCopy cp = {at + 48, (VkDeviceSize)i * 8, 8};
vkCmdCopyBuffer(cb, M.rec, M.stat, 1, &cp);
M.noted[i] = 1;
}
uint64_t vk_meshfb_short(void) {
if (!M.ok) return 0;
for (uint32_t i = 0; i < MESHFB_RECS; i++) record_check(i);
return M.short_draws;
}
void vk_meshfb_shutdown(void) {
VkCtx *c = vk_ctx();
if (!c || !M.tried) return;
if (vk_meshfb_short())
rt_log("vk: mesh fallback: %llu of %llu draws emitted more than the buffers hold (mesh_fallback_mb of %llu or more would "
"hold the largest)", (unsigned long long)M.short_draws, (unsigned long long)M.draws,
(unsigned long long)((M.worst_bytes * 8 / 7 >> 20) + 1));
if (M.scan) vkDestroyPipeline(c->device, M.scan, NULL);
if (M.layout) vkDestroyPipelineLayout(c->device, M.layout, NULL);
if (M.dsl) vkDestroyDescriptorSetLayout(c->device, M.dsl, NULL);
if (M.task) vmaDestroyBuffer(c->vma, M.task, M.task_a);
if (M.mesh) vmaDestroyBuffer(c->vma, M.mesh, M.mesh_a);
if (M.list) vmaDestroyBuffer(c->vma, M.list, M.list_a);
if (M.rec) vmaDestroyBuffer(c->vma, M.rec, M.rec_a);
if (M.stat) vmaDestroyBuffer(c->vma, M.stat, M.stat_a);
memset(&M, 0, sizeof M);
}

29
runtime/vk_mesh_fb.h Normal file
View File

@@ -0,0 +1,29 @@
// The mesh fallback's buffers and its records (vk_mesh_fb.c); the modules are mesh_fb.c's, the draws
// vk_exec_draw.c's ("mesh fallback").
#pragma once
#include "vk.h"
// Creates the buffers and the scan pipeline on first use. 0 when that failed (logged once): the draws are skipped.
int vk_meshfb_init(void);
// The buffer a module's extra storage binding reads or writes (mesh_fb.h MESHFB_CB_*), whole or as much of it as one
// storage binding may span.
VkDescriptorBufferInfo vk_meshfb_buffer(int cb_slot);
uint64_t vk_meshfb_mesh_words(void); // the mesh buffer's size as a binding sees it, in words
// A draw's record, 64 bytes of the record buffer (*rec) at the offset returned: the task dispatch at +0, the mesh
// dispatch at +16, the draw (VkDrawIndirectCommand) at +32. Records the fills and copies that start it, outside a
// pass: the workgroup count from 4 bytes at src_off of src, or `count` when src is VK_NULL_HANDLE. Without a task
// stage that count is the mesh dispatch and the draw's instance count; with one it is the task dispatch, and
// vk_meshfb_scan writes the rest. slot_words: of the draw's mesh program, for the report of a draw the mesh buffer
// was too small for. Returns the number of commands recorded in *cmds.
VkDeviceSize vk_meshfb_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, uint32_t count, int has_task,
uint32_t verts, uint32_t slot_words, VkBuffer *rec, int *cmds);
// The pass between the task and the mesh dispatch (shaders/meshfb_scan.comp): binds its own compute pipeline and set.
void vk_meshfb_scan(VkCommandBuffer cb, VkDeviceSize at, uint32_t task_words, uint32_t fan, uint32_t slot_words, uint32_t verts);
// After the record is complete (the copies or the scan, and a barrier): keeps what the draw asked for where the host
// reads it later. One copy command.
void vk_meshfb_note(VkCommandBuffer cb, VkDeviceSize at);
// Draws so far whose programs emitted more than the buffers hold, the records not looked at yet included: call it
// with the device idle (a probe, the shutdown).
uint64_t vk_meshfb_short(void);
void vk_meshfb_shutdown(void); // the device must be idle

View File

@@ -3427,6 +3427,7 @@ static void keys_note(const VkPipeState *s) {
static const uint8_t none[32];
if (s->fs && !memcmp(gfx_shader_input_id(s->fs), none, 32)) return; // the runtime's own shader (gfx_shader_own)
if (s->ts && !memcmp(gfx_shader_input_id(s->ts), none, 32)) return;
if (!memcmp(gfx_shader_input_id(s->vs), none, 32)) return; // the mesh fallback's vertex stage (vk_exec_draw.c mfb_vs_get)
KeyRec r;
rec_fill(&r, s);
if (!seen_add(rec_hash(&r))) return;

View File

@@ -49,7 +49,7 @@ enum {
R_SetRasterizerDiscardEnable, R_SetScissorWithCount, R_SetStencilCompareMask, R_SetStencilOp, R_SetStencilReference,
R_SetStencilTestEnable, R_SetStencilWriteMask, R_SetViewportWithCount, R_WriteTimestamp, R_WriteTimestamp2,
R_PushTpl, R_Push, R_VertexInput, R_Bool, R_PolygonMode, R_BlendEnable, R_BlendEquation, R_WriteMask, R_EndCb,
R_PassMark, R_DrawMesh, R_DrawMeshInd, R_ShadingRate,
R_PassMark, R_DrawMesh, R_DrawMeshInd, R_ShadingRate, R_DrawIndirect,
};
static uint64_t now_ns(void) { return plat_now_ns(); }
@@ -347,6 +347,9 @@ FIXED(DrawIndexed, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, int3
#define PACK_DrawIndexedIndirect *a = (A_DrawIndexedIndirect){b, o, n, s}
FIXED(DrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s),
VkBuffer b; VkDeviceSize o; uint32_t n, s)
#define PACK_DrawIndirect *a = (A_DrawIndirect){b, o, n, s}
FIXED(DrawIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s),
VkBuffer b; VkDeviceSize o; uint32_t n, s)
#define PACK_EndQuery *a = (A_EndQuery){p, q}
FIXED(EndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), (cb, p, q), VkQueryPool p; uint32_t q)
#define PACK_EndRendering (void)a
@@ -705,6 +708,7 @@ static void exec_packet(uint32_t op, void *p) {
F(Draw, a->n, a->i, a->f, a->fi)
F(DrawIndexed, a->n, a->i, a->f, a->v, a->fi)
F(DrawIndexedIndirect, a->b, a->o, a->n, a->s)
F(DrawIndirect, a->b, a->o, a->n, a->s)
F(EndQuery, a->p, a->q)
F(FillBuffer, a->b, a->o, a->n, a->v)
F(ResetQueryPool, a->p, a->f, a->n)

View File

@@ -80,6 +80,7 @@ void rec_pass_mark(VkCommandBuffer cb, uint32_t id, int end);
X(Draw, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, uint32_t fi), (cb, n, i, f, fi)) \
X(DrawIndexed, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, int32_t v, uint32_t fi), (cb, n, i, f, v, fi)) \
X(DrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s)) \
X(DrawIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s)) \
X(EndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), (cb, p, q)) \
X(EndRendering, (VkCommandBuffer cb), (cb)) \
X(FillBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkDeviceSize n, uint32_t v), (cb, b, o, n, v)) \
@@ -137,6 +138,7 @@ REC_CMDS(REC_DECL)
#define vkCmdDraw rc_vkCmdDraw
#define vkCmdDrawIndexed rc_vkCmdDrawIndexed
#define vkCmdDrawIndexedIndirect rc_vkCmdDrawIndexedIndirect
#define vkCmdDrawIndirect rc_vkCmdDrawIndirect
#define vkCmdEndQuery rc_vkCmdEndQuery
#define vkCmdEndRendering rc_vkCmdEndRendering
#define vkCmdFillBuffer rc_vkCmdFillBuffer