mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 09:19:58 -04:00
Draw task and mesh programs through compute passes when the mesh fallback is on.
A mesh draw becomes its record (the workgroup count, copied on the GPU for an indirect draw), the task dispatch, a scan that lists the mesh workgroups the tasks started, the mesh dispatch, and an instanced indirect draw with a generated vertex stage and the program's fragment stage. The compute modules bind the resources of the task and mesh stages, and their pipelines are made on a thread of their own while the program's draws are skipped. The indirect mesh ring and the texture pool marks are released at a probe's shutdown. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -100,6 +100,7 @@ size_t gfx_shader_warm_list(GfxWarmEnt **out); // malloc'd; recorde
|
||||
const GfxShader *gfx_shader_load_private(const uint8_t iid[32], int *owned);
|
||||
void gfx_shader_release_private(const GfxShader *s);
|
||||
const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words);
|
||||
const char *gfx_shader_res_json(const GfxShader *s); // the translator's .res.json text
|
||||
int gfx_shader_write_code(const GfxShader *s, const char *path); // the guest program, from the stored inputs
|
||||
// A shader of the runtime's own (SPIR-V and a .res.json text as the translator writes them), outside the cache and
|
||||
// never freed; its key is the sha256 of both. NULL when the SPIR-V is malformed.
|
||||
|
||||
@@ -2424,6 +2424,8 @@ const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words) {
|
||||
return s->spirv;
|
||||
}
|
||||
|
||||
const char *gfx_shader_res_json(const GfxShader *s) { return s->reflection; }
|
||||
|
||||
// The guest program a shader was translated from, as its stored inputs hold it (the whole program: NVN2 code from its
|
||||
// first byte), written to path for a disassembler (NX2NAVI_DUMP's sh_<hash>.code), and next to it <path>.job, the
|
||||
// translator request that translates it (one line for nx2navi_xlate --serve; its "out" names the shader cache, and
|
||||
|
||||
47
runtime/shaders/meshfb_scan.comp
Normal file
47
runtime/shaders/meshfb_scan.comp
Normal file
@@ -0,0 +1,47 @@
|
||||
#version 450
|
||||
// The mesh fallback's pass between a draw's task dispatch and its mesh dispatch (vk_mesh_fb.c vk_meshfb_scan,
|
||||
// mesh_fb.c): every task workgroup wrote how many mesh workgroups it starts into its payload (word 1). This lists them
|
||||
// in task order, so that mesh workgroup g finds its task and its index among that task's in list entry g, and writes
|
||||
// the draw's record: how many mesh workgroups to dispatch and how many instances to draw.
|
||||
//
|
||||
// One thread walks the tasks: the order of the list is the order the primitives are drawn in, which has to be the
|
||||
// same in every frame and in the depth and colour passes of one frame.
|
||||
layout(local_size_x = 1) in;
|
||||
|
||||
layout(push_constant) uniform Params {
|
||||
uint rec; // the record's first word in the record buffer
|
||||
uint task_words; // words of a task workgroup's payload in the task buffer
|
||||
uint fan; // mesh workgroups a task workgroup starts at most
|
||||
uint slot_words; // words of a mesh workgroup's output in the mesh buffer
|
||||
uint verts; // vertices drawn per mesh workgroup
|
||||
uint max_groups; // workgroups one dispatch may have (maxComputeWorkGroupCount)
|
||||
} pc;
|
||||
|
||||
// The record: words 0-2 the task dispatch (x written before this pass), 4-6 the mesh dispatch, 8-11 the draw (vertices,
|
||||
// instances, first vertex, first instance), 12 the mesh workgroups the tasks asked for, 13 the task workgroups.
|
||||
layout(set = 0, binding = 0) buffer Rec { uint w[]; } rec;
|
||||
layout(set = 0, binding = 1) readonly buffer Task { uint w[]; } task;
|
||||
layout(set = 0, binding = 2) writeonly buffer List { uint e[]; } list;
|
||||
layout(set = 0, binding = 3) readonly buffer Mesh { uint w[]; } mesh; // read for its size alone
|
||||
|
||||
void main() {
|
||||
uint asked = rec.w[pc.rec];
|
||||
uint tasks = min(min(asked, uint(task.w.length()) / pc.task_words), 1u << 24);
|
||||
uint room = min(min(uint(mesh.w.length()) / pc.slot_words, uint(list.e.length())), pc.max_groups);
|
||||
uint fan = min(pc.fan, 255u);
|
||||
uint m = 0u, want = 0u;
|
||||
for (uint t = 0u; t < tasks; t++) {
|
||||
uint c = min(task.w[t * pc.task_words + 1u], fan);
|
||||
want += c;
|
||||
for (uint j = 0u; j < c && m < room; j++) list.e[m++] = t | j << 24;
|
||||
}
|
||||
rec.w[pc.rec + 4u] = m;
|
||||
rec.w[pc.rec + 5u] = 1u;
|
||||
rec.w[pc.rec + 6u] = 1u;
|
||||
rec.w[pc.rec + 8u] = pc.verts;
|
||||
rec.w[pc.rec + 9u] = m;
|
||||
rec.w[pc.rec + 10u] = 0u;
|
||||
rec.w[pc.rec + 11u] = 0u;
|
||||
rec.w[pc.rec + 12u] = want;
|
||||
rec.w[pc.rec + 13u] = asked;
|
||||
}
|
||||
@@ -29,6 +29,8 @@
|
||||
#endif
|
||||
#include "vk_exec.h"
|
||||
#include "vk_gpa.h"
|
||||
#include "mesh_fb.h"
|
||||
#include "vk_mesh_fb.h"
|
||||
#include "platform/platform.h"
|
||||
#include "vk_dlaa.h"
|
||||
|
||||
@@ -446,7 +448,12 @@ static void twins_release(VkCtx *c);
|
||||
|
||||
// Releases the stand-ins and the depth twins. The device must be idle; the game path leaves them to process exit like
|
||||
// the pipelines.
|
||||
static void mesh_ring_release(VkCtx *c);
|
||||
static void pool_marks_release(VkCtx *c);
|
||||
|
||||
void vk_exec_draw_shutdown(void) {
|
||||
vk_meshfb_shutdown();
|
||||
if (vk_ctx()) mesh_ring_release(vk_ctx()), pool_marks_release(vk_ctx()); // made without the stand-ins below too
|
||||
if (!Z.ready) return;
|
||||
VkCtx *c = vk_ctx();
|
||||
twins_release(c);
|
||||
@@ -948,6 +955,13 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G
|
||||
// and *pending = 1 says the wait ended first.
|
||||
static int sf_mvprev; // shader_for asks for the vertex shader's last-frame variant (per-object motion)
|
||||
|
||||
// Probes only (mesh_fb_probe.c): the shader of the program stage whose control section is ctl, in place of a
|
||||
// translation. The probes build their programs from modules made here, which no translator has an input for.
|
||||
static struct { const uint8_t *ctl; const GfxShader *s; } probe_sh[16];
|
||||
static int n_probe_sh;
|
||||
void vk_exec_probe_shader(const uint8_t *ctl, const GfxShader *s) {
|
||||
if (n_probe_sh < 16) probe_sh[n_probe_sh].ctl = ctl, probe_sh[n_probe_sh].s = s, n_probe_sh++;
|
||||
}
|
||||
static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pending, uint32_t await_ms) {
|
||||
const uint8_t *ctl, *code;
|
||||
uint32_t len;
|
||||
@@ -955,6 +969,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
if (pending) *pending = 0;
|
||||
if (!ex->prog || !find_stage(ex->prog, st, &ctl, &code, &len)) return NULL;
|
||||
*present = 1;
|
||||
for (int i = 0; i < n_probe_sh; i++)
|
||||
if (probe_sh[i].ctl == ctl) return probe_sh[i].s;
|
||||
GfxShaderAnswers a;
|
||||
uint64_t serial = ex->prog->serial;
|
||||
answers_for(ex, st, ctl, slot_mask_of(serial, st), &a);
|
||||
@@ -994,7 +1010,12 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
|
||||
// ---- descriptors
|
||||
|
||||
// The mesh fallback (below, "mesh fallback") runs a program's task and mesh stages as compute modules: while it binds
|
||||
// one's resources, a compute module's bindings are those of this NVN stage (6 task, 7 mesh), not of the compute stage.
|
||||
static int mfb_stage;
|
||||
|
||||
static int nvn_stage_of(int gfx_stage) {
|
||||
if (gfx_stage == GFX_STAGE_CS && mfb_stage) return mfb_stage;
|
||||
return gfx_stage == GFX_STAGE_VS ? NVN_STAGE_VS : gfx_stage == GFX_STAGE_FS ? NVN_STAGE_FS
|
||||
: gfx_stage == GFX_STAGE_TS ? NVN_STAGE_TASK : gfx_stage == GFX_STAGE_MS ? NVN_STAGE_MESH : NVN_STAGE_CS;
|
||||
}
|
||||
@@ -1373,6 +1394,8 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *
|
||||
VkDescriptorBufferInfo t = vk_gpa_info();
|
||||
return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE};
|
||||
}
|
||||
// The mesh fallback's own buffers (mesh_fb.c): no guest memory behind them
|
||||
if (r->cbuf_slot >= MESHFB_CB_TASK && r->cbuf_slot <= MESHFB_CB_LIST) return vk_meshfb_buffer(r->cbuf_slot);
|
||||
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
|
||||
uint64_t max = LIM.sbo_max, align = LIM.sbo_align;
|
||||
VkDescriptorBufferInfo bi;
|
||||
@@ -1908,6 +1931,15 @@ static void pool_marks_make(void) {
|
||||
__atomic_store_n(&PL.marks_ready, 1, __ATOMIC_RELEASE); // the recording thread reads the rest after this
|
||||
}
|
||||
|
||||
static void pool_marks_release(VkCtx *c) { // vk_exec_draw_shutdown: a probe ends with the device idle
|
||||
if (!PL.marks) return;
|
||||
__atomic_store_n(&PL.marks_ready, 0, __ATOMIC_RELEASE);
|
||||
vmaDestroyBuffer(c->vma, PL.marks, PL.marks_alloc);
|
||||
for (int i = 0; i < MARKS_RB; i++)
|
||||
if (PL.rb[i].buf) vmaDestroyBuffer(c->vma, PL.rb[i].buf, PL.rb[i].alloc), PL.rb[i].buf = VK_NULL_HANDLE;
|
||||
PL.marks = VK_NULL_HANDLE;
|
||||
}
|
||||
|
||||
void vk_exec_pool_marks(VkExec *ex) {
|
||||
if (!__atomic_load_n(&PL.marks_ready, __ATOMIC_ACQUIRE)) return;
|
||||
VkCtx *c = vk_ctx();
|
||||
@@ -3121,7 +3153,7 @@ static int pb_key(const VkExec *ex, const VkPipeBinding *b, uint32_t k, uint64_t
|
||||
return 1; // slot 1: the program's constants (pb_globals); others: the zero buffer
|
||||
}
|
||||
if (b->kind == GFX_RES_SSBO) {
|
||||
if ((r->flags & 1) || r->ptr || r->cbuf_slot == VK_GPA_CB_SLOT) return 0;
|
||||
if ((r->flags & 1) || r->ptr || r->cbuf_slot >= VK_GPA_CB_SLOT) return 0; // the table, the mesh fallback's buffers
|
||||
int ubo, slot = ssbo_slot(r, st, &ubo);
|
||||
if (slot < 0 || slot >= 16) return 0;
|
||||
const p_bind_buf *p = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot];
|
||||
@@ -3860,8 +3892,13 @@ static void prog_lead(const prog_t *p, int mesh) {
|
||||
}
|
||||
|
||||
// mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing
|
||||
// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted.
|
||||
// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. 2: a mesh draw of
|
||||
// the mesh fallback (mfb_draw): the vertex stage is mfb_cur_vs, in an ordinary pipeline without vertex input.
|
||||
static const GfxShader *mfb_cur_vs;
|
||||
|
||||
static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) {
|
||||
int fb = mesh == 2; // from here on mesh says what the draw is, and !fb that its pipeline is a mesh pipeline
|
||||
mesh = mesh != 0;
|
||||
if (ex->render_disabled) return vk_cond_draws_skip += ex->has_render_cond, draw_skipped(ex, mesh, SK_DISABLED);
|
||||
cb_res(ex);
|
||||
dummies_init();
|
||||
@@ -3892,8 +3929,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
|
||||
}
|
||||
} else {
|
||||
SM.valid = 0;
|
||||
vs = shader_for(ex, vst, &has_vs, &vs_wait, 0);
|
||||
if (mesh) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0);
|
||||
if (fb) vs = mfb_cur_vs, has_vs = 1;
|
||||
else vs = shader_for(ex, vst, &has_vs, &vs_wait, 0);
|
||||
if (mesh && !fb) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0);
|
||||
fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, 0);
|
||||
if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs))
|
||||
memo_key(ex, gen, &SM.k), SM.vs = vs, SM.fs = fs, SM.has_fs = has_fs, SM.valid = 1;
|
||||
@@ -3927,6 +3965,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
|
||||
VkPipeState s = ex->pipe;
|
||||
s.vs = vs, s.fs = fs, s.ts = ts;
|
||||
if (mesh) foliage_shadow_cull(ex, &s);
|
||||
if (fb) memset(s.vattr, 0, sizeof s.vattr), memset(s.vstream, 0, sizeof s.vstream); // whatever the last vertex draw left
|
||||
s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0));
|
||||
s.prim = (uint8_t)prim;
|
||||
s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0);
|
||||
@@ -3946,7 +3985,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
|
||||
int fullscreen = 0; // the wait is a full-screen draw's
|
||||
if (oneshot > 0 && oneshot_budget_on()) {
|
||||
budget = frame_wait();
|
||||
wait_ms = oneshot_ms(budget, mesh, &s);
|
||||
wait_ms = oneshot_ms(budget, mesh && !fb, &s);
|
||||
if (wait_ms) t0 = kern_now_ns();
|
||||
} else if (oneshot > 0) {
|
||||
wait_ms = ONESHOT_WAIT_MS;
|
||||
@@ -3984,7 +4023,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
|
||||
vk_exec_ahead(ex);
|
||||
// A mesh pipeline that is known to be compiling gets no fast link, so the few milliseconds a draw waits
|
||||
// for one are lost in every frame that meets it until the compile ends, 0.2 to 1.1 s later (MESH_WAIT_MS).
|
||||
if (wait_ms && linking && mesh && vk_pipeline_building_ms(&s) > MESH_WAIT_MS) wait_ms = 0;
|
||||
if (wait_ms && linking && mesh && !fb && vk_pipeline_building_ms(&s) > MESH_WAIT_MS) wait_ms = 0;
|
||||
if (wait_ms) p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
|
||||
}
|
||||
if (t0 && budget) {
|
||||
@@ -3999,7 +4038,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
|
||||
if (!w0) w0 = kern_now_ns();
|
||||
if (oneshot_budget_on()) {
|
||||
uint64_t *pool = frame_wait(), b0 = kern_now_ns();
|
||||
p = vk_pipeline_graphics_async(&s, &info, &pending, oneshot_ms(pool, mesh, &s));
|
||||
p = vk_pipeline_graphics_async(&s, &info, &pending, oneshot_ms(pool, mesh && !fb, &s));
|
||||
pool_charge(pool, b0);
|
||||
} else {
|
||||
p = vk_pipeline_graphics_async(&s, &info, &pending, ONESHOT_WAIT_MS);
|
||||
@@ -4045,7 +4084,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
|
||||
delta = cfg_bool("NX2NAVI_DYN_DELTA");
|
||||
if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)");
|
||||
}
|
||||
if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh,
|
||||
if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh && !fb,
|
||||
&vk_exec_frame.dyn_left);
|
||||
else vk_pipeline_set_dynamic(ex->cb, &s);
|
||||
ex->recorded++;
|
||||
@@ -4068,7 +4107,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
|
||||
if (fbl & ex->fb_dirty) feedback_barrier(ex);
|
||||
feedback_written(ex, &s, fbl);
|
||||
if (!mesh) bind_vertex_buffers(ex);
|
||||
if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh);
|
||||
if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh && !fb);
|
||||
if (vk_hdr_track) vk_hdr_draw(ex, info);
|
||||
vk_exec_oq_draw(ex);
|
||||
DRS.issued[mesh]++;
|
||||
@@ -4951,12 +4990,266 @@ static void mesh_shading_rate(VkExec *ex) {
|
||||
ex->recorded++;
|
||||
}
|
||||
|
||||
// ---- mesh fallback: a mesh draw on a device without VK_EXT_mesh_shader (mesh_fallback, vk_device.c), or on any
|
||||
// device with mesh_fallback = 1. The program's task and mesh stages run as compute dispatches that write the
|
||||
// fallback's buffers, and an ordinary draw reads them (mesh_fb.c makes the modules, vk_mesh_fb.c holds the buffers):
|
||||
// the draw's record its workgroup count, copied on the GPU when the game's own record holds it
|
||||
// [task dispatch the task module, with the task stage's bindings
|
||||
// scan lists the mesh workgroups the tasks started and counts them into the record]
|
||||
// mesh dispatch the mesh module, with the mesh stage's bindings
|
||||
// draw one instance per mesh workgroup, with the generated vertex stage and the program's
|
||||
// fragment stage, the instance count from the record
|
||||
// Every step reads what the one before wrote, so a full barrier stands between them, and the dispatches cannot be
|
||||
// recorded inside a pass: each such draw ends the open pass, and the next draw begins it again. The buffers are
|
||||
// reused by every draw (vk_mesh_fb.c), which the barrier before a draw's first dispatch makes safe: the previous
|
||||
// draw's vertex stage has read them by then.
|
||||
//
|
||||
// The modules of a program are made once, on a thread of their own, with their compute pipelines (a mesh program of
|
||||
// the game is up to 900,000 SPIR-V words, and its pipeline compiles for as long as the mesh pipeline it replaces):
|
||||
// until they are ready the program's draws are skipped, like draws whose pipeline is being built. Without background
|
||||
// pipeline builds (the probes) they are made at the draw.
|
||||
typedef struct MfbCs {
|
||||
const GfxShader *ms, *ts; // the key: the translated mesh and task shaders (ts NULL: none)
|
||||
MeshFbInfo mi, ti;
|
||||
volatile int32_t state; // 0 queued or being made, 1 ready, -1 failed
|
||||
const GfxShader *cs_mesh, *cs_task; // the compute modules (gfx_shader_own)
|
||||
struct MfbCs *next; // the worker's queue
|
||||
} MfbCs;
|
||||
typedef struct { const GfxShader *ms; uint32_t mask; uint8_t type[32]; const GfxShader *vs; } MfbVs;
|
||||
#define MFB_TAB 4096
|
||||
static MfbCs *mfb_cs_tab[MFB_TAB];
|
||||
static MfbVs mfb_vs_tab[MFB_TAB];
|
||||
static struct {
|
||||
PlatMutex lock;
|
||||
PlatCond wake;
|
||||
PlatThread *th;
|
||||
MfbCs *head, *tail;
|
||||
uint64_t made, failed, pending_skips;
|
||||
} MFB = {.lock = PLAT_MUTEX_INIT};
|
||||
|
||||
static uint32_t mfb_binding_base(const GfxShader *s) { // one past the highest binding the module declares
|
||||
uint64_t n;
|
||||
const uint32_t *w = gfx_shader_spirv(s, &n);
|
||||
uint32_t top = 0;
|
||||
for (uint64_t i = 5; i < n;) {
|
||||
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
||||
if (!len) break;
|
||||
if (op == 71 && len == 4 && w[i + 2] == 33 && w[i + 3] > top) top = w[i + 3]; // OpDecorate Binding
|
||||
i += len;
|
||||
}
|
||||
return top + 1;
|
||||
}
|
||||
|
||||
// The compute module of stage (6 task, 7 mesh) of e, as a shader of the runtime's own with the stage's resources and
|
||||
// the fallback's buffers.
|
||||
static const GfxShader *mfb_module(const MfbCs *e, int stage, char *err, size_t cap) {
|
||||
const GfxShader *src = stage == 6 ? e->ts : e->ms;
|
||||
uint64_t n, on = 0;
|
||||
const uint32_t *w = gfx_shader_spirv(src, &n);
|
||||
uint32_t base = mfb_binding_base(src);
|
||||
uint32_t *out = stage == 6 ? meshfb_task_cs(w, n, &e->ti, base, &on, err, cap)
|
||||
: meshfb_mesh_cs(w, n, &e->mi, base, e->ti.payload_bytes / 4, &on, err, cap);
|
||||
if (!out) return NULL;
|
||||
static const int task_slots[1] = {MESHFB_CB_TASK}, mesh_slots[3] = {MESHFB_CB_TASK, MESHFB_CB_MESH, MESHFB_CB_LIST};
|
||||
char *res = stage == 6 ? meshfb_res_json(gfx_shader_res_json(src), base, task_slots, 1)
|
||||
: e->mi.has_task ? meshfb_res_json(gfx_shader_res_json(src), base, mesh_slots, 3)
|
||||
: meshfb_res_json(gfx_shader_res_json(src), base + 1, mesh_slots + 1, 1);
|
||||
const GfxShader *s = res ? gfx_shader_own(out, on, res) : NULL;
|
||||
if (!s) snprintf(err, cap, "the converted module or its reflection is malformed");
|
||||
free(out), free(res);
|
||||
return s;
|
||||
}
|
||||
|
||||
static void mfb_make(MfbCs *e) {
|
||||
char err[200] = "";
|
||||
const VkPipeInfo *info;
|
||||
int ok = (e->cs_mesh = mfb_module(e, 7, err, sizeof err)) != NULL;
|
||||
if (ok && e->ts) ok = (e->cs_task = mfb_module(e, 6, err, sizeof err)) != NULL;
|
||||
if (ok && !vk_pipeline_compute(e->cs_mesh, &info)) ok = 0, snprintf(err, sizeof err, "the mesh stage's compute pipeline failed");
|
||||
if (ok && e->ts && !vk_pipeline_compute(e->cs_task, &info)) ok = 0, snprintf(err, sizeof err, "the task stage's compute pipeline failed");
|
||||
plat_mutex_lock(&MFB.lock);
|
||||
uint64_t nth = ok ? ++MFB.made : ++MFB.failed;
|
||||
plat_mutex_unlock(&MFB.lock);
|
||||
if (!ok && (nth <= 8 || !(nth & (nth - 1))))
|
||||
rt_log("vk: mesh fallback: a program's modules could not be made (%s): its draws are skipped (%llu such programs)", err,
|
||||
(unsigned long long)nth);
|
||||
__atomic_store_n(&e->state, ok ? 1 : -1, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
static void mfb_worker(void *arg) {
|
||||
(void)arg;
|
||||
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); // as the pipeline workers' optimised builds: the game's threads go first
|
||||
for (;;) {
|
||||
plat_mutex_lock(&MFB.lock);
|
||||
while (!MFB.head) plat_cond_wait(&MFB.wake, &MFB.lock);
|
||||
MfbCs *e = MFB.head;
|
||||
if (!(MFB.head = e->next)) MFB.tail = NULL;
|
||||
plat_mutex_unlock(&MFB.lock);
|
||||
mfb_make(e);
|
||||
}
|
||||
}
|
||||
|
||||
// The entry of (ms, ts), made and queued at its first draw; NULL (logged) when the program's layout is not carried.
|
||||
static MfbCs *mfb_cs_get(VkExec *ex, const GfxShader *ms, const GfxShader *ts) {
|
||||
uint32_t i = (uint32_t)((((uintptr_t)ms >> 4) * 0x9e3779b1u) ^ (((uintptr_t)ts >> 4) * 0x85ebca6bu)) & (MFB_TAB - 1);
|
||||
for (uint32_t k = 0; k < MFB_TAB; k++, i = (i + 1) & (MFB_TAB - 1)) {
|
||||
MfbCs *e = mfb_cs_tab[i];
|
||||
if (e && (e->ms != ms || e->ts != ts)) continue;
|
||||
if (e) return e;
|
||||
const uint8_t *ctl, *code;
|
||||
uint32_t len;
|
||||
char err[160] = "";
|
||||
e = calloc(1, sizeof *e);
|
||||
if (!e) rt_fatal("vk: out of memory");
|
||||
e->ms = ms, e->ts = ts, e->state = -1;
|
||||
mfb_cs_tab[i] = e; // a program that is refused is not looked at again
|
||||
int ok = find_stage_scan(ex->prog, NVN_STAGE_MESH, &ctl, &code, &len) && len >= 0x80 &&
|
||||
meshfb_info(ctl, code, 7, &e->mi, err, sizeof err);
|
||||
if (ok && ts) ok = find_stage_scan(ex->prog, NVN_STAGE_TASK, &ctl, &code, &len) && meshfb_info(ctl, code, 6, &e->ti, err, sizeof err);
|
||||
if (ok && e->mi.has_task != (ts != NULL)) ok = 0, snprintf(err, sizeof err, "the mesh stage %s a task stage and the program has %s",
|
||||
e->mi.has_task ? "expects" : "does not expect", ts ? "one" : "none");
|
||||
if (!ok) {
|
||||
warn_once(W_MESH, "mesh fallback: a program is not carried (%s): its draws are skipped", *err ? err : "no stage data");
|
||||
return e;
|
||||
}
|
||||
e->state = 0;
|
||||
if (!vk_pipeline_async_on()) {
|
||||
mfb_make(e);
|
||||
return e;
|
||||
}
|
||||
plat_mutex_lock(&MFB.lock);
|
||||
if (!MFB.th) plat_cond_init(&MFB.wake), MFB.th = plat_thread_new(mfb_worker, NULL, "mesh fallback");
|
||||
if (MFB.tail) MFB.tail->next = e;
|
||||
else MFB.head = e;
|
||||
MFB.tail = e;
|
||||
plat_cond_signal(&MFB.wake);
|
||||
plat_mutex_unlock(&MFB.lock);
|
||||
return e;
|
||||
}
|
||||
rt_fatal("vk: mesh fallback: more than %d task and mesh program pairs", MFB_TAB);
|
||||
}
|
||||
|
||||
// The vertex stage that draws e's mesh buffer slots for fragment shader fs: an output for every input fs reads, in
|
||||
// its type.
|
||||
static const GfxShader *mfb_vs_get(const MfbCs *e, const GfxShader *fs) {
|
||||
GfxShaderIo fin[32];
|
||||
uint8_t type[32] = {0};
|
||||
uint32_t mask = fs ? gfx_shader_inputs(fs, fin) : 0;
|
||||
for (int g = 0; g < 32; g++)
|
||||
if (mask >> g & 1) type[g] = fin[g].type;
|
||||
uint32_t i = (uint32_t)((((uintptr_t)e->ms >> 4) * 0x9e3779b1u) ^ (mask * 0x85ebca6bu)) & (MFB_TAB - 1);
|
||||
for (uint32_t k = 0; k < MFB_TAB; k++, i = (i + 1) & (MFB_TAB - 1)) {
|
||||
MfbVs *v = &mfb_vs_tab[i];
|
||||
if (v->ms && (v->ms != e->ms || v->mask != mask || memcmp(v->type, type, 32))) continue;
|
||||
if (v->ms) return v->vs;
|
||||
static const char res[] = "{\"key\":\"meshfb\",\"stage\":\"vs\",\"notes\":[],\"cbufs\":[],\"sbufs\":[{\"binding\": 1, "
|
||||
"\"slot\": -1, \"cb_slot\": 66, \"cb_offset\": 0, \"flags\": 0}],\"textures\":[],\"images\":[]}";
|
||||
_Static_assert(MESHFB_CB_MESH == 66, "the vertex stage's reflection names the mesh buffer");
|
||||
uint64_t n = 0;
|
||||
uint32_t *w = meshfb_vs(&e->mi, mask, type, &n);
|
||||
v->ms = e->ms, v->mask = mask, memcpy(v->type, type, 32);
|
||||
v->vs = w ? gfx_shader_own(w, n, res) : NULL;
|
||||
free(w);
|
||||
if (!v->vs) warn_once(W_MESH, "mesh fallback: a vertex stage could not be made: its draws are skipped");
|
||||
return v->vs;
|
||||
}
|
||||
rt_fatal("vk: mesh fallback: more than %d vertex stages", MFB_TAB);
|
||||
}
|
||||
|
||||
// One of the draw's compute dispatches: module cs with the bindings of NVN stage `stage`, its group counts read from
|
||||
// the record at `at`. 0 when its pipeline is not there.
|
||||
static int mfb_dispatch(VkExec *ex, const GfxShader *cs, int stage, VkBuffer rec, VkDeviceSize at) {
|
||||
const VkPipeInfo *info = NULL;
|
||||
VkPipeline p = vk_pipeline_compute(cs, &info);
|
||||
if (!p || !info || (info->pool && !pool_on())) return 0;
|
||||
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
|
||||
mfb_stage = stage;
|
||||
bind_resources(ex, info);
|
||||
mfb_stage = 0;
|
||||
VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0);
|
||||
vkCmdDispatchIndirect(ex->cb, rec, at);
|
||||
vk_pass_mark_end(ex->cb, m);
|
||||
ex->recorded++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// A mesh draw of `count` workgroups, or of the count in the game's record at ind (4 bytes, GPU-written).
|
||||
static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) {
|
||||
if (ex->render_disabled) {
|
||||
vk_cond_draws_skip += ex->has_render_cond;
|
||||
draw_skipped(ex, 1, SK_DISABLED);
|
||||
return;
|
||||
}
|
||||
if (!ex->prog) { draw_skipped(ex, 1, SK_NO_PROGRAM); return; }
|
||||
cb_res(ex);
|
||||
dummies_init();
|
||||
int has_ms, has_ts, has_fs, w_ms = 0, w_ts = 0, w_fs = 0;
|
||||
const GfxShader *ms = shader_for(ex, NVN_STAGE_MESH, &has_ms, &w_ms, 0);
|
||||
const GfxShader *ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &w_ts, 0);
|
||||
const GfxShader *fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &w_fs, 0);
|
||||
if (!has_ms) { draw_skipped(ex, 1, SK_NO_VERTEX_STAGE); return; }
|
||||
if (w_ms || w_ts || w_fs) {
|
||||
vk_exec_ahead(ex);
|
||||
perf_count(PERF_CNT_SHADER_SKIP);
|
||||
draw_skipped(ex, 1, SK_SHADER_PENDING);
|
||||
return;
|
||||
}
|
||||
if (!ms || (has_ts && !ts) || (has_fs && !fs)) { draw_skipped(ex, 1, SK_SHADER_FAILED); return; }
|
||||
MfbCs *c = mfb_cs_get(ex, ms, ts);
|
||||
int state = __atomic_load_n(&c->state, __ATOMIC_ACQUIRE);
|
||||
if (!state && !vk_exec_targets_recent(ex, pipe_skip_mode() >= 1)) {
|
||||
// a target the game does not render every frame: the draw may not come again, so it waits for its modules
|
||||
// from the frame's wait pool, as such a draw waits for a pipeline (prepare_draw_ex)
|
||||
uint64_t *pool = frame_wait(), t0 = kern_now_ns();
|
||||
while (!state && pool_ms(pool, t0)) plat_sleep_ms(1), state = __atomic_load_n(&c->state, __ATOMIC_ACQUIRE);
|
||||
pool_charge(pool, t0);
|
||||
}
|
||||
if (!state) { MFB.pending_skips++; draw_skipped(ex, 1, SK_PIPE_PENDING); return; }
|
||||
const GfxShader *vs = state > 0 ? mfb_vs_get(c, fs) : NULL;
|
||||
if (!vs || !vk_meshfb_init()) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; }
|
||||
uint32_t verts = c->mi.maxp * c->mi.ipp, slot_words = meshfb_slot_words(&c->mi);
|
||||
uint32_t most = vk_ctx()->props.limits.maxComputeWorkGroupCount[0];
|
||||
if (count > most) count = most; // a count the GPU wrote is not looked at here: the dispatch reads it as it is
|
||||
VkBuffer rec;
|
||||
int cmds;
|
||||
vk_exec_barrier(ex); // ends the pass; the culling's record and the last draw's reads come before what follows
|
||||
VkDeviceSize at = vk_meshfb_record(ex->cb, ind ? ind->buffer : VK_NULL_HANDLE, ind ? ind->offset : 0, count, ts != NULL,
|
||||
verts, slot_words, &rec, &cmds);
|
||||
ex->recorded += (uint32_t)cmds;
|
||||
vk_exec_barrier(ex);
|
||||
if (ts) {
|
||||
if (!mfb_dispatch(ex, c->cs_task, NVN_STAGE_TASK, rec, at)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; }
|
||||
vk_exec_barrier(ex);
|
||||
vk_meshfb_scan(ex->cb, at, c->ti.payload_bytes / 4, c->ti.threads, slot_words, verts);
|
||||
B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own
|
||||
ex->recorded += 4;
|
||||
vk_exec_barrier(ex);
|
||||
}
|
||||
if (!mfb_dispatch(ex, c->cs_mesh, NVN_STAGE_MESH, rec, at + 16)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; }
|
||||
vk_exec_barrier(ex);
|
||||
vk_meshfb_note(ex->cb, at);
|
||||
ex->recorded++;
|
||||
vk_exec_barrier(ex);
|
||||
c0_binst = 0;
|
||||
mfb_cur_vs = vs;
|
||||
int prim = c->mi.ipp == 1 ? NVN_DRAW_PRIMITIVE_POINTS : c->mi.ipp == 2 ? NVN_DRAW_PRIMITIVE_LINES : NVN_DRAW_PRIMITIVE_TRIANGLES;
|
||||
if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 2)) return;
|
||||
vk_exec_frame.draws++;
|
||||
vkCmdDrawIndirect(ex->cb, rec, at + 32, 1, sizeof(VkDrawIndirectCommand));
|
||||
ex->recorded++;
|
||||
}
|
||||
|
||||
static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) {
|
||||
PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn();
|
||||
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
|
||||
static int on = -1;
|
||||
if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis)
|
||||
if (!d->count || !on) return;
|
||||
if (vk_ctx()->mesh_fallback) {
|
||||
if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first);
|
||||
mfb_draw(ex, d->count, NULL);
|
||||
return;
|
||||
}
|
||||
PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn();
|
||||
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
|
||||
c0_binst = 0;
|
||||
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;
|
||||
if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first);
|
||||
@@ -4980,13 +5273,21 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) {
|
||||
// MESH_RING entries outlast the frames in flight: an entry is rewritten only after a barrier behind its last draw.
|
||||
enum { MESH_RING = 4096 };
|
||||
static VkBuffer mesh_ring;
|
||||
static VmaAllocation mesh_ring_alloc;
|
||||
static uint32_t mesh_ring_next;
|
||||
|
||||
// NX2NAVI_GPU_PASSES: the workgroup counts of the indirect mesh draws of the timed frames, copied to host memory.
|
||||
int vk_gpu_passes_capturing(void);
|
||||
enum { MESH_COUNTS = 4096 };
|
||||
static VkBuffer mesh_count_buf;
|
||||
static VmaAllocation mesh_count_alloc;
|
||||
static uint32_t *mesh_count_map, mesh_count_n, mesh_count_was;
|
||||
|
||||
static void mesh_ring_release(VkCtx *c) { // vk_exec_draw_shutdown: a probe ends with the device idle
|
||||
if (mesh_ring) vmaDestroyBuffer(c->vma, mesh_ring, mesh_ring_alloc);
|
||||
if (mesh_count_buf) vmaDestroyBuffer(c->vma, mesh_count_buf, mesh_count_alloc);
|
||||
mesh_ring = mesh_count_buf = VK_NULL_HANDLE, mesh_count_map = NULL;
|
||||
}
|
||||
uint32_t vk_exec_mesh_counts(const uint32_t **out) {
|
||||
*out = mesh_count_map;
|
||||
return mesh_count_map ? mesh_count_n : 0;
|
||||
@@ -5001,8 +5302,7 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) {
|
||||
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
||||
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
||||
VmaAllocationInfo info;
|
||||
VmaAllocation a;
|
||||
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_count_buf, &a, &info) != VK_SUCCESS) return;
|
||||
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_count_buf, &mesh_count_alloc, &info) != VK_SUCCESS) return;
|
||||
mesh_count_map = info.pMappedData;
|
||||
}
|
||||
if (mesh_count_n >= MESH_COUNTS) return;
|
||||
@@ -5053,8 +5353,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING,
|
||||
.usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
||||
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
||||
VmaAllocation a;
|
||||
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &a, NULL) != VK_SUCCESS) {
|
||||
if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &mesh_ring_alloc, NULL) != VK_SUCCESS) {
|
||||
mesh_ring = VK_NULL_HANDLE;
|
||||
return 0;
|
||||
}
|
||||
@@ -5107,7 +5406,8 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) {
|
||||
if (vk_ctx()->has_mesh_shader)
|
||||
fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT");
|
||||
}
|
||||
if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
|
||||
int fallback = vk_ctx()->mesh_fallback;
|
||||
if (!fn && !fallback) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped");
|
||||
static int on = -1;
|
||||
if (on < 0) on = rt_knob("MESH", "1");
|
||||
if (!on) return;
|
||||
@@ -5117,6 +5417,12 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) {
|
||||
(unsigned long long)d->ind);
|
||||
c0_binst = 0;
|
||||
VK_MEM_USE(ip);
|
||||
if (fallback) {
|
||||
mfb_draw(ex, 0, &ip);
|
||||
vk_mem_count(ip.cls, VK_MUSE_IND, 4);
|
||||
VK_MEM_BOUND(ip);
|
||||
return;
|
||||
}
|
||||
VkDeviceSize at;
|
||||
if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped");
|
||||
if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return;
|
||||
|
||||
215
runtime/vk_mesh_fb.c
Normal file
215
runtime/vk_mesh_fb.c
Normal file
@@ -0,0 +1,215 @@
|
||||
// The mesh fallback's buffers and records. A task or mesh program runs as a compute dispatch that writes a storage
|
||||
// buffer (mesh_fb.c), and the draw reads it:
|
||||
// task buffer a task workgroup's payload at its index, payload-size apart
|
||||
// list per mesh workgroup after a task stage: its task and its index among that task's (meshfb_scan.comp)
|
||||
// mesh buffer a mesh workgroup's vertex lines and primitive area at its slot
|
||||
// records per draw 64 bytes: the indirect arguments of its task dispatch, its mesh dispatch and its draw
|
||||
// The three buffers are reused by every draw: a draw's dispatches write them and its vertex stage has read them
|
||||
// before the next draw's dispatches run, with a barrier in between (vk_exec_draw.c mfb_draw), so their size is what
|
||||
// the largest draw needs, not what a frame's draws need together. mesh_fallback_mb sets it: an eighth for the task
|
||||
// buffer, the rest for the mesh buffer. Workgroups past either buffer's end write nothing and draw nothing (the
|
||||
// modules check the buffer's length); the draws that happened to are counted and the log names the size they needed.
|
||||
// A record is used by one draw and written again only after MESHFB_RECS further draws.
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "cfg.h"
|
||||
#include "host.h"
|
||||
#include "mesh_fb.h"
|
||||
#include "vk.h"
|
||||
#include "vk_mesh_fb.h"
|
||||
#include "meshfb_scan_spv.h"
|
||||
|
||||
enum { MESHFB_RECS = 4096, REC_BYTES = 64 };
|
||||
|
||||
static struct {
|
||||
int tried, ok;
|
||||
VkBuffer task, mesh, list, rec, stat;
|
||||
VmaAllocation task_a, mesh_a, list_a, rec_a, stat_a;
|
||||
VkDeviceSize task_bytes, mesh_bytes, list_bytes; // as one storage binding spans them
|
||||
const uint32_t *stat_map; // per record: mesh workgroups and task workgroups it asked for
|
||||
uint32_t cap_slots[MESHFB_RECS], slot_words[MESHFB_RECS], cap_tasks[MESHFB_RECS]; // what the record's draw had room for
|
||||
uint8_t noted[MESHFB_RECS];
|
||||
uint32_t next;
|
||||
uint64_t draws, short_draws, worst_bytes;
|
||||
PFN_vkCmdPushDescriptorSetKHR push;
|
||||
VkDescriptorSetLayout dsl;
|
||||
VkPipelineLayout layout;
|
||||
VkPipeline scan;
|
||||
} M;
|
||||
|
||||
static int make_buffer(VkCtx *c, VkDeviceSize size, VkBufferUsageFlags usage, int host, VkBuffer *b, VmaAllocation *a, void **map) {
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size, .usage = usage};
|
||||
VmaAllocationCreateInfo ai = {.usage = host ? VMA_MEMORY_USAGE_AUTO_PREFER_HOST : VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
||||
.flags = host ? VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT : 0};
|
||||
VmaAllocationInfo info;
|
||||
if (vmaCreateBuffer(c->vma, &bi, &ai, b, a, &info) != VK_SUCCESS) return *b = VK_NULL_HANDLE, 0;
|
||||
if (map) *map = info.pMappedData;
|
||||
return !host || info.pMappedData;
|
||||
}
|
||||
|
||||
int vk_meshfb_init(void) {
|
||||
if (M.tried) return M.ok;
|
||||
M.tried = 1;
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!c) return 0;
|
||||
uint64_t mb = (uint64_t)cfg_int("NX2NAVI_MESH_FALLBACK_MB");
|
||||
if (mb < 16) mb = 16;
|
||||
if (mb > 4096) mb = 4096;
|
||||
VkDeviceSize range = c->props.limits.maxStorageBufferRange & ~(VkDeviceSize)3, task = (mb << 20) / 8, mesh = (mb << 20) - task;
|
||||
VkDeviceSize list = mesh / 128; // an entry per slot of 512 bytes, and no mesh program's slot is smaller
|
||||
void *map = NULL;
|
||||
const VkBufferUsageFlags st = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
|
||||
if (!make_buffer(c, task, st, 0, &M.task, &M.task_a, NULL) || !make_buffer(c, mesh, st, 0, &M.mesh, &M.mesh_a, NULL) ||
|
||||
!make_buffer(c, list, st, 0, &M.list, &M.list_a, NULL) ||
|
||||
!make_buffer(c, (VkDeviceSize)MESHFB_RECS * REC_BYTES,
|
||||
st | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT, 0,
|
||||
&M.rec, &M.rec_a, NULL) ||
|
||||
!make_buffer(c, (VkDeviceSize)MESHFB_RECS * 8, VK_BUFFER_USAGE_TRANSFER_DST_BIT, 1, &M.stat, &M.stat_a, &map)) {
|
||||
rt_log("vk: mesh fallback: its buffers (%llu MiB, mesh_fallback_mb) could not be allocated: task and mesh draws are skipped",
|
||||
(unsigned long long)mb);
|
||||
return 0;
|
||||
}
|
||||
M.stat_map = map;
|
||||
memset(map, 0, (size_t)MESHFB_RECS * 8);
|
||||
M.task_bytes = task < range ? task : range, M.mesh_bytes = mesh < range ? mesh : range, M.list_bytes = list < range ? list : range;
|
||||
|
||||
M.push = (PFN_vkCmdPushDescriptorSetKHR)vk_dpa(c->device, "vkCmdPushDescriptorSetKHR");
|
||||
VkDescriptorSetLayoutBinding b[4];
|
||||
for (int i = 0; i < 4; i++)
|
||||
b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
||||
.descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
|
||||
VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
||||
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
|
||||
.bindingCount = 4, .pBindings = b};
|
||||
VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 24};
|
||||
VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof meshfb_scan_spv,
|
||||
.pCode = meshfb_scan_spv};
|
||||
VkShaderModule m = VK_NULL_HANDLE;
|
||||
if (!M.push || vkCreateDescriptorSetLayout(c->device, &dci, NULL, &M.dsl) != VK_SUCCESS) goto fail;
|
||||
VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
|
||||
.pSetLayouts = &M.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
|
||||
if (vkCreatePipelineLayout(c->device, &li, NULL, &M.layout) != VK_SUCCESS ||
|
||||
vkCreateShaderModule(c->device, &mi, NULL, &m) != VK_SUCCESS)
|
||||
goto fail;
|
||||
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
||||
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
||||
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
||||
.layout = M.layout};
|
||||
VkResult res = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &M.scan);
|
||||
vkDestroyShaderModule(c->device, m, NULL);
|
||||
if (res != VK_SUCCESS) goto fail;
|
||||
rt_log("vk: mesh fallback: %llu MiB for task payloads, %llu MiB for mesh output (mesh_fallback_mb = %llu)",
|
||||
(unsigned long long)(M.task_bytes >> 20), (unsigned long long)(M.mesh_bytes >> 20), (unsigned long long)mb);
|
||||
return M.ok = 1;
|
||||
fail:
|
||||
rt_log("vk: mesh fallback: the scan pipeline could not be created: task and mesh draws are skipped");
|
||||
return 0;
|
||||
}
|
||||
|
||||
VkDescriptorBufferInfo vk_meshfb_buffer(int cb_slot) {
|
||||
if (cb_slot == MESHFB_CB_TASK) return (VkDescriptorBufferInfo){M.task, 0, M.task_bytes};
|
||||
if (cb_slot == MESHFB_CB_LIST) return (VkDescriptorBufferInfo){M.list, 0, M.list_bytes};
|
||||
return (VkDescriptorBufferInfo){M.mesh, 0, M.mesh_bytes};
|
||||
}
|
||||
|
||||
uint64_t vk_meshfb_mesh_words(void) { return M.mesh_bytes / 4; }
|
||||
|
||||
// The record about to be written again: what its last draw asked for against what it had room for.
|
||||
static void record_check(uint32_t i) {
|
||||
if (!M.noted[i]) return;
|
||||
M.noted[i] = 0;
|
||||
uint32_t want = M.stat_map[2 * i], tasks = M.stat_map[2 * i + 1];
|
||||
if (want <= M.cap_slots[i] && tasks <= M.cap_tasks[i]) return;
|
||||
uint64_t need = (uint64_t)want * M.slot_words[i] * 4;
|
||||
if (need > M.worst_bytes) M.worst_bytes = need;
|
||||
uint64_t n = ++M.short_draws;
|
||||
if (n <= 4 || !(n & (n - 1))) {
|
||||
char from[96] = "";
|
||||
if (tasks) snprintf(from, sizeof from, " from %u task workgroups (room for %u)", tasks, M.cap_tasks[i]);
|
||||
rt_log("vk: mesh fallback: a draw's programs emitted %u mesh workgroups of %u bytes%s, and the mesh buffer holds %u: "
|
||||
"what did not fit was not drawn (%llu such draws of %llu; mesh_fallback_mb of %llu or more would hold the "
|
||||
"largest so far)", want, M.slot_words[i] * 4, from, M.cap_slots[i], (unsigned long long)n,
|
||||
(unsigned long long)M.draws, (unsigned long long)((M.worst_bytes * 8 / 7 >> 20) + 1));
|
||||
}
|
||||
}
|
||||
|
||||
VkDeviceSize vk_meshfb_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, uint32_t count, int has_task,
|
||||
uint32_t verts, uint32_t slot_words, VkBuffer *rec, int *cmds) {
|
||||
uint32_t i = M.next++ % MESHFB_RECS;
|
||||
VkDeviceSize at = (VkDeviceSize)i * REC_BYTES;
|
||||
record_check(i);
|
||||
M.draws++;
|
||||
M.slot_words[i] = slot_words, M.cap_slots[i] = (uint32_t)(M.mesh_bytes / 4 / (slot_words ? slot_words : 1));
|
||||
M.cap_tasks[i] = 0xffffffffu; // vk_meshfb_scan knows the payload size
|
||||
if (has_task && M.cap_slots[i] > M.list_bytes / 4) M.cap_slots[i] = (uint32_t)(M.list_bytes / 4);
|
||||
*rec = M.rec;
|
||||
int n = 0;
|
||||
vkCmdFillBuffer(cb, M.rec, at, REC_BYTES, 1), n++;
|
||||
// where the workgroup count goes: the task dispatch, or the mesh dispatch, the draw's instances and the report
|
||||
VkDeviceSize to[3] = {has_task ? at : at + 16, at + 36, at + 48};
|
||||
int nto = has_task ? 1 : 3;
|
||||
if (!has_task) {
|
||||
vkCmdFillBuffer(cb, M.rec, at + 32, 4, verts), n++;
|
||||
vkCmdFillBuffer(cb, M.rec, at + 40, 8, 0), n++;
|
||||
vkCmdFillBuffer(cb, M.rec, at + 52, 4, 0), n++;
|
||||
}
|
||||
if (src) {
|
||||
VkBufferCopy cp[3];
|
||||
for (int k = 0; k < nto; k++) cp[k] = (VkBufferCopy){src_off, to[k], 4};
|
||||
vkCmdCopyBuffer(cb, src, M.rec, (uint32_t)nto, cp), n++;
|
||||
} else {
|
||||
for (int k = 0; k < nto; k++) vkCmdFillBuffer(cb, M.rec, to[k], 4, count), n++;
|
||||
}
|
||||
*cmds = n;
|
||||
return at;
|
||||
}
|
||||
|
||||
void vk_meshfb_scan(VkCommandBuffer cb, VkDeviceSize at, uint32_t task_words, uint32_t fan, uint32_t slot_words, uint32_t verts) {
|
||||
uint32_t i = (uint32_t)(at / REC_BYTES);
|
||||
M.cap_tasks[i] = (uint32_t)(M.task_bytes / 4 / (task_words ? task_words : 1));
|
||||
VkDescriptorBufferInfo bi[4] = {{M.rec, 0, VK_WHOLE_SIZE}, {M.task, 0, M.task_bytes}, {M.list, 0, M.list_bytes},
|
||||
{M.mesh, 0, M.mesh_bytes}};
|
||||
VkWriteDescriptorSet w[4];
|
||||
for (int k = 0; k < 4; k++)
|
||||
w[k] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = (uint32_t)k,
|
||||
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
||||
.pBufferInfo = &bi[k]};
|
||||
uint32_t pc[6] = {(uint32_t)(at / 4), task_words, fan, slot_words, verts, vk_ctx()->props.limits.maxComputeWorkGroupCount[0]};
|
||||
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, M.scan);
|
||||
rec_push(M.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, M.layout, 0, 4, w);
|
||||
vkCmdPushConstants(cb, M.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc);
|
||||
vkCmdDispatch(cb, 1, 1, 1);
|
||||
}
|
||||
|
||||
void vk_meshfb_note(VkCommandBuffer cb, VkDeviceSize at) {
|
||||
uint32_t i = (uint32_t)(at / REC_BYTES);
|
||||
VkBufferCopy cp = {at + 48, (VkDeviceSize)i * 8, 8};
|
||||
vkCmdCopyBuffer(cb, M.rec, M.stat, 1, &cp);
|
||||
M.noted[i] = 1;
|
||||
}
|
||||
|
||||
uint64_t vk_meshfb_short(void) {
|
||||
if (!M.ok) return 0;
|
||||
for (uint32_t i = 0; i < MESHFB_RECS; i++) record_check(i);
|
||||
return M.short_draws;
|
||||
}
|
||||
|
||||
void vk_meshfb_shutdown(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!c || !M.tried) return;
|
||||
if (vk_meshfb_short())
|
||||
rt_log("vk: mesh fallback: %llu of %llu draws emitted more than the buffers hold (mesh_fallback_mb of %llu or more would "
|
||||
"hold the largest)", (unsigned long long)M.short_draws, (unsigned long long)M.draws,
|
||||
(unsigned long long)((M.worst_bytes * 8 / 7 >> 20) + 1));
|
||||
if (M.scan) vkDestroyPipeline(c->device, M.scan, NULL);
|
||||
if (M.layout) vkDestroyPipelineLayout(c->device, M.layout, NULL);
|
||||
if (M.dsl) vkDestroyDescriptorSetLayout(c->device, M.dsl, NULL);
|
||||
if (M.task) vmaDestroyBuffer(c->vma, M.task, M.task_a);
|
||||
if (M.mesh) vmaDestroyBuffer(c->vma, M.mesh, M.mesh_a);
|
||||
if (M.list) vmaDestroyBuffer(c->vma, M.list, M.list_a);
|
||||
if (M.rec) vmaDestroyBuffer(c->vma, M.rec, M.rec_a);
|
||||
if (M.stat) vmaDestroyBuffer(c->vma, M.stat, M.stat_a);
|
||||
memset(&M, 0, sizeof M);
|
||||
}
|
||||
29
runtime/vk_mesh_fb.h
Normal file
29
runtime/vk_mesh_fb.h
Normal file
@@ -0,0 +1,29 @@
|
||||
// The mesh fallback's buffers and its records (vk_mesh_fb.c); the modules are mesh_fb.c's, the draws
|
||||
// vk_exec_draw.c's ("mesh fallback").
|
||||
#pragma once
|
||||
#include "vk.h"
|
||||
|
||||
// Creates the buffers and the scan pipeline on first use. 0 when that failed (logged once): the draws are skipped.
|
||||
int vk_meshfb_init(void);
|
||||
// The buffer a module's extra storage binding reads or writes (mesh_fb.h MESHFB_CB_*), whole or as much of it as one
|
||||
// storage binding may span.
|
||||
VkDescriptorBufferInfo vk_meshfb_buffer(int cb_slot);
|
||||
uint64_t vk_meshfb_mesh_words(void); // the mesh buffer's size as a binding sees it, in words
|
||||
|
||||
// A draw's record, 64 bytes of the record buffer (*rec) at the offset returned: the task dispatch at +0, the mesh
|
||||
// dispatch at +16, the draw (VkDrawIndirectCommand) at +32. Records the fills and copies that start it, outside a
|
||||
// pass: the workgroup count from 4 bytes at src_off of src, or `count` when src is VK_NULL_HANDLE. Without a task
|
||||
// stage that count is the mesh dispatch and the draw's instance count; with one it is the task dispatch, and
|
||||
// vk_meshfb_scan writes the rest. slot_words: of the draw's mesh program, for the report of a draw the mesh buffer
|
||||
// was too small for. Returns the number of commands recorded in *cmds.
|
||||
VkDeviceSize vk_meshfb_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_off, uint32_t count, int has_task,
|
||||
uint32_t verts, uint32_t slot_words, VkBuffer *rec, int *cmds);
|
||||
// The pass between the task and the mesh dispatch (shaders/meshfb_scan.comp): binds its own compute pipeline and set.
|
||||
void vk_meshfb_scan(VkCommandBuffer cb, VkDeviceSize at, uint32_t task_words, uint32_t fan, uint32_t slot_words, uint32_t verts);
|
||||
// After the record is complete (the copies or the scan, and a barrier): keeps what the draw asked for where the host
|
||||
// reads it later. One copy command.
|
||||
void vk_meshfb_note(VkCommandBuffer cb, VkDeviceSize at);
|
||||
// Draws so far whose programs emitted more than the buffers hold, the records not looked at yet included: call it
|
||||
// with the device idle (a probe, the shutdown).
|
||||
uint64_t vk_meshfb_short(void);
|
||||
void vk_meshfb_shutdown(void); // the device must be idle
|
||||
@@ -3427,6 +3427,7 @@ static void keys_note(const VkPipeState *s) {
|
||||
static const uint8_t none[32];
|
||||
if (s->fs && !memcmp(gfx_shader_input_id(s->fs), none, 32)) return; // the runtime's own shader (gfx_shader_own)
|
||||
if (s->ts && !memcmp(gfx_shader_input_id(s->ts), none, 32)) return;
|
||||
if (!memcmp(gfx_shader_input_id(s->vs), none, 32)) return; // the mesh fallback's vertex stage (vk_exec_draw.c mfb_vs_get)
|
||||
KeyRec r;
|
||||
rec_fill(&r, s);
|
||||
if (!seen_add(rec_hash(&r))) return;
|
||||
|
||||
@@ -49,7 +49,7 @@ enum {
|
||||
R_SetRasterizerDiscardEnable, R_SetScissorWithCount, R_SetStencilCompareMask, R_SetStencilOp, R_SetStencilReference,
|
||||
R_SetStencilTestEnable, R_SetStencilWriteMask, R_SetViewportWithCount, R_WriteTimestamp, R_WriteTimestamp2,
|
||||
R_PushTpl, R_Push, R_VertexInput, R_Bool, R_PolygonMode, R_BlendEnable, R_BlendEquation, R_WriteMask, R_EndCb,
|
||||
R_PassMark, R_DrawMesh, R_DrawMeshInd, R_ShadingRate,
|
||||
R_PassMark, R_DrawMesh, R_DrawMeshInd, R_ShadingRate, R_DrawIndirect,
|
||||
};
|
||||
|
||||
static uint64_t now_ns(void) { return plat_now_ns(); }
|
||||
@@ -347,6 +347,9 @@ FIXED(DrawIndexed, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, int3
|
||||
#define PACK_DrawIndexedIndirect *a = (A_DrawIndexedIndirect){b, o, n, s}
|
||||
FIXED(DrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s),
|
||||
VkBuffer b; VkDeviceSize o; uint32_t n, s)
|
||||
#define PACK_DrawIndirect *a = (A_DrawIndirect){b, o, n, s}
|
||||
FIXED(DrawIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s),
|
||||
VkBuffer b; VkDeviceSize o; uint32_t n, s)
|
||||
#define PACK_EndQuery *a = (A_EndQuery){p, q}
|
||||
FIXED(EndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), (cb, p, q), VkQueryPool p; uint32_t q)
|
||||
#define PACK_EndRendering (void)a
|
||||
@@ -705,6 +708,7 @@ static void exec_packet(uint32_t op, void *p) {
|
||||
F(Draw, a->n, a->i, a->f, a->fi)
|
||||
F(DrawIndexed, a->n, a->i, a->f, a->v, a->fi)
|
||||
F(DrawIndexedIndirect, a->b, a->o, a->n, a->s)
|
||||
F(DrawIndirect, a->b, a->o, a->n, a->s)
|
||||
F(EndQuery, a->p, a->q)
|
||||
F(FillBuffer, a->b, a->o, a->n, a->v)
|
||||
F(ResetQueryPool, a->p, a->f, a->n)
|
||||
|
||||
@@ -80,6 +80,7 @@ void rec_pass_mark(VkCommandBuffer cb, uint32_t id, int end);
|
||||
X(Draw, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, uint32_t fi), (cb, n, i, f, fi)) \
|
||||
X(DrawIndexed, (VkCommandBuffer cb, uint32_t n, uint32_t i, uint32_t f, int32_t v, uint32_t fi), (cb, n, i, f, v, fi)) \
|
||||
X(DrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s)) \
|
||||
X(DrawIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), (cb, b, o, n, s)) \
|
||||
X(EndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), (cb, p, q)) \
|
||||
X(EndRendering, (VkCommandBuffer cb), (cb)) \
|
||||
X(FillBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkDeviceSize n, uint32_t v), (cb, b, o, n, v)) \
|
||||
@@ -137,6 +138,7 @@ REC_CMDS(REC_DECL)
|
||||
#define vkCmdDraw rc_vkCmdDraw
|
||||
#define vkCmdDrawIndexed rc_vkCmdDrawIndexed
|
||||
#define vkCmdDrawIndexedIndirect rc_vkCmdDrawIndexedIndirect
|
||||
#define vkCmdDrawIndirect rc_vkCmdDrawIndirect
|
||||
#define vkCmdEndQuery rc_vkCmdEndQuery
|
||||
#define vkCmdEndRendering rc_vkCmdEndRendering
|
||||
#define vkCmdFillBuffer rc_vkCmdFillBuffer
|
||||
|
||||
Reference in New Issue
Block a user