diff --git a/docs/architecture.md b/docs/architecture.md index 7c3830dc..683d3638 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -128,7 +128,9 @@ keeps per-core data without locks and relies on that, so the host models it (`ru their stand-in, and the frame's later refreshes skip them (`runtime/vk_exec_draw.c` `pool_refill`). An area load spreads over several frames instead of holding one for 200-400 ms. - Mesh and task shaders run as `VK_EXT_mesh_shader`; indirect mesh draws read NVN's `{count, first}` record through a - ring of Vulkan `{x, y, z}` records filled on the GPU. + ring of Vulkan `{x, y, z}` records filled on the GPU. The copy cannot be recorded inside a pass, so the first such + draw of a pass fills the records of the mesh draws that follow it too, and the pass is not cut at each of them + (`runtime/vk_exec_draw.c` `mesh_ring_entry`). - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link at a new state, an optimised pipeline replacing it from a background thread. A draw whose pipeline is not ready is skipped when its pass is one the game renders frame after frame: one of its targets was rendered in the last few diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index f4877e70..740e2aba 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1310,6 +1310,7 @@ static void evo_resume(VkExec *ex) { if (!c || !cont_ready(c)) continue; progress = 1; evo_switches++; + vk_exec_brk++; vk_exec_pass_end(ex); vk_exec_barrier(ex); uint8_t *mine = malloc(ST_HI - ST_LO); @@ -1463,12 +1464,78 @@ void vk_exec_ahead(VkExec *ex) { } } +// Ops that only store state or draw: nothing they record runs outside a pass, and the walk goes on with the op after +// them. Counter reports and resets are among them: they close and open an occlusion query inside the pass, and the +// value reaches guest memory from the completion thread once the submission's fence has passed (vk_exec_xfer.c +// oq_cb, counter_cb), never from the GPU inside the run; so are the render conditions, which the replay decides on +// the CPU. Ocarina reports a counter after most of its indirect mesh draws. Any other op the walk replays bumps +// vk_exec_brk, and so does a held submission that goes on in between (evo_resume): what was prepared for the draws +// of a run of such ops (vk_exec_draw.c, the indirect mesh draws' records) holds only while it stays the same. +uint32_t vk_exec_brk; + +static int op_linear(uint32_t op) { + switch (op) { + case OP_BEGIN: case OP_JUMP: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_STENCIL: + case OP_POLY_OFFSET: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP: case OP_RASTER_DISCARD: case OP_PRIM_RESTART: + case OP_COPY_STRIDE: case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: + case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO: + case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC: case OP_DRAW_MESH: + case OP_DRAW_MESH_IND: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: + return 1; + default: return 0; + } +} + +#define MESH_RUN_OPS 2048 +static uint32_t mesh_run_stop[OP_COUNT_]; // the op each run of vk_exec_mesh_run ended at, since the start + +// For the perf log: the ops the runs ended at most often. +void vk_exec_mesh_run_stops(char *out, size_t n) { + uint32_t c[OP_COUNT_]; + memcpy(c, mesh_run_stop, sizeof c); + size_t at = 0; + *out = 0; + for (int k = 0; k < 4; k++) { + int best = 0; + for (int i = 1; i < OP_COUNT_; i++) + if (c[i] > c[best]) best = i; + if (!c[best]) break; + int w = snprintf(out + at, n - at, "%s%u at %s", k ? ", " : "", c[best], nvn_op_name((uint32_t)best)); + if (w < 0 || (size_t)w >= n - at) break; + at += (size_t)w, c[best] = 0; + } +} + +// The indirect mesh draws that follow the op being replayed (cur: its payload) with only ops of op_linear between +// them, MESH_RUN_OPS at most: their payloads into out, at most max. 0 when the walk is not at cur. +int vk_exec_mesh_run(const void *cur, const void **out, int max) { + const op_hdr *h = AH.at[AH.depth]; + if (!h || nvn_op_payload(h) != cur) return 0; + int n = 0; + h = nvn_op_next(h); + for (int ops = 0; ops < MESH_RUN_OPS && n < max; ops++) { + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; + if (!op_linear(h->op)) { + mesh_run_stop[h->op]++; + break; + } + if (h->op == OP_JUMP) { + h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target; + continue; + } + if (h->op == OP_DRAW_MESH_IND) out[n++] = nvn_op_payload(h); + h = nvn_op_next(h); + } + return n; +} + static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { for (;;) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); const void *a = nvn_op_payload(h); AH.at[depth] = h, AH.depth = depth; // where vk_exec_ahead reads on from + if (!op_linear(h->op)) vk_exec_brk++; switch (h->op) { case OP_END: return NULL; case OP_BEGIN: break; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 4758dda0..4e4e533a 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -263,6 +263,9 @@ uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after t // signal met, and the sum of how many submissions later it came. void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs); extern uint64_t vk_exec_submit_seq; // replay thread: submissions replayed so far +extern uint32_t vk_exec_brk; // replay thread: bumped by every op that is not state or a draw (vk_exec.c op_linear) +int vk_exec_mesh_run(const void *cur, const void **out, int max); +void vk_exec_mesh_run_stops(char *out, size_t n); // NX2NAVI_EVO_TRACE=1 (diagnostics, vk_exec.c "event order trace"): one token per step of the replay's event order. void vk_exec_evt(char what, const void *event, uint32_t value); void vk_exec_evt_frame(uint64_t frame); // the game's present call: the frame's tokens are closed and logged diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 8e5eea68..98a5da9b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5003,7 +5003,44 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { ex->recorded++; } -static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) { +// NX2NAVI_MESH_AHEAD (default 1; 0 = each draw by itself, "ab" switches with the A/B blocks): the copy of a record +// cannot be recorded inside a pass, so every indirect mesh draw ended its pass, and a pass that draws its meshes one +// after another was cut into as many pieces (a village: 45 of them a frame, each with a gap of its own on the GPU). +// The first such draw of a run now fills the entries of the ones that follow it as well, with nothing but state, +// draws and counter reports between them (vk_exec.c op_linear, vk_exec_mesh_run): those draws then find their entry +// and stay in the pass. The culling compute that writes the records cannot run inside that run, and an entry is taken +// only by the draw it was filled for, at the same record address, while no other op was replayed since +// (vk_exec_brk). In the village 42 of the 45 draws a frame are served that way: 3 gaps with these copies a frame +// where there were 41 (0.04 ms of GPU time for 0.22), and about 40 passes fewer to begin and end. +enum { MESH_AHEAD = 64 }; +static struct { const p_draw_mesh_ind *d; uint64_t ind; VkDeviceSize at; } mesh_ahead[MESH_AHEAD]; +static int mesh_ahead_n, mesh_ahead_i; +static uint32_t mesh_ahead_brk; +static uint64_t mesh_ahead_taken, mesh_ahead_own; // since the start: draws served by an entry filled ahead; the others + +static int mesh_ahead_on(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("MESH_AHEAD", "1"); + return RT_KNOB_ON(mode); +} + +static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { + if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) { + char stops[160]; + vk_exec_mesh_run_stops(stops, sizeof stops); + rt_log("perf: indirect mesh draws: %llu so far, %llu of them with their record filled ahead by an earlier draw of " + "their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1), + (unsigned long long)mesh_ahead_taken, stops); + } + if (mesh_ahead_i < mesh_ahead_n) { + if (mesh_ahead[mesh_ahead_i].d == d && mesh_ahead[mesh_ahead_i].ind == d->ind && mesh_ahead_brk == vk_exec_brk) { + *at = mesh_ahead[mesh_ahead_i++].at; + mesh_ahead_taken++; + return 1; + } + mesh_ahead_n = mesh_ahead_i = 0; // the walk went another way: the entries left are never drawn from + } + mesh_ahead_own++; if (!mesh_ring) { VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING, .usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; @@ -5021,6 +5058,27 @@ static int mesh_ring_entry(VkExec *ex, const VkMemSpan *src, VkDeviceSize *at) { VkBufferCopy cp = {src->offset, *at, 4}; vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); mesh_count_note(ex, src); + mesh_ahead_n = mesh_ahead_i = 0; + if (mesh_ahead_on()) { + const void *run[MESH_AHEAD]; + int n = vk_exec_mesh_run(d, run, MESH_AHEAD); + for (int i = 0; i < n; i++) { + const p_draw_mesh_ind *f = run[i]; + VkMemSpan ip; + if (!vk_mem_resolve(f->ind, 12, &ip) || ip.offset % 4) break; // that draw reports it when the walk gets there + VK_MEM_USE(ip); + VkDeviceSize fa = 12ull * (mesh_ring_next++ % MESH_RING); + vk_gpu_gap_note(VK_GAP_MESH_RING, 12); + vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1); + VkBufferCopy fc = {ip.offset, fa, 4}; + vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc); + mesh_count_note(ex, &ip); + ex->recorded += 2; + mesh_ahead[mesh_ahead_n].d = f, mesh_ahead[mesh_ahead_n].ind = f->ind, mesh_ahead[mesh_ahead_n].at = fa; + mesh_ahead_n++; + } + mesh_ahead_brk = vk_exec_brk; + } VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, @@ -5052,7 +5110,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { c0_binst = 0; VK_MEM_USE(ip); VkDeviceSize at; - if (!mesh_ring_entry(ex, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); + if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; vk_exec_frame.draws++; static uint32_t n_logged;