diff --git a/runtime/cfg.c b/runtime/cfg.c index e546128b..6add4819 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -91,6 +91,7 @@ static const char *const known[] = { // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). static const struct { const char *key, *section; } sections[] = { // the menu's Lama patches tab (ui/menu.cpp tab_lama) + {"MESH_EMU_MB", "performance"}, {"MESH_EMU_TASKS", "performance"}, {"MESH_EMU_BATCH", "performance"}, {"VEC_INDEX_FIX", "performance"}, {"GUEST_OPT", "performance"}, {"PASS_TIMES", "diagnostics"}, {"DUMP_KEY", "diagnostics"}, {"FPS_LIMIT", "frame rate"}, {"FPS60", "frame rate"}, {"RTSS", "frame rate"}, {"REALTIME", "frame rate"}, {"CLOTH_REALTIME", "frame rate"}, {"HIGHFPS_FIX", "frame rate"}, @@ -302,6 +303,7 @@ static const CfgMeta metas[] = { {"MIRROR_HOT", "25", "0..1000", RESTART}, {"FREEZE_IDLE", "1800", "0..100000", RESTART}, {"VRAM_HEAP_LIMIT", "auto", NULL, RESTART}, // vk_device.c vram_heap_limits: auto, 0 or MiB {"UPLOAD_BATCH_MB", "256", NULL, RESTART}, // vk_memory.c: MiB of staging before a batch goes early, 0 no limit + {"MESH_EMU_TASKS", "16384", "256..262144", RESTART}, {"MESH_EMU_BATCH", "1", "0|1", RESTART}, {"VEC_INDEX_FIX", "0", "0|auto|1", RESTART}, {"PASS_TIMES", "0", "0|1", RESTART}, {"DUMP_KEY", "", NULL, RESTART}, // vk_crumbs.c, vk_dump.c // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index eabd4e64..49dd4c48 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1645,10 +1645,15 @@ static void tab_lama(float scale) { opt_toggle("NX2NAVI_ASTC_SCRATCH_KEEP", UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC); opt_group(UI_MENU_LAMA_RENDER); const bool native = vk_mesh_shader_native() != 0; // vk_exec_draw.c: the emulation only where the GPU lacks them + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_toggle("NX2NAVI_MESH_EMULATION", UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC); } + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_slider("NX2NAVI_MESH_EMU_MB", UI_MENU_LAMA_MESH_MB, 64, 2048, UI_MENU_LAMA_MESH_MB_DESC, 64, "%.0f MB"); + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_slider("NX2NAVI_MESH_EMU_TASKS", UI_MENU_LAMA_MESH_TASKS, 4096, 65536, UI_MENU_LAMA_MESH_TASKS_DESC, 4096, "%.0f"); + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); + opt_toggle("NX2NAVI_MESH_EMU_BATCH", UI_MENU_LAMA_MESH_BATCH, UI_MENU_LAMA_MESH_BATCH_DESC); // vk_exec_draw.c opt_toggle("NX2NAVI_TEX_CLOBBER", UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC); static const UiTextId vec[] = {UI_MENU_OFF, UI_MENU_LAMA_VEC_AUTO, UI_MENU_ON}; // gfx_shader.c patch_vec_index opt_choice("NX2NAVI_VEC_INDEX_FIX", UI_MENU_LAMA_VEC_INDEX, vec, 3, UI_MENU_LAMA_VEC_INDEX_DESC); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index ed89bbe4..a497bcda 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -123,6 +123,10 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_LAMA_MESH_NATIVE] = "this GPU has mesh shaders", [UI_MENU_LAMA_MESH_MB] = "Mesh emulation memory", [UI_MENU_LAMA_MESH_MB_DESC] = "Video memory for the emulated meshes' vertices and indices, in MB.", + [UI_MENU_LAMA_MESH_BATCH] = "Mesh emulation before the pass", + [UI_MENU_LAMA_MESH_BATCH_DESC] = "The emulated mesh stages of a render pass run before the pass in a command buffer " + "of their own, instead of splitting the pass at every emulated draw. Off: the " + "split, as before.", [UI_MENU_LAMA_MESH_TASKS] = "Mesh emulation task records", [UI_MENU_LAMA_MESH_TASKS_DESC] = "Task records per submission for the emulated meshes. Raise it when the log reports " "that a submission asked for more.", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 7857fbf3..6ef06b5c 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -342,6 +342,7 @@ typedef enum { UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC, UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC, UI_MENU_LAMA_MESH_NATIVE, UI_MENU_LAMA_MESH_MB, UI_MENU_LAMA_MESH_MB_DESC, UI_MENU_LAMA_MESH_TASKS, UI_MENU_LAMA_MESH_TASKS_DESC, + UI_MENU_LAMA_MESH_BATCH, UI_MENU_LAMA_MESH_BATCH_DESC, UI_MENU_LAMA_VEC_INDEX, UI_MENU_LAMA_VEC_INDEX_DESC, UI_MENU_LAMA_VEC_AUTO, UI_MENU_LAMA_GUEST_OPT, UI_MENU_LAMA_GUEST_OPT_DESC, UI_MENU_LAMA_GUEST_OPT_NOTE, UI_MENU_LAMA_PASS_TIMES, UI_MENU_LAMA_PASS_TIMES_DESC, diff --git a/runtime/vk.h b/runtime/vk.h index 57fcd927..8564149c 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -750,6 +750,9 @@ VkCommandBuffer vk_sync_append_later(int n, int *at); void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n); VkCommandBuffer vk_sync_worker_begin(int w); void vk_sync_worker_end(VkCommandBuffer cb); +// The part's pre-pass command buffer: submitted right before the part being recorded (after its mirror prologue), begun +// with a full barrier at the first call of a part; VK_NULL_HANDLE when it cannot be made. Replay thread, sub_lock held. +VkCommandBuffer vk_sync_pre(void); extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) // Runs fn(arg) on the completion thread once fence is done on the GPU (event writes, counter reports, sync state). // Callbacks run in the order they were added, after those of earlier values; a value already done runs at once there. diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 74a6f35a..6f7dd7f2 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1112,6 +1112,9 @@ static void set_rt(VkExec *ex, const p_set_rt *r) { static int split_n = -1, split_idle, split_in_pass, split_draws, split_at, part_start; static uint64_t split_ns, part_t0; static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits; +int vk_exec_split_now(VkExec *ex) { + return 1; +} static int split_wanted(void) { if (split_draws >= split_at) return 1; @@ -2082,9 +2085,10 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end int begun = 0; const p_set_rt *rt = &ex->rt; *why = GE_LIMIT; - // mesh draws stay on the submitting thread with the mesh fallback (its buffers are shared) and while - // NX2NAVI_GPU_PASSES copies their workgroup counts (one list) - int mesh_serial = vk_ctx()->mesh_fallback || vk_gpu_passes_capturing(); + // mesh draws stay on the submitting thread with the mesh fallback (its buffers are shared), with the mesh emulation + // (no VK_EXT_mesh_shader: its arena, queue and the part's pre-pass command buffer are the submitting thread's) and + // while NX2NAVI_GPU_PASSES copies their workgroup counts (one list) + int mesh_serial = vk_ctx()->mesh_fallback || !vk_ctx()->has_mesh_shader || vk_gpu_passes_capturing(); for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) { *why = GE_RUN; break; } // the walk reports it const void *a = nvn_op_payload(h); diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 47fa8696..837153e0 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -125,6 +125,8 @@ typedef struct VkExec { const void *fb_fs; // feedback_scan's memo: fragment shader, texture bind serial, pass serial uint32_t fb_serial, fb_pass, fb_sampled; uint32_t tex_serial, pass_serial; // bumped by every texture or image bind and pass begin + uint32_t pre_pass; // the pass (pass_serial) the part's pre-pass command buffer serves (vk_sync_pre; + // vk_exec_draw.c mesh emulation); a pass begins after it with another serial } VkExec; // Begins dynamic rendering with the current targets unless a pass is open (a full memory barrier precedes it). Call @@ -140,6 +142,7 @@ void vk_exec_barrier(VkExec *ex); void vk_exec_barrier_dispatch(VkExec *ex); // NX2NAVI_TRACE=perf, every 600 frames: the barriers per frame by kind. void vk_exec_barrier_log(uint64_t frame); +int vk_exec_split_now(VkExec *ex); // vk_texture.c (NX2NAVI_TEX_ALIAS): commands that must run at the replay's current position. vk_exec_inline_begin returns // the command buffer being recorded after a full barrier (an open pass is ended; *reopen says so), or VK_NULL_HANDLE // when this thread is not recording a replay. vk_exec_inline_end begins the pass again when it was open. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 21093d83..6af2736b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5575,6 +5575,7 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { ex->recorded++; } +static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args); // The primitive an emulated draw's mesh stage outputs: point meshes draw point lists, the rest triangles. static int mesh_emu_draw_prim(VkExec *ex) { static struct { const GfxShader *s; int prim; } memo[256]; @@ -5601,6 +5602,9 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); if (!fn && mesh_emu_on()) { if (d->first) warn_once(W_MESH, "an emulated mesh draw from workgroup %u: drawn from workgroup 0", d->first); + if (d->count && rt_knob("MESH", "1")) mesh_emu_draw(ex, d->count, NULL, NULL); + return; + } static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis) if (!d->count || !on) return; @@ -5652,7 +5656,7 @@ uint32_t vk_exec_mesh_counts(const uint32_t **out) { *out = mesh_count_map; return mesh_count_map ? mesh_count_n : 0; } -static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { +static void mesh_count_note(VkExec *ex, VkCommandBuffer cb, const VkMemSpan *src) { uint32_t cap = (uint32_t)vk_gpu_passes_capturing(); if (!cap) { mesh_count_was = 0; return; } if (!mesh_count_was) mesh_count_was = 1, mesh_count_n = 0; @@ -5667,8 +5671,8 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { } if (mesh_count_n >= MESH_COUNTS) return; VkBufferCopy cp = {src->offset, 4ull * mesh_count_n++, 4}; - vkCmdCopyBuffer(ex->cb, src->buffer, mesh_count_buf, 1, &cp); - ex->recorded++; + vkCmdCopyBuffer(cb, src->buffer, mesh_count_buf, 1, &cp); + if (cb == ex->cb) ex->recorded++; } // NX2NAVI_MESH_AHEAD (default 1; 0 = each draw by itself, "ab" switches with the A/B blocks): the copy of a record @@ -5705,7 +5709,11 @@ static int mesh_ahead_on(void) { return RT_KNOB_ON(mode); } -static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { +// A ring entry {count from src, 1, 1}, filled in cb; ex->cb: after a full barrier, which ends the pass (the next draw +// opens it again) and orders the culling's writes before the copy. Another cb (the pre-pass one of the mesh +// emulation) begins with one. d: the draw it is for, which may take an entry filled ahead (MESH_AHEAD, ex->cb only); +// NULL: none (the mesh emulation's). +static int mesh_ring_entry(VkExec *ex, VkCommandBuffer cb, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { DrawCtx *const cx = ex->dc; if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) { char stops[160]; @@ -5714,7 +5722,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan "their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1), (unsigned long long)mesh_ahead_taken, stops); } - if (cx->ma_i < cx->ma_n) { + if (d && cb == ex->cb && cx->ma_i < cx->ma_n) { if (cx->ma[cx->ma_i].d == d && cx->ma[cx->ma_i].ind == d->ind && cx->ma_brk == vk_exec_brk_now()) { *at = cx->ma[cx->ma_i++].at; __atomic_add_fetch(&mesh_ahead_taken, 1, __ATOMIC_RELAXED); @@ -5725,15 +5733,18 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan __atomic_add_fetch(&mesh_ahead_own, 1, __ATOMIC_RELAXED); if (!__atomic_load_n(&mesh_ring, __ATOMIC_ACQUIRE) && (vk_exec_in_chunk() || !mesh_ring_make())) return 0; *at = 12ull * (__atomic_fetch_add(&mesh_ring_next, 1, __ATOMIC_RELAXED) % MESH_RING); - vk_exec_barrier(ex); // ends the pass (the next draw opens it again) and orders the culling's writes before the copy - vk_gpu_gap_note(VK_GAP_MESH_RING, 12); - vkCmdFillBuffer(ex->cb, mesh_ring, *at + 4, 8, 1); + int own = cb == ex->cb; + if (own) { + vk_exec_barrier(ex); + cb = ex->cb; + vk_gpu_gap_note(VK_GAP_MESH_RING, 12); } + vkCmdFillBuffer(cb, mesh_ring, *at + 4, 8, 1); VkBufferCopy cp = {src->offset, *at, 4}; - vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); - mesh_count_note(ex, src); + vkCmdCopyBuffer(cb, src->buffer, mesh_ring, 1, &cp); + mesh_count_note(ex, cb, src); cx->ma_n = cx->ma_i = 0; - if (mesh_ahead_on()) { + if (d && own && mesh_ahead_on()) { const void *run[MESH_AHEAD]; int n = vk_exec_mesh_run(d, run, MESH_AHEAD); for (int i = 0; i < n; i++) { @@ -5746,7 +5757,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1); VkBufferCopy fc = {ip.offset, fa, 4}; vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc); - mesh_count_note(ex, &ip); + mesh_count_note(ex, ex->cb, &ip); ex->recorded += 2; cx->ma[cx->ma_n].d = f, cx->ma[cx->ma_n].ind = f->ind, cx->ma[cx->ma_n].at = fa; cx->ma_n++; @@ -5759,6 +5770,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan .dstAccessMask = VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); + if (own) ex->recorded += 3; return 1; } @@ -5777,6 +5789,11 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { (unsigned long long)d->ind); ex->dc->c0_binst = 0; VK_MEM_USE(ip); + mesh_emu_draw(ex, 0, NULL, &ip); + vk_mem_count(ip.cls, VK_MUSE_IND, 4); + VK_MEM_BOUND(ip); + return; + } if (fallback) { mfb_draw(ex, 0, &ip); vk_mem_count(ip.cls, VK_MUSE_IND, 4); @@ -5874,9 +5891,12 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla .dstAccessMask = da | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); + } } } // count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from +// args: an indirect draw's arguments in game memory (count, 1, 1 copied into a ring entry, which ind then is) +static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args) { if (!me_arena()) return 0; ex->dc->c0_binst = 0; int prim = mesh_emu_draw_prim(ex); @@ -5895,13 +5915,41 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla : "its descriptors cannot be bound for compute"); ME.skipped++; return 0; + } VkPipelineLayout layout = info->layout; int push = info->push, pool = info->pool; VkDescriptorUpdateTemplate tpl_c = info->emu_tpl_c; VkDescriptorSet set0 = ex->dc->B.ds_bound[0]; + // NX2NAVI_MESH_EMU_BATCH (default on): the stages run in the part's pre-pass command buffer (vk_sync_pre), which + // the queue runs before the part, and the pass is not split for them. At the first emulated draw of a pass the part + // pass of 20 emulated draws was 20 passes, each emulated draw a pass end and begin with a full barrier, three more + // barriers and the targets stored and loaded again: on an RX 580 6 to 7 ms of a frame's GPU time went to the gaps + // between passes (gpu_passes, Kokiri Forest, 2026-11-10). + static int batch = -1; + if (batch < 0) { + batch = rt_knob("MESH_EMU_BATCH", "1"); + rt_log("vk: mesh emulation: the stages of a pass's emulated draws %s (NX2NAVI_MESH_EMU_BATCH=%d)", + batch ? "run before the pass, in a command buffer of their own" : "split the pass", batch); } + VkCommandBuffer pre = VK_NULL_HANDLE; + if (batch && ex->in_pass) { + if (ex->pre_pass != ex->pass_serial && vk_exec_split_now(ex)) { + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { // the pass again, in the new part + ex->pre_pass = ex->pass_serial; + set0 = ex->dc->B.ds_bound[0]; } + if (ex->pre_pass == ex->pass_serial) pre = vk_sync_pre(); } + if (!pre) vk_exec_barrier(ex); // ends the pass; what the game wrote before (its culling) is visible to the stages + VkCommandBuffer cb = pre ? pre : ex->cb; + VkMemSpan ring = {0}; + if (args) { + if (!mesh_ring_entry(ex, cb, NULL, args, &ring.offset)) { + warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); + ME.skipped++; + return 0; + ring.buffer = mesh_ring, ind = ˚ + cb = pre ? pre : ex->cb; uint32_t ep = vk_texture_epoch(); if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over ME.epoch = ep; @@ -5925,6 +5973,14 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1); uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), (uint32_t)(ro / 4), ME.iw, ME.icap, ME.vw, ME.vcap, ME.trecs}; } + if (!pre) { + // the draw: prepared again (the pass reopens, every binding is recorded again) + vk_exec_draw_new_cb(); + ex->dirty = VK_EXEC_DIRTY_ALL; + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { + ME.skipped++; + return 0; + } } cb = ex->cb; vkCmdBindIndexBuffer(cb, ME.buf, (VkDeviceSize)ME.iw * 4, VK_INDEX_TYPE_UINT32); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index f9109378..f0cbf4ff 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -764,6 +764,55 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { return &slots[cur_slot]; } +// ---- the part's pre-pass command buffer (vk.h vk_sync_pre; vk_exec_draw.c mesh emulation, NX2NAVI_MESH_EMU_BATCH) +// Begun at the first call of a part, submitted right before the part (after the mirror prologue) and ended there. Its +// own pool: with the recorder on, the recorder thread records the part (cmd_pool) while the replay records this. +static VkCommandPool pre_pool; +static CmdSlot *pre_slots; +static int npre, pre_pair = -1; +static VkCommandBuffer pre_cb; +VkCommandBuffer vk_sync_pre(void) { + if (pre_cb) return pre_cb; + if (!pre_pool) { + VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, + .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, + .queueFamilyIndex = c->gfx_family}; + if (vkCreateCommandPool(c->device, &pi, NULL, &pre_pool) != VK_SUCCESS) return VK_NULL_HANDLE; + } + uint64_t gpu = 0; + vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu); + int i = 0; + while (i < npre && pre_slots[i].fence > gpu) i++; + if (i == npre) { + CmdSlot *n = realloc(pre_slots, sizeof *n * (size_t)(npre + 1)); + if (!n) return VK_NULL_HANDLE; + pre_slots = n; + VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = pre_pool, + .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; + if (vkAllocateCommandBuffers(c->device, &ai, &pre_slots[npre].cb) != VK_SUCCESS) return VK_NULL_HANDLE; + npre++; + } + pre_slots[i].fence = cur_fence; // the submission's value: its parts are done by then + VkCommandBuffer cb = pre_slots[i].cb; + pre_pair = ts_open(cb); + // everything earlier in queue order (the parts before, the mirror prologue) before what it records + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); + pre_cb = cb; + return cb; +// The pre-pass command buffer of the part that goes now, ended; VK_NULL_HANDLE without one. +static VkCommandBuffer pre_end(uint64_t fence) { + VkCommandBuffer cb = pre_cb; + if (!cb) return VK_NULL_HANDLE; + pre_pair = -1; + vkEndCommandBuffer(cb); + pre_cb = VK_NULL_HANDLE; + return cb; +} + // The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is // submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the // part was recorded, sub_lock held; may move slots[]. @@ -878,10 +927,8 @@ static VkCommandBuffer split_part(int exec) { ts_end(cb, cur_fence); if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder vk_upload_flush(0); // the uploads this part asked for precede it in queue order - VkCommandBuffer pro = prologue(c, cur_fence); - VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = part_list(cbi, pro, cb, 1), - .pCommandBufferInfos = cbi}; + VkCommandBuffer pro = prologue(c, cur_fence), pre = pre_end(cur_fence); + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi}; VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; if (exec && exec_tl && exec_sig < cur_fence) @@ -954,7 +1001,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { if (cpu0) perf_replay_cpu(plat_thread_cpu_ns(me) - cpu0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. vk_upload_flush(0); - VkCommandBuffer pro = prologue(c, fence); + VkCommandBuffer pro = prologue(c, fence), pre = pre_end(fence); s = &slots[cur_slot]; VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; uint32_t ncbi = part_list(cbi, pro, s->cb, recorded); @@ -965,10 +1012,8 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}}; int both = exec_tl && exec_sig < fence; if (both) exec_sig = fence; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncbi, - .pCommandBufferInfos = cbi, .signalSemaphoreInfoCount = both ? 2u : 1u, - .pSignalSemaphoreInfos = sig}; - ts_submitted(); + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi, + .signalSemaphoreInfoCount = both ? 2u : 1u, .pSignalSemaphoreInfos = sig}; queue_submit(c, &si, fence); // held or handed to the queue (submit batching) vk_mem_lazy_handed(); s->fence = recorded ? fence : 0;