diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 0f1c228e..4fd2183d 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -66,6 +66,10 @@ # of latency, and can show a dark or flickering frame now and then: the other thread reads vertex and buffer memory # the game has sometimes rewritten by then. # replay_async = 0 +# EXPERIMENTAL, work in progress. Record the draws of a submission on replay worker threads, 0 to 16. Default: 0, the +# replay records everything on one thread. With 1 a worker records each long run of draws while the submitting thread +# waits for it: no faster yet, it checks that the pictures stay the same before several workers record at once. +# replay_workers = 0 # EXPERIMENTAL, THE GAME CRASHES WITH 1. 1 lifts the virtual core gate: guest threads run without the console's limit # of one thread per core, and the game keeps per-core data without locks. With 1 the game crashes or hangs, # usually at start. Keep 0 to play. diff --git a/runtime/cfg.c b/runtime/cfg.c index 3f8193e1..fe041765 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -62,7 +62,7 @@ static const char *const known[] = { "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", - "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", "SCENE_SHARE", + "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", @@ -121,7 +121,8 @@ static const struct { const char *key, *section; } sections[] = { {"BIND_RIGHT", "input"}, {"BIND_DOWN", "input"}, {"BIND_LSTICK_LEFT", "input"}, {"BIND_LSTICK_UP", "input"}, {"BIND_LSTICK_RIGHT", "input"}, {"BIND_LSTICK_DOWN", "input"}, {"BIND_RSTICK_LEFT", "input"}, {"BIND_RSTICK_UP", "input"}, {"BIND_RSTICK_RIGHT", "input"}, {"BIND_RSTICK_DOWN", "input"}, {"BIND_SHAKE", "input"}, - {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"FREE_CORES", "performance"}, + {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, + {"FREE_CORES", "performance"}, {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, @@ -248,6 +249,7 @@ static const CfgMeta metas[] = { {"COOP_ACCEPT", "ask", NULL, LIVE}, // Advanced: scheduling {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, + {"REPLAY_WORKERS", "0", "0..16", RESTART}, {"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART}, // dynamic resolution (dynres.c): the scene follows the GPU time toward DYN_RES_TARGET milliseconds {"DYN_RES", "0", "0|1", LIVE}, {"DYN_RES_TARGET", "16.7", "1..100", LIVE}, {"DYN_RES_STEP", "45", "8..600", LIVE}, diff --git a/runtime/vk.h b/runtime/vk.h index 5bd49b35..ad1d10ec 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -705,6 +705,15 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t // batch) without a signal, and returns a new begun one for the rest of the submission. The submission's timeline value // is signalled by its last part, whose signal covers every earlier part in submission order. VkCommandBuffer vk_sync_split(void); +// Inside vk_exec_record, for the replay workers (vk_exec.c): vk_sync_append ends the command buffer recorded so far, +// puts it and then cbs (recorded by the workers) into the part, in that order, and returns a new begun one for the rest +// of the part; vk_sync_append_room says whether the part has room for n more. A worker claims and begins a command +// buffer of its own pool for the submission being recorded with vk_sync_worker_begin(w) and ends it with +// vk_sync_worker_end, on its own thread, while the submitting thread waits for it. +int vk_sync_append_room(int n); +VkCommandBuffer vk_sync_append(const VkCommandBuffer *cbs, int n); +VkCommandBuffer vk_sync_worker_begin(int w); +void vk_sync_worker_end(VkCommandBuffer cb); extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) // Runs fn(arg) on the completion thread once fence is done on the GPU (event writes, counter reports, sync state). // Callbacks run in the order they were added, after those of earlier values; a value already done runs at once there. diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index e6a9443a..dc7f89ef 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -64,10 +64,83 @@ void vk_exec_frame_written(const tex_t *t) { fw[nfw++].n = 1; } +// ---- draw runs (NX2NAVI_REPLAY_RUNS=1, diagnostics for recording a submission on several threads) +// How the draws of a frame fall into runs that a recording thread of their own could take: a run is the draws between +// two ops that have to stay in the replay's order on one thread (dispatches, copies, texture and buffer clears, events, +// the game's barriers, a held submission going on, the end of a submission). The ops a run carries that a serial front +// pass would have to settle in order before the threads record (counter reports and resets, render conditions, inline +// uniform updates, render target changes, attachment clears) are counted apart. Every 600 presented frames one line: +// the draws per frame, the share of them in runs of at least 16, 64 and 256 draws, the runs, and the breaks by kind. +enum { RB_DISPATCH, RB_COPY, RB_CLEAR, RB_EVENT, RB_BARRIER, RB_HELD, RB_SUBMIT, RB_N }; +enum { RO_COUNTER, RO_COND, RO_UB, RO_RT, RO_ATT_CLEAR, RO_N }; +static const char *const rb_name[RB_N] = {"dispatches", "copies", "clears", "events", "barriers", "held", "submits"}; +static const char *const ro_name[RO_N] = {"counters", "conditions", "uniform updates", "target changes", + "attachment clears"}; +static struct { + int on; + uint32_t draws; // of the run open now + uint64_t total, in16, in64, in256, runs, longest, brk[RB_N], carried[RO_N], frame0; +} RR = {.on = -1}; + +static void runs_break(int why) { + uint32_t d = RR.draws; + RR.brk[why]++; + if (!d) return; + RR.runs++, RR.total += d, RR.draws = 0; + if (d >= 16) RR.in16 += d; + if (d >= 64) RR.in64 += d; + if (d >= 256) RR.in256 += d; + if (d > RR.longest) RR.longest = d; +} + +static void runs_op(uint32_t op) { + switch (op) { + case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: case OP_DRAW_MESH: case OP_DRAW_MESH_IND: RR.draws++; break; + case OP_DISPATCH: case OP_DISPATCH_IND: runs_break(RB_DISPATCH); break; + case OP_COPY_B2B: case OP_COPY_B2T: case OP_COPY_T2B: case OP_COPY_T2T: runs_break(RB_COPY); break; + case OP_CLEAR_TEX: case OP_CLEAR_BUF: runs_break(RB_CLEAR); break; + case OP_SIGNAL_EVENT: case OP_WAIT_EVENT: runs_break(RB_EVENT); break; + case OP_BARRIER: runs_break(RB_BARRIER); break; + case OP_REPORT_COUNTER: case OP_RESET_COUNTER: RR.carried[RO_COUNTER]++; break; + case OP_RENDER_ENABLE: case OP_RENDER_COND: RR.carried[RO_COND]++; break; + case OP_UPDATE_UB: RR.carried[RO_UB]++; break; + case OP_SET_RT: RR.carried[RO_RT]++; break; + case OP_CLEAR_COLOR: case OP_CLEAR_DS: RR.carried[RO_ATT_CLEAR]++; break; + default: break; + } +} + +static int runs_on(void) { + if (RR.on < 0) RR.on = rt_knob("REPLAY_RUNS", "0"); + return RR.on; +} + +static void runs_frame(uint64_t frame) { + if (!runs_on() || frame % 600) return; + if (RR.frame0 && frame > RR.frame0) { + double f = (double)(frame - RR.frame0), t = RR.total ? (double)RR.total : 1.0; + char b[256], c[256]; + int o = 0; + for (int k = 0; k < RB_N; k++) + o += snprintf(b + o, sizeof b - (size_t)o, "%s %s %.1f", k ? "," : "", rb_name[k], RR.brk[k] / f); + o = 0; + for (int k = 0; k < RO_N; k++) + o += snprintf(c + o, sizeof c - (size_t)o, "%s %s %.1f", k ? "," : "", ro_name[k], RR.carried[k] / f); + rt_log("vk_exec: frame %llu: draw runs: %.0f draws a frame in %.1f runs; in runs of 16 or more %.0f%%, " + "64 or more %.0f%%, 256 or more %.0f%%, longest %llu; breaks a frame:%s; carried a frame:%s", + (unsigned long long)frame, RR.total / f, RR.runs / f, 100.0 * RR.in16 / t, 100.0 * RR.in64 / t, + 100.0 * RR.in256 / t, (unsigned long long)RR.longest, b, c); + } + uint32_t open = RR.draws; + memset(&RR.total, 0, sizeof RR - offsetof(typeof(RR), total)); + RR.draws = open, RR.frame0 = frame; +} + uint64_t vk_exec_submit_seq; void vk_exec_frame_log(const tex_t *p, uint64_t frame) { vk_exec_cond_frame_end(frame); // a frame boundary in replay order (present_image runs on the replay thread) + runs_frame(frame); if (!(rt_trace & RT_TRACE_GFX)) return; VkExecFrame *f = &vk_exec_frame; uint64_t ew, eu, el, ed; @@ -1319,6 +1392,7 @@ static void evo_resume(VkExec *ex) { progress = 1; evo_switches++; vk_exec_brk++; + if (RR.on > 0) runs_break(RB_HELD); vk_exec_pass_end(ex); vk_exec_barrier(ex); uint8_t *mine = malloc(ST_HI - ST_LO); @@ -1537,85 +1611,280 @@ int vk_exec_mesh_run(const void *cur, const void **out, int max) { return n; } +// One op of the stream other than those that steer the walk (END, JUMP, CALL and WAIT_EVENT: walk). +static void op_one(VkExec *ex, const op_hdr *h) { + const void *a = nvn_op_payload(h); + switch (h->op) { + case OP_BEGIN: break; + case OP_SET_RT: set_rt(ex, a); break; + case OP_CLEAR_COLOR: clear_color(ex, a); break; + case OP_CLEAR_DS: clear_ds(ex, a); break; + case OP_VIEWPORTS: { + const p_viewports *v = a; + for (int i = 0; i < v->n && v->first + i < 16; i++) + if (v->first + i >= 0) memcpy(ex->vp[v->first + i], v->v[i], sizeof ex->vp[0]); + ex->has_vp = 1; + ex->dirty |= VK_EXEC_DIRTY_VIEWPORT; + break; + } + case OP_SCISSOR: + ex->scissor = *(const p_scissor *)a; + ex->has_scissor = 1; + ex->dirty |= VK_EXEC_DIRTY_SCISSOR; + break; + case OP_DEPTH_RANGE: { + const p_depth_range *r = a; + ex->depth_range[0] = r->n, ex->depth_range[1] = r->f; + ex->dirty |= VK_EXEC_DIRTY_VIEWPORT; + break; + } + case OP_STENCIL: { + const p_stencil *st = a; + for (int f = 0; f < 2; f++) { + if (!(st->face & (1 << f))) continue; + if (st->which == 0) ex->st_vmask[f] = (uint32_t)st->value_mask; + else if (st->which == 1) ex->st_wmask[f] = (uint32_t)st->mask; + else ex->st_ref[f] = (uint32_t)st->ref; + } + ex->dirty |= VK_EXEC_DIRTY_STENCIL; + break; + } + case OP_POLY_OFFSET: + ex->poly_offset = *(const p_poly_offset *)a; + ex->dirty |= VK_EXEC_DIRTY_DEPTH_BIAS; + break; + case OP_BLEND_COLOR: + memcpy(ex->blend_color, ((const p_blend_color *)a)->rgba, sizeof ex->blend_color); + ex->dirty |= VK_EXEC_DIRTY_BLEND_CONST; + break; + case OP_DEPTH_CLAMP: ex->pipe.depth_clamp = ((const p_flag *)a)->value != 0; break; + case OP_RASTER_DISCARD: ex->pipe.raster_discard = ((const p_flag *)a)->value != 0; break; + case OP_PRIM_RESTART: ex->prim_restart = *(const p_prim_restart *)a; break; + case OP_COPY_STRIDE: ex->copy_stride = ((const p_copy_stride *)a)->stride; break; + case OP_SET_TEX_POOL: ex->tex_pool = ((const p_set_pool *)a)->pool, ex->tex_serial++; break; + case OP_SET_SAMP_POOL: ex->samp_pool = ((const p_set_pool *)a)->pool; break; + case OP_BARRIER: barrier_to_why(ex, NULL, NULL, 0, BR_GAME); break; + case OP_DRAW: case OP_DRAW_IDX: case OP_DISPATCH: case OP_UPDATE_UB: case OP_DRAW_IDX_IND: case OP_DISPATCH_IND: + if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) vk_exec_frame.draws_seen++, split_draws++; + if (h->op == OP_DISPATCH || h->op == OP_DISPATCH_IND) vk_exec_frame.dispatches++, split_draws++; + // fall through + case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: + case OP_BIND_UBO: case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC: + case OP_DRAW_MESH: case OP_DRAW_MESH_IND: + vk_exec_op_draw(ex, h->op, a); + break; + case OP_COPY_B2B: case OP_COPY_B2T: case OP_COPY_T2B: case OP_COPY_T2T: case OP_CLEAR_TEX: + if (h->op != OP_CLEAR_TEX) vk_exec_frame.copies++; + if (vk_dump_armed) vk_dump_copy(ex, h->op, a); + // fall through + case OP_SIGNAL_EVENT: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: + case OP_CLEAR_BUF: + vk_exec_op_xfer(ex, h->op, a); + break; + default: rt_fatal("vk: op %s has no replay handler", nvn_op_name(h->op)); + } +} + +// ---- replay workers (NX2NAVI_REPLAY_WORKERS, default 0; stage 1 of recording a submission on several threads) +// A submission's draws come in runs: between two ops that have to stay in the replay's order on its own thread +// (dispatches, copies, texture and buffer clears, events, the game's barriers, calls and the end of a command buffer) +// there are only state, binds, draws and the ops a draw run carries (counter reports and resets, render conditions, +// inline uniform updates, render target changes and attachment clears). Hyrule Field, standing (NX2NAVI_REPLAY_RUNS, +// 2026-11-11): about 3,500 draws a frame in 82 runs, 96% of them in runs of 16 draws or more and 67% in runs of 256 +// or more, with a counter report or reset beside nearly every draw and a render condition beside every second one. +// With NX2NAVI_REPLAY_WORKERS set, a run of RUN_MIN draws or more is recorded by a worker thread: it takes a copy of +// the replay state, records the run into a command buffer of its own pool (vk_sync_worker_begin) with a draw context +// of its own (vk_exec_draw.c DrawCtx), ends it behind a full barrier, and hands the state the run left back. The +// replay's command buffer up to the run ended behind a full barrier too, and vk_sync_append puts the worker's command +// buffer after it in the same part, so the GPU runs everything in the order of the op stream. Stage 1 has one worker +// and the submitting thread waits for each run: the order of every side effect is the replay's own, and what it shows +// is whether a run recorded apart, from a copy of the state, gives the same pictures. Later stages record the runs +// of a submission on several workers at once. +#define RW_MAX 16 +#define RUN_MIN 16 // draws a run needs to go to a worker +#define RUN_PEEK 65536u // ops the submitting thread reads ahead to find a run's draws, at most + +// The ops a run may hold; JUMP is followed. +static int run_op(uint32_t op) { + switch (op) { + case OP_BEGIN: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_STENCIL: case OP_POLY_OFFSET: + case OP_BLEND_COLOR: case OP_DEPTH_CLAMP: case OP_RASTER_DISCARD: case OP_PRIM_RESTART: case OP_COPY_STRIDE: + case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: + case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO: case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: + case OP_BIND_IMG: case OP_PUSH_CC: case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: case OP_DRAW_MESH: + case OP_DRAW_MESH_IND: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: + case OP_UPDATE_UB: case OP_SET_RT: case OP_CLEAR_COLOR: case OP_CLEAR_DS: + return 1; + default: return 0; + } +} + +static int is_draw(uint32_t op) { + return op == OP_DRAW || op == OP_DRAW_IDX || op == OP_DRAW_IDX_IND || op == OP_DRAW_MESH || op == OP_DRAW_MESH_IND; +} + +typedef struct { + PlatThread *t; + int idx; + void *dc; // its draw context (vk_exec_draw_ctx_new) + volatile int32_t go, done; // a run is handed over / recorded + volatile int32_t sleeping; // waits for wake rather than spinning + PlatEvent wake; + VkExec ex; // the replay state it records the run with, and leaves + const op_hdr *h; // the run's first op; then the op that ended it + int depth; // the call depth of the run (for vk_exec_ahead) +} RwWorker; + +static struct { + int n; // workers; 0 off, -1 not read yet + RwWorker *w[RW_MAX]; + const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op and found too few draws + uint64_t runs, run_draws, short_runs, wait_ns; // since the last perf line +} RW = {.n = -1}; + +static void rw_walk(RwWorker *w); + +static void rw_main(void *arg) { + RwWorker *w = arg; + plat_thread_priority(PLAT_PRIO_HIGHEST); // as the recording thread (rec_claim) + vk_exec_draw_ctx_use(w->dc); + for (;;) { + // spins a while after a run (the next one is usually a few microseconds away), then sleeps + for (unsigned i = 0; !__atomic_load_n(&w->go, __ATOMIC_ACQUIRE); i++) { + if (i < 200000) { + plat_pause(); + continue; + } + __atomic_store_n(&w->sleeping, 1, __ATOMIC_SEQ_CST); + if (!__atomic_load_n(&w->go, __ATOMIC_SEQ_CST)) plat_event_wait_ns(&w->wake, 2000000); + __atomic_store_n(&w->sleeping, 0, __ATOMIC_RELAXED); + i = 0; + } + __atomic_store_n(&w->go, 0, __ATOMIC_RELAXED); + rw_walk(w); + __atomic_store_n(&w->done, 1, __ATOMIC_RELEASE); + } +} + +static void rw_init(void) { + int n = cfg_int("NX2NAVI_REPLAY_WORKERS"); + RW.n = n < 0 ? 0 : n > RW_MAX ? RW_MAX : n; + if (RW.n > 1) RW.n = 1; // stage 1: one worker + for (int k = 0; k < RW.n; k++) { + RwWorker *w = calloc(1, sizeof *w); + if (!w) rt_fatal("vk: out of memory (replay worker)"); + w->idx = k, w->dc = vk_exec_draw_ctx_new(); + plat_event_init(&w->wake, 0, 0); + char name[32]; + snprintf(name, sizeof name, "vk replay worker %d", k); + if (!(w->t = plat_thread_new(rw_main, w, name))) rt_fatal("vk: cannot start a replay worker"); + RW.w[k] = w; + } + if (RW.n) rt_log("vk: runs of %d draws or more are recorded by %d replay worker%s (NX2NAVI_REPLAY_WORKERS); the " + "submitting thread waits for each run", RUN_MIN, RW.n, RW.n > 1 ? "s" : ""); +} + +// The draws from h up to the op that ends its run (*end), reading at most RUN_PEEK ops. +static uint32_t run_peek(const op_hdr *h, const op_hdr **end) { + uint32_t draws = 0; + for (uint32_t ops = 0; ops < RUN_PEEK; ops++) { + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the walk reports it + if (h->op == OP_JUMP) { + h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target; + continue; + } + if (!run_op(h->op)) break; + draws += (uint32_t)is_draw(h->op); + h = nvn_op_next(h); + } + *end = h; + return draws; +} + +// The worker's side: records the run from w->h into a command buffer of its own; w->h becomes the op that ended it. +static void rw_walk(RwWorker *w) { + VkExec *ex = &w->ex; + x_cur = ex; + ex->cb = vk_sync_worker_begin(w->idx); + // the replay's command buffer before the run ended behind a full barrier, with every image in GENERAL + ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL; + ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0; + vk_exec_draw_new_cb(); + const op_hdr *h = w->h; + for (;;) { + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the walk reports it + if (h->op == OP_JUMP) { + h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target; + continue; + } + if (!run_op(h->op)) break; + AH.at[w->depth] = h, AH.depth = w->depth; // the submitting thread waits: where vk_exec_ahead reads on from + if (RR.on > 0) runs_op(h->op); + if (!op_linear(h->op)) vk_exec_brk++; + op_one(ex, h); + h = nvn_op_next(h); + } + vk_exec_barrier(ex); // the replay's next command buffer starts after the run, every image back in GENERAL + vk_sync_worker_end(ex->cb); + w->h = h; + x_cur = NULL; +} + +// The submitting thread's side, at op h of the walk: when a run of RUN_MIN draws or more starts here, a worker records +// it and the walk goes on with the op that ended it, which is returned; otherwise h. +static const op_hdr *run_start(VkExec *ex, const op_hdr *h, int depth) { + const op_hdr *end; + uint32_t draws = run_peek(h, &end); + if (draws < RUN_MIN || !vk_sync_append_room(1)) { + RW.skip_to[depth] = end, RW.short_runs += draws != 0; + return h; + } + uint64_t t0 = kern_now_ns(); + vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL + RwWorker *w = RW.w[0]; + memcpy(&w->ex, ex, sizeof *ex); + w->h = h, w->depth = depth; + __atomic_store_n(&w->done, 0, __ATOMIC_RELAXED); + __atomic_store_n(&w->go, 1, __ATOMIC_SEQ_CST); + if (__atomic_load_n(&w->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&w->wake); + while (!__atomic_load_n(&w->done, __ATOMIC_ACQUIRE)) plat_pause(); + // the state the run left goes on in a new command buffer of the replay's, after the worker's in the part + int recorded = ex->recorded + w->ex.recorded; + VkCommandBuffer wcb = w->ex.cb; + memcpy(ex, &w->ex, sizeof *ex); + ex->cb = vk_sync_append(&wcb, 1); + ex->recorded = ex->barrier_at = recorded; // the worker's command buffer ended with a full barrier + ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL, ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0; + vk_exec_draw_new_cb(); + RW.runs++, RW.run_draws += draws, RW.wait_ns += kern_now_ns() - t0; + return w->h; +} + static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { + RW.skip_to[depth] = NULL; for (;;) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); + if (RW.skip_to[depth] == h) RW.skip_to[depth] = NULL; // the short run read ahead is behind + if (RW.n > 0 && !RW.skip_to[depth] && !evo_in && h->op != OP_JUMP && run_op(h->op)) { + const op_hdr *stop = run_start(ex, h, depth); + if (stop != h) { + if (split_draws && split_wanted()) maybe_split(ex); + h = stop; + continue; + } + } const void *a = nvn_op_payload(h); AH.at[depth] = h, AH.depth = depth; // where vk_exec_ahead reads on from + if (RR.on > 0) runs_op(h->op); if (!op_linear(h->op)) vk_exec_brk++; switch (h->op) { case OP_END: return NULL; - case OP_BEGIN: break; case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; case OP_CALL: if (depth >= CALL_DEPTH) rt_fatal("vk: command buffer calls nested deeper than %d", CALL_DEPTH); walk(ex, nvn_op_first(((const p_jump *)a)->target), depth + 1); break; - case OP_SET_RT: set_rt(ex, a); break; - case OP_CLEAR_COLOR: clear_color(ex, a); break; - case OP_CLEAR_DS: clear_ds(ex, a); break; - case OP_VIEWPORTS: { - const p_viewports *v = a; - for (int i = 0; i < v->n && v->first + i < 16; i++) - if (v->first + i >= 0) memcpy(ex->vp[v->first + i], v->v[i], sizeof ex->vp[0]); - ex->has_vp = 1; - ex->dirty |= VK_EXEC_DIRTY_VIEWPORT; - break; - } - case OP_SCISSOR: - ex->scissor = *(const p_scissor *)a; - ex->has_scissor = 1; - ex->dirty |= VK_EXEC_DIRTY_SCISSOR; - break; - case OP_DEPTH_RANGE: { - const p_depth_range *r = a; - ex->depth_range[0] = r->n, ex->depth_range[1] = r->f; - ex->dirty |= VK_EXEC_DIRTY_VIEWPORT; - break; - } - case OP_STENCIL: { - const p_stencil *s = a; - for (int f = 0; f < 2; f++) { - if (!(s->face & (1 << f))) continue; - if (s->which == 0) ex->st_vmask[f] = (uint32_t)s->value_mask; - else if (s->which == 1) ex->st_wmask[f] = (uint32_t)s->mask; - else ex->st_ref[f] = (uint32_t)s->ref; - } - ex->dirty |= VK_EXEC_DIRTY_STENCIL; - break; - } - case OP_POLY_OFFSET: - ex->poly_offset = *(const p_poly_offset *)a; - ex->dirty |= VK_EXEC_DIRTY_DEPTH_BIAS; - break; - case OP_BLEND_COLOR: - memcpy(ex->blend_color, ((const p_blend_color *)a)->rgba, sizeof ex->blend_color); - ex->dirty |= VK_EXEC_DIRTY_BLEND_CONST; - break; - case OP_DEPTH_CLAMP: ex->pipe.depth_clamp = ((const p_flag *)a)->value != 0; break; - case OP_RASTER_DISCARD: ex->pipe.raster_discard = ((const p_flag *)a)->value != 0; break; - case OP_PRIM_RESTART: ex->prim_restart = *(const p_prim_restart *)a; break; - case OP_COPY_STRIDE: ex->copy_stride = ((const p_copy_stride *)a)->stride; break; - case OP_SET_TEX_POOL: ex->tex_pool = ((const p_set_pool *)a)->pool, ex->tex_serial++; break; - case OP_SET_SAMP_POOL: ex->samp_pool = ((const p_set_pool *)a)->pool; break; - case OP_BARRIER: barrier_to_why(ex, NULL, NULL, 0, BR_GAME); break; - case OP_DRAW: case OP_DRAW_IDX: case OP_DISPATCH: case OP_UPDATE_UB: case OP_DRAW_IDX_IND: case OP_DISPATCH_IND: - if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) vk_exec_frame.draws_seen++, split_draws++; - if (h->op == OP_DISPATCH || h->op == OP_DISPATCH_IND) vk_exec_frame.dispatches++, split_draws++; - case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: - case OP_BIND_UBO: case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC: - case OP_DRAW_MESH: case OP_DRAW_MESH_IND: - vk_exec_op_draw(ex, h->op, a); - break; - case OP_COPY_B2B: case OP_COPY_B2T: case OP_COPY_T2B: case OP_COPY_T2T: case OP_CLEAR_TEX: - if (h->op != OP_CLEAR_TEX) vk_exec_frame.copies++; - if (vk_dump_armed) vk_dump_copy(ex, h->op, a); - // fall through - case OP_SIGNAL_EVENT: case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: - case OP_CLEAR_BUF: - vk_exec_op_xfer(ex, h->op, a); - break; case OP_WAIT_EVENT: { int met = vk_exec_wait_event(a); vk_exec_evt(met ? 'm' : 'u', (const void *)((const p_wait_event *)a)->val, 0); @@ -1625,7 +1894,7 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { } break; } - default: rt_fatal("vk: op %s has no replay handler", nvn_op_name(h->op)); + default: op_one(ex, h); break; } if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); if (split_draws && split_wanted()) maybe_split(ex); @@ -1684,6 +1953,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t perf_rec_waits = rec_wait_totals; VkExec *ex = &X; if (!x_ready) state_reset(ex), vk_dump_init(), x_ready = 1; + if (RW.n < 0) rw_init(); const tex_t *keep[9]; // the render targets carry over to this submission (set_rt compares them by pointer) memcpy(keep, ex->rt.color, sizeof ex->rt.color); keep[8] = ex->rt.depth; @@ -1748,6 +2018,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t evo_queue = 0; } vk_exec_pass_end(ex); + if (RR.on > 0) runs_break(RB_SUBMIT); x_cur = NULL; vk_exec_event_sync(0); if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes @@ -1791,6 +2062,12 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t char cbs[1600]; vk_sync_cb_stats(cbs, sizeof cbs); rt_log("perf: completion thread since the last line, by callback (distance from vk_completion_add):%s", cbs); + if (RW.n > 0) + rt_log("perf: replay workers: %.1f runs per submission of %.0f draws each, %.2f ms each with the hand-over; " + "%.1f shorter runs per submission stayed on the submitting thread", (double)RW.runs / (double)n_subs, + RW.runs ? (double)RW.run_draws / (double)RW.runs : 0.0, + RW.runs ? (double)RW.wait_ns / 1e6 / (double)RW.runs : 0.0, (double)RW.short_runs / (double)n_subs); + RW.runs = RW.run_draws = RW.short_runs = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; n_subs = n_parts = n_idle_parts = n_pass_splits = 0; } diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 5296b355..4a707c70 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -148,6 +148,10 @@ void vk_exec_inline_end(int reopen); // recording a replay into cb for submission `fence`, so vk_exec_inline_begin hands cb out. Called from the record // callback of vk_sync_submit_own, where a replay would be recorded. void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence); +// vk_exec_draw.c, the replay workers: a draw context of their own (what draws keep from one to the next while they +// record: vk_exec_draw.c DrawCtx), and the one the calling thread records with from now on (NULL: the replay's). +void *vk_exec_draw_ctx_new(void); +void vk_exec_draw_ctx_use(void *ctx); // vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex // buffers): the replay continues in a new one (vk_sync_split). void vk_exec_draw_new_cb(void); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index e3830681..56dc8c4c 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -282,6 +282,15 @@ typedef struct DrawCtx { static DrawCtx dc_main = {.bind_gen = 1}; static _Thread_local DrawCtx *dc = &dc_main; +void *vk_exec_draw_ctx_new(void) { + DrawCtx *c = calloc(1, sizeof *c); + if (!c) rt_fatal("vk: out of memory (draw context)"); + c->bind_gen = 1; + return c; +} + +void vk_exec_draw_ctx_use(void *c) { dc = c ? c : &dc_main; } + static void cb_done(void *arg) { // completion thread CbRes *r = arg; VkCtx *c = vk_ctx(); @@ -309,8 +318,9 @@ static void cb_done(void *arg) { // completion thread void vk_exec_draw_new_cb(void) { memset(&dc->B, 0, sizeof dc->B); } void vk_exec_draw_shaders(const GfxShader **vs, const GfxShader **fs) { - *vs = dc->B.st_valid ? dc->B.st.vs : NULL; - *fs = dc->B.st_valid ? dc->B.st.fs : NULL; + DrawCtx *const cx = dc; + *vs = cx->B.st_valid ? cx->B.st.vs : NULL; + *fs = cx->B.st_valid ? cx->B.st.fs : NULL; } // Marks a dispatch about to be recorded (vk.h vk_pass_mark_begin); the caller ends it after the dispatch. @@ -1107,19 +1117,20 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) { // vk_mem_count, also kept per (class, use) while a graphics build runs (RS). static void count_use(int cls, int use, uint64_t n) { + DrawCtx *const cx = dc; vk_mem_count(cls, use, n); - if (dc->pb_cap) { - if (dc->pb_cap->has) dc->pb_spoiled = 1; - else *dc->pb_cap = (PbCount){cls, use, 1, n}; + if (cx->pb_cap) { + if (cx->pb_cap->has) cx->pb_spoiled = 1; + else *cx->pb_cap = (PbCount){cls, use, 1, n}; } - if (!dc->RS.rec) return; - for (int i = 0; i < dc->RS.ncnt; i++) - if (dc->RS.cnt[i].cls == cls && dc->RS.cnt[i].use == use) { - dc->RS.cnt[i].bytes += n; + if (!cx->RS.rec) return; + for (int i = 0; i < cx->RS.ncnt; i++) + if (cx->RS.cnt[i].cls == cls && cx->RS.cnt[i].use == use) { + cx->RS.cnt[i].bytes += n; return; } - if (dc->RS.ncnt < (int)(sizeof dc->RS.cnt / sizeof dc->RS.cnt[0])) dc->RS.cnt[dc->RS.ncnt++] = (RsCount){cls, use, n}; - else dc->RS.cnt_over = 1; + if (cx->RS.ncnt < (int)(sizeof cx->RS.cnt / sizeof cx->RS.cnt[0])) cx->RS.cnt[cx->RS.ncnt++] = (RsCount){cls, use, n}; + else cx->RS.cnt_over = 1; } // A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when @@ -1206,30 +1217,31 @@ static void nvn2_cs_tables(const uint8_t *ctl, uint32_t *ubos, uint32_t *ssbos); static uint32_t nvn2_cs_tables_at(const uint8_t *ctl); static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { + DrawCtx *const cx = dc; int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : 2; // vk_c0_fill reads the storage slots, and the uniform slots and dispatch size for compute or the base instance for // graphics: equal inputs give the same bank, so neither the fill nor the 2 KiB compare is needed. // A vertex or fragment bank depends on the storage slots and the base instance alone (the rest of its inputs record // is zero): those two are compared without building and comparing the whole 1.8 KiB record on every draw. - if (k < 2 && dc->B.c0_valid[k] && dc->B.c0_in[k].binst == dc->c0_binst && - !memcmp(dc->B.c0_in[k].ssbo, ex->ssbo[st], sizeof dc->B.c0_in[k].ssbo)) - return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; - typeof(dc->B.c0_in[0]) in; + if (k < 2 && cx->B.c0_valid[k] && cx->B.c0_in[k].binst == cx->c0_binst && + !memcmp(cx->B.c0_in[k].ssbo, ex->ssbo[st], sizeof cx->B.c0_in[k].ssbo)) + return (VkDescriptorBufferInfo){cx->B.c0_buf[k], cx->B.c0_off[k], VK_C0_BYTES}; + typeof(cx->B.c0_in[0]) in; memset(&in, 0, sizeof in); memcpy(in.ssbo, ex->ssbo[st], sizeof in.ssbo); - if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, dc->c0_dim, sizeof in.dim); - else in.binst = dc->c0_binst; + if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, cx->c0_dim, sizeof in.dim); + else in.binst = cx->c0_binst; // NVN2 compute: the bank holds the push constants from byte 0x10 (then the program's handle table, which the // host does not read: textures and images are bound by slot). const uint8_t *pctl, *pcode; uint32_t plen; int nvn2_push = st == NVN_STAGE_CS && ex->prog && find_stage(ex->prog, st, &pctl, &pcode, &plen) && ctl_nvn2(pctl); if (nvn2_push) memcpy(in.push, ex->cs_push, sizeof in.push); - if (dc->B.c0_valid[k] && !memcmp(&in, &dc->B.c0_in[k], sizeof in)) - return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; - dc->B.c0_in[k] = in; + if (cx->B.c0_valid[k] && !memcmp(&in, &cx->B.c0_in[k], sizeof in)) + return (VkDescriptorBufferInfo){cx->B.c0_buf[k], cx->B.c0_off[k], VK_C0_BYTES}; + cx->B.c0_in[k] = in; uint8_t tmp[VK_C0_BYTES]; - vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], dc->c0_dim, dc->c0_binst); + vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], cx->c0_dim, cx->c0_binst); if (nvn2_push) { // the pushed constants from 0x10, the driver's tables after them (push_ptr, nvn2_cs_tables_at) uint32_t nu, ns, ta = nvn2_cs_tables_at(pctl); @@ -1239,7 +1251,7 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { // Bytes 0-0xb of an NVN2 compute bank are the dispatch's group counts (x, y, z). The fog pass after the // integration (passes 0115/0116 of a Kokiri frame) maps each thread to tid / (groups * 16 - 1): with zeros // there every thread read the same point. An indirect dispatch leaves them 0 (op_dispatch_indirect). - memcpy(tmp, dc->c0_dim, 12); + memcpy(tmp, cx->c0_dim, 12); for (uint32_t k = 0; k < nu + ns && ta + 16 * k + 12 <= VK_C0_BYTES; k++) { const p_bind_buf *sb = k < nu ? (k < 18 ? &ex->ubo[st][k] : NULL) : (k - nu < 16 ? &ex->ssbo[st][k - nu] : NULL); if (!sb) continue; @@ -1247,12 +1259,12 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { memcpy(tmp + ta + 16 * k, &sb->addr, 8), memcpy(tmp + ta + 16 * k + 8, &sz, 4); } } - if (!dc->B.c0_valid[k] || memcmp(tmp, dc->B.c0[k], VK_C0_BYTES)) { - memcpy(scratch(ex, VK_C0_BYTES, &dc->B.c0_buf[k], &dc->B.c0_off[k]), tmp, VK_C0_BYTES); - memcpy(dc->B.c0[k], tmp, VK_C0_BYTES); - dc->B.c0_valid[k] = 1; + if (!cx->B.c0_valid[k] || memcmp(tmp, cx->B.c0[k], VK_C0_BYTES)) { + memcpy(scratch(ex, VK_C0_BYTES, &cx->B.c0_buf[k], &cx->B.c0_off[k]), tmp, VK_C0_BYTES); + memcpy(cx->B.c0[k], tmp, VK_C0_BYTES); + cx->B.c0_valid[k] = 1; } - return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; + return (VkDescriptorBufferInfo){cx->B.c0_buf[k], cx->B.c0_off[k], VK_C0_BYTES}; } // The buffer range limits, read once: the device's properties are a cold cache line on every binding of every draw. @@ -1270,9 +1282,10 @@ static void limits_init(void) { // (DrawCtx.mo_ovr). static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { - if (dc->mo_ovr_on && (st == NVN_STAGE_VS || st == NVN_STAGE_FS) && slot >= 0 && slot < 32 && - dc->mo_ovr[st][slot].buffer) - return dc->mo_ovr[st][slot]; + DrawCtx *const cx = dc; + if (cx->mo_ovr_on && (st == NVN_STAGE_VS || st == NVN_STAGE_FS) && slot >= 0 && slot < 32 && + cx->mo_ovr[st][slot].buffer) + return cx->mo_ovr[st][slot]; if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); uint64_t max = LIM.ubo_max, align = LIM.ubo_align; VkDescriptorBufferInfo bi; @@ -3052,7 +3065,8 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si } static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { - if (info == dc->rs_last) return dc->rs_last_l; + DrawCtx *const cx = dc; + if (info == cx->rs_last) return cx->rs_last_l; if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info); RsList *l = nvn_hm_get(&rs_lists, &info); if (!l) { @@ -3102,7 +3116,7 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { memcpy(l->r, tmp, sizeof *tmp * n); nvn_hm_put(&rs_lists, &info, l); } - dc->rs_last = info, dc->rs_last_l = l; + cx->rs_last = info, cx->rs_last_l = l; return l; } @@ -3110,9 +3124,10 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { // are compared only when a bind op ran since they were stored or the pipeline changed: otherwise they are the same. // Check mode compares them anyway and stops when the bind op generation said they could not differ but they do. static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { + DrawCtx *const cx = dc; const RsList *l = rs_list(ex, info); - if (!l->ok) return dc->RS.hn = 0, dc->RS.info = NULL, 0; - uint64_t *h = dc->RS.h, diff = 0; + if (!l->ok) return cx->RS.hn = 0, cx->RS.info = NULL, 0; + uint64_t *h = cx->RS.h, diff = 0; uint32_t n = 0; #define PUT(v) do { uint64_t v_ = (uint64_t)(v); diff |= h[n] ^ v_; h[n++] = v_; } while (0) PUT((uintptr_t)info); @@ -3131,16 +3146,16 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { PUT(a); PUT(ps); } - if (l->c0) PUT(dc->c0_binst); + if (l->c0) PUT(cx->c0_binst); for (int i = 0; i < l->np; i++) PUT(l->p[i].res->ptr ? ptr_addr(ex, l->p[i].st, l->p[i].res) : ubo_ptr(ex, l->p[i].st, l->p[i].res)); #undef PUT - diff |= n != dc->RS.hn; - dc->RS.hn = n; - int fast = dc->RS.info == info && dc->RS.gen == dc->bind_gen; + diff |= n != cx->RS.hn; + cx->RS.hn = n; + int fast = cx->RS.info == info && cx->RS.gen == cx->bind_gen; if (fast && rs_on != 2) return !diff; const uint8_t *e = (const uint8_t *)ex; - uint8_t *d = dc->RS.body; + uint8_t *d = cx->RS.body; int bd = 0; for (uint32_t i = 0; i < l->n; i++) { const RsRange *r = &l->r[i]; @@ -3149,7 +3164,7 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { d += r->len; } if (fast && bd) rt_fatal("vk_exec: descriptor build skip check: bound slots changed without a bind op"); - dc->RS.info = info, dc->RS.gen = dc->bind_gen; + cx->RS.info = info, cx->RS.gen = cx->bind_gen; return !diff && !bd; } @@ -3244,39 +3259,41 @@ static int pb_key(const VkExec *ex, const VkPipeBinding *b, uint32_t k, uint64_t // Before descriptor j (descriptor k of binding b) resolves: 1 when its kept result serves, *o filled and its census // counted again. 0: resolve it, then pb_keep. static int pb_take(const VkExec *ex, const VkPipeBinding *b, uint32_t j, uint32_t k, VkDescSlot *o) { - dc->pb_key_ok = 0; - if (!dc->pb_e || j >= PB_DESC || !pb_key(ex, b, k, dc->pb_key_cur)) return 0; - dc->pb_key_ok = 1; - if (pb_mode != 2 && dc->pb_e->ok[j] && !memcmp(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur)) { - const PbCount *c = &dc->pb_e->cnt[j]; - *o = dc->pb_e->d[j]; + DrawCtx *const cx = dc; + cx->pb_key_ok = 0; + if (!cx->pb_e || j >= PB_DESC || !pb_key(ex, b, k, cx->pb_key_cur)) return 0; + cx->pb_key_ok = 1; + if (pb_mode != 2 && cx->pb_e->ok[j] && !memcmp(cx->pb_e->in[j], cx->pb_key_cur, sizeof cx->pb_key_cur)) { + const PbCount *c = &cx->pb_e->cnt[j]; + *o = cx->pb_e->d[j]; if (c->has) count_use(c->cls, c->use, c->n); vk_exec_frame.bind_reused++; return 1; } - dc->pb_cnt_cur = (PbCount){0}; - dc->pb_cap = &dc->pb_cnt_cur, dc->pb_spoiled = 0; + cx->pb_cnt_cur = (PbCount){0}; + cx->pb_cap = &cx->pb_cnt_cur, cx->pb_spoiled = 0; return 0; } // After descriptor j resolved to *o. static void pb_keep(uint32_t j, const VkDescSlot *o) { - dc->pb_cap = NULL; - if (!dc->pb_e || j >= PB_DESC) return; + DrawCtx *const cx = dc; + cx->pb_cap = NULL; + if (!cx->pb_e || j >= PB_DESC) return; vk_exec_frame.bind_resolved++; - if (!dc->pb_key_ok) { - dc->pb_e->ok[j] = 0; + if (!cx->pb_key_ok) { + cx->pb_e->ok[j] = 0; return; } - if (pb_mode == 2 && dc->pb_e->ok[j] && !dc->pb_spoiled && - !memcmp(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur) && - memcmp(&dc->pb_e->d[j], o, sizeof *o)) + if (pb_mode == 2 && cx->pb_e->ok[j] && !cx->pb_spoiled && + !memcmp(cx->pb_e->in[j], cx->pb_key_cur, sizeof cx->pb_key_cur) && + memcmp(&cx->pb_e->d[j], o, sizeof *o)) rt_fatal("vk_exec: per-binding descriptor reuse check: descriptor %u resolves differently from its kept result " "with the same inputs", j); - dc->pb_e->ok[j] = (uint8_t)!dc->pb_spoiled; - memcpy(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur); - dc->pb_e->d[j] = *o; - dc->pb_e->cnt[j] = dc->pb_cnt_cur; + cx->pb_e->ok[j] = (uint8_t)!cx->pb_spoiled; + memcpy(cx->pb_e->in[j], cx->pb_key_cur, sizeof cx->pb_key_cur); + cx->pb_e->d[j] = *o; + cx->pb_e->cnt[j] = cx->pb_cnt_cur; } // Writes allocated set s from its packed descriptors d: through the layout's update template, or with the write @@ -3299,32 +3316,33 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c // The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update // template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. static void bind_resources(VkExec *ex, const VkPipeInfo *info) { - VkWriteDescriptorSet *w = dc->bind_w; - VkDescSlot *d = dc->bind_d; - VkBufferView *bv = dc->bind_bv; + DrawCtx *const cx = dc; + VkWriteDescriptorSet *w = cx->bind_w; + VkDescSlot *d = cx->bind_d; + VkBufferView *bv = cx->bind_bv; if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS); bind_stats(info); int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS; - int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = dc->B.res_valid; + int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = cx->B.res_valid; if (skip) { uint64_t sig_t0 = bt_now(); same = bind_sig(ex, info); vk_exec_frame.bind_sig_ns += (uint64_t)(bt_now() - sig_t0); if (same && was_valid && skip == 1) { - for (int i = 0; i < dc->RS.ncnt; i++) vk_mem_count(dc->RS.cnt[i].cls, dc->RS.cnt[i].use, dc->RS.cnt[i].bytes); + for (int i = 0; i < cx->RS.ncnt; i++) vk_mem_count(cx->RS.cnt[i].cls, cx->RS.cnt[i].use, cx->RS.cnt[i].bytes); BS.bind_skip++; ex->recorded++; return; } - dc->B.res_valid = 0; // until the build below ends - if (skip == 2) memcpy(dc->RS.pcnt, dc->RS.cnt, sizeof dc->RS.cnt), dc->RS.pncnt = dc->RS.ncnt; - dc->RS.rec = 1, dc->RS.ncnt = 0, dc->RS.cnt_over = 0, dc->RS.volatile_ = 0, dc->RS.cmds = 0; + cx->B.res_valid = 0; // until the build below ends + if (skip == 2) memcpy(cx->RS.pcnt, cx->RS.cnt, sizeof cx->RS.cnt), cx->RS.pncnt = cx->RS.ncnt; + cx->RS.rec = 1, cx->RS.ncnt = 0, cx->RS.cnt_over = 0, cx->RS.volatile_ = 0, cx->RS.cmds = 0; } - dc->pb_e = NULL; - if (gfx && !dc->mo_ovr_on && pb_on()) { + cx->pb_e = NULL; + if (gfx && !cx->mo_ovr_on && pb_on()) { uint64_t g[PB_G]; pb_globals(ex, g); - dc->pb_e = pb_entry(info, g); + cx->pb_e = pb_entry(info, g); } uint32_t n_sets = info->n_sets ? info->n_sets : 1; uint32_t lo[2] = {0, 0}, hi[2] = {0, 0}, d0[2] = {0, 0}; // bindings [lo, hi) and first descriptor per set @@ -3347,20 +3365,20 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { if (b->kind == VK_PIPE_SUPPORT) { float in[3]; support_in(ex, in); - if (!dc->B.sup_valid || memcmp(in, dc->B.sup_in, sizeof in)) { // support_fill reads only these + if (!cx->B.sup_valid || memcmp(in, cx->B.sup_in, sizeof in)) { // support_fill reads only these uint8_t tmp[SUPPORT_BYTES]; support_fill(tmp, in[0], in[1], in[2]); - memcpy(dc->B.sup_in, in, sizeof in); + memcpy(cx->B.sup_in, in, sizeof in); // a new buffer only when the contents changed - if (!dc->B.sup_valid || memcmp(tmp, dc->B.support, SUPPORT_BYTES)) { + if (!cx->B.sup_valid || memcmp(tmp, cx->B.support, SUPPORT_BYTES)) { VkBuffer sb; VkDeviceSize so; memcpy(scratch(ex, SUPPORT_BYTES, &sb, &so), tmp, SUPPORT_BYTES); - memcpy(dc->B.support, tmp, SUPPORT_BYTES); - dc->B.sup_buf = sb, dc->B.sup_off = so, dc->B.sup_valid = 1; + memcpy(cx->B.support, tmp, SUPPORT_BYTES); + cx->B.sup_buf = sb, cx->B.sup_off = so, cx->B.sup_valid = 1; } } - o->buffer = (VkDescriptorBufferInfo){dc->B.sup_buf, dc->B.sup_off, SUPPORT_BYTES}; + o->buffer = (VkDescriptorBufferInfo){cx->B.sup_buf, cx->B.sup_off, SUPPORT_BYTES}; } else if (b->kind == GFX_RES_UBO) { uint64_t bt = bt_now(); for (uint32_t k = 0; k < cnt; k++) { @@ -3416,7 +3434,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } nd += cnt; } - dc->pb_e = NULL; + cx->pb_e = NULL; uint64_t push_t0 = bt_now(); // An empty set 1 (a fragment stage without resources: depth and shadow draws) starts past the last binding, which // the loop never reaches: d0[1] stayed 0 and set 0 was pushed with 0 bytes, so the driver read stale descriptors @@ -3427,10 +3445,10 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { // layout lacks (false VUID-vkCmdDrawIndexed-None-08114, then an access violation inside the layer). A bind at set 1 // disturbs set 0 by its canonical id, which drops the layer's push set: the push after it gets a new one. int set1_first = 0; - if (gfx && info->layout != dc->B.ds_layout) { // another layout: sets bound before may be disturbed - dc->B.ds_layout = info->layout; - dc->B.ds_bound[0] = dc->B.ds_bound[1] = VK_NULL_HANDLE; - dc->B.ds_pushed[0] = dc->B.ds_pushed[1] = 0; + if (gfx && info->layout != cx->B.ds_layout) { // another layout: sets bound before may be disturbed + cx->B.ds_layout = info->layout; + cx->B.ds_bound[0] = cx->B.ds_bound[1] = VK_NULL_HANDLE; + cx->B.ds_pushed[0] = cx->B.ds_pushed[1] = 0; set1_first = n_sets == 2 && vk_ctx()->validation; } for (uint32_t k = 0; k < n_sets; k++) { @@ -3441,24 +3459,24 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { if (set1_first) { // bound only to drop the layer's push set (comment above) VkDescriptorSet set = alloc_set(ex, info->set_layout1); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 1, 1, &set, 0, NULL); - dc->B.ds_bound[1] = set, dc->B.ds_pushed[1] = 0; - dc->RS.cmds++; + cx->B.ds_bound[1] = set, cx->B.ds_pushed[1] = 0; + cx->RS.cmds++; } continue; } if (s ? info->push1 : info->push) { uint32_t n = gfx && push_skip_on() ? dbytes : 0; - if (n && dc->B.ds_pushed[s] && dc->DS.pn[s] == n && !memcmp(dc->DS.psig[s], ds, n)) { + if (n && cx->B.ds_pushed[s] && cx->DS.pn[s] == n && !memcmp(cx->DS.psig[s], ds, n)) { BS.push_skip++; continue; } if (info->tpl[s]) rec_push_tpl(vk_pipeline_push_tpl_fn(), ex->cb, info->tpl[s], info->layout, s, ds, dbytes); else rec_push(vk_pipeline_push_fn(), ex->cb, info->bind_point, info->layout, s, i1 - i0, w + i0); BS.push++; - dc->RS.cmds++; + cx->RS.cmds++; if (gfx) { - dc->B.ds_bound[s] = VK_NULL_HANDLE, dc->B.ds_pushed[s] = n != 0; - if (n) dc->DS.pn[s] = n, memcpy(dc->DS.psig[s], ds, n); + cx->B.ds_bound[s] = VK_NULL_HANDLE, cx->B.ds_pushed[s] = n != 0; + if (n) cx->DS.pn[s] = n, memcpy(cx->DS.psig[s], ds, n); } } else if (!gfx || s > 1) { VkDescriptorSet set = alloc_set(ex, s ? info->set_layout1 : info->set_layout); @@ -3466,50 +3484,50 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL); } else { VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout; - VkDescriptorSet set = dc->B.ds_last[s]; - if (!set || dc->B.ds_last_layout[s] != sl || dc->DS.n[s] != dbytes || memcmp(dc->DS.sig[s], ds, dbytes)) { + VkDescriptorSet set = cx->B.ds_last[s]; + if (!set || cx->B.ds_last_layout[s] != sl || cx->DS.n[s] != dbytes || memcmp(cx->DS.sig[s], ds, dbytes)) { set = alloc_set(ex, sl); write_set(info, s, set, ds, w, i0, i1); - dc->B.ds_last[s] = set, dc->B.ds_last_layout[s] = sl, dc->DS.n[s] = dbytes; - memcpy(dc->DS.sig[s], ds, dbytes); + cx->B.ds_last[s] = set, cx->B.ds_last_layout[s] = sl, cx->DS.n[s] = dbytes; + memcpy(cx->DS.sig[s], ds, dbytes); BS.set_new++; - dc->RS.cmds++; + cx->RS.cmds++; } else { BS.set_reuse++; } - if (dc->B.ds_bound[s] != set) { + if (cx->B.ds_bound[s] != set) { vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL); - dc->B.ds_bound[s] = set, dc->B.ds_pushed[s] = 0; - dc->RS.cmds++; + cx->B.ds_bound[s] = set, cx->B.ds_pushed[s] = 0; + cx->RS.cmds++; } } } if (info->pool) { // set 2: the texture and sampler pools pool_sync(ex); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 2, 1, &PL.set, 0, NULL); - dc->RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again + cx->RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again } if (skip) { - dc->RS.rec = 0; + cx->RS.rec = 0; // An equal next draw records nothing when every push set holds its push (NX2NAVI_PUSH_SKIP on) and nothing made // the build unrepeatable; allocated sets are always bound to the one written last. - int valid = !dc->RS.volatile_ && !dc->RS.cnt_over && dc->RS.hn; + int valid = !cx->RS.volatile_ && !cx->RS.cnt_over && cx->RS.hn; for (uint32_t s = 0; s < n_sets; s++) - if (hi[s] > lo[s] && (s ? info->push1 : info->push) && !dc->B.ds_pushed[s]) valid = 0; + if (hi[s] > lo[s] && (s ? info->push1 : info->push) && !cx->B.ds_pushed[s]) valid = 0; if (skip == 2) { // with the pipeline info equal (bind_sig), the packed descriptors determine every write uint32_t n = (uint32_t)(nd * sizeof *d); - int wr = n != dc->RS.ckn || memcmp(dc->RS.ck, d, n); - int cn = dc->RS.ncnt != dc->RS.pncnt || - memcmp(dc->RS.cnt, dc->RS.pcnt, sizeof dc->RS.cnt[0] * (size_t)dc->RS.ncnt); - if (same && was_valid && (wr || cn || dc->RS.cmds)) + int wr = n != cx->RS.ckn || memcmp(cx->RS.ck, d, n); + int cn = cx->RS.ncnt != cx->RS.pncnt || + memcmp(cx->RS.cnt, cx->RS.pcnt, sizeof cx->RS.cnt[0] * (size_t)cx->RS.ncnt); + if (same && was_valid && (wr || cn || cx->RS.cmds)) rt_fatal("vk_exec: descriptor build skip check: a draw of program %llu with the inputs of the previous one " "gets %s", (unsigned long long)ex->prog->serial, - dc->RS.cmds ? "descriptor commands" : wr ? "other writes" : "other census counts"); + cx->RS.cmds ? "descriptor commands" : wr ? "other writes" : "other census counts"); if (same && was_valid) BS.bind_skip++; // the draws a skip would have spared, compared instead - dc->RS.ckn = n; - memcpy(dc->RS.ck, d, n); + cx->RS.ckn = n; + memcpy(cx->RS.ck, d, n); } - dc->B.res_valid = valid; + cx->B.res_valid = valid; } vk_exec_frame.bind_push_ns += (uint64_t)(bt_now() - push_t0); ex->recorded++; @@ -3518,12 +3536,13 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { // ---- draws static void bind_pipeline(VkExec *ex, VkPipelineBindPoint bp, VkPipeline p) { + DrawCtx *const cx = dc; int k = bp == VK_PIPELINE_BIND_POINT_COMPUTE; - if (dc->B.pipe[k] == p) return; + if (cx->B.pipe[k] == p) return; vkCmdBindPipeline(ex->cb, bp, p); ex->recorded++; - dc->B.pipe[k] = p; - if (!k) dc->B.st_valid = 0; + cx->B.pipe[k] = p; + if (!k) cx->B.st_valid = 0; } static VkViewport draw_viewport(VkExec *ex) { @@ -3967,6 +3986,7 @@ static void prog_lead(const prog_t *p, int mesh) { // the mesh fallback (mfb_draw): the vertex stage is DrawCtx.mfb_cur_vs, in an ordinary pipeline without vertex input. static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) { + DrawCtx *const cx = dc; int fb = mesh == 2; // from here on mesh says what the draw is, and !fb that its pipeline is a mesh pipeline mesh = mesh != 0; if (ex->render_disabled) return vk_cond_draws_skip += ex->has_render_cond, draw_skipped(ex, mesh, SK_DISABLED); @@ -3989,8 +4009,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int int has_ts = 0, ts_wait = 0, vst = mesh ? NVN_STAGE_MESH : NVN_STAGE_VS; const GfxShader *vs, *fs, *ts = NULL; uint32_t gen = memo ? vk_texture_slots_gen() : 0; // before shader_for, whose lookups may bump it - if (memo && dc->SM.valid && memo_match(ex, gen, &dc->SM.k)) { - vs = dc->SM.vs, fs = dc->SM.fs, has_vs = 1, has_fs = dc->SM.has_fs; + if (memo && cx->SM.valid && memo_match(ex, gen, &cx->SM.k)) { + vs = cx->SM.vs, fs = cx->SM.fs, has_vs = 1, has_fs = cx->SM.has_fs; if (memo == 2) { int hv, hf, wv, wf; const GfxShader *v2 = shader_for(ex, NVN_STAGE_VS, &hv, &wv, 0), *f2 = shader_for(ex, NVN_STAGE_FS, &hf, &wf, 0); @@ -3998,13 +4018,13 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int rt_fatal("vk_exec: shader memo of program %llu differs from the lookup", (unsigned long long)ex->prog->serial); } } else { - dc->SM.valid = 0; - if (fb) vs = dc->mfb_cur_vs, has_vs = 1; + cx->SM.valid = 0; + if (fb) vs = cx->mfb_cur_vs, has_vs = 1; else vs = shader_for(ex, vst, &has_vs, &vs_wait, 0); if (mesh && !fb) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0); fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, 0); if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs)) - memo_key(ex, gen, &dc->SM.k), dc->SM.vs = vs, dc->SM.fs = fs, dc->SM.has_fs = has_fs, dc->SM.valid = 1; + memo_key(ex, gen, &cx->SM.k), cx->SM.vs = vs, cx->SM.fs = fs, cx->SM.has_fs = has_fs, cx->SM.valid = 1; } if (!has_vs) return draw_skipped(ex, mesh, SK_NO_VERTEX_STAGE); // A draw into a target the previous frame did not render (or into none) waits for a pending shader or pipeline: @@ -4077,8 +4097,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int const char *why = NULL; uint64_t w0 = 0, m0 = 0; // m0: when the lookup missed (wait_note) int kind = WK_LINK; - if (pipe_memo_on() && dc->PM.valid && !memcmp(&dc->PM.s, &s, sizeof s)) { - info = dc->PM.info, pending = 0; + if (pipe_memo_on() && cx->PM.valid && !memcmp(&cx->PM.s, &s, sizeof s)) { + info = cx->PM.info, pending = 0; p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); } else { // Nearly every draw finds its pipeline, so the first lookup waits for nothing and the clock is read only for a @@ -4127,8 +4147,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int pool_charge(last, b0); } if (m0) wait_note(kind, m0, pending), draw_stats_tick(); - dc->PM.valid = !pending && p && info; - if (dc->PM.valid) dc->PM.s = s, dc->PM.info = info; + cx->PM.valid = !pending && p && info; + if (cx->PM.valid) cx->PM.s = s, cx->PM.info = info; } uint64_t waited = w0 ? kern_now_ns() - w0 : 0; static int wait_logs; @@ -4146,7 +4166,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int if (!p || !info) return draw_skipped(ex, mesh, SK_PIPE_FAILED); if (info->pool && !pool_on()) return draw_skipped(ex, mesh, SK_POOL_OFF); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p); - if (!dc->B.st_valid || memcmp(&dc->B.st, &s, sizeof s)) { + if (!cx->B.st_valid || memcmp(&cx->B.st, &s, sizeof s)) { // Only the dynamic states whose values changed (vk_pipeline_set_dynamic_delta); NX2NAVI_DYN_DELTA=0 records // them all after every pipeline bind, as before. static int delta = -1; @@ -4154,11 +4174,11 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int delta = cfg_bool("NX2NAVI_DYN_DELTA"); if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)"); } - if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &dc->B.dyn, mesh && !fb, + if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &cx->B.dyn, mesh && !fb, &vk_exec_frame.dyn_left); else vk_pipeline_set_dynamic(ex->cb, &s); ex->recorded++; - dc->B.st = s, dc->B.st_valid = 1; + cx->B.st = s, cx->B.st_valid = 1; if (ex->mark && vk_exec_frame.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; } if (!mesh) { @@ -4589,15 +4609,16 @@ static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSiz // NX2NAVI_MVOBJ_BANKS=1 (test): for scene draws, the words of each other uniform bank (c3, c6, c8-c16) that changed // since the same shader's draw last frame, logged at frame 3000: where the time that moves wind and water lives static void mvobj_banks(VkExec *ex) { + DrawCtx *const cx = dc; static int on = -1; if (on < 0) on = rt_knob("MVOBJ_BANKS", "0"); - if (!on || !dc->B.st_valid) return; + if (!on || !cx->B.st_valid) return; uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (frame < 2999 || frame > 3000) return; static struct { const GfxShader *vs; int bank; uint64_t frame; float w[64]; int n; } tab[4096]; static int logged; const GfxShaderRes *ur; - int nu = gfx_shader_resources(dc->B.st.vs, GFX_RES_UBO, &ur); + int nu = gfx_shader_resources(cx->B.st.vs, GFX_RES_UBO, &ur); for (int i = 0; i < nu; i++) { int bank = ur[i].slot; if (bank < 3 || bank == 4 || bank == 5 || bank == 7 || bank > 16) continue; @@ -4606,16 +4627,16 @@ static void mvobj_banks(VkExec *ex) { const float *now = n ? (const float *)vk_usnap_bytes(u->addr, n * 4) : NULL; if (!now) continue; uint32_t k = - (uint32_t)(((uintptr_t)dc->B.st.vs ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull >> 52); + (uint32_t)(((uintptr_t)cx->B.st.vs ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull >> 52); if (frame == 2999) { - tab[k].vs = dc->B.st.vs, tab[k].bank = bank, tab[k].frame = frame, tab[k].n = (int)n; + tab[k].vs = cx->B.st.vs, tab[k].bank = bank, tab[k].frame = frame, tab[k].n = (int)n; memcpy(tab[k].w, now, n * 4); - } else if (tab[k].vs == dc->B.st.vs && tab[k].bank == bank && tab[k].frame == 2999 && logged < 80) { + } else if (tab[k].vs == cx->B.st.vs && tab[k].bank == bank && tab[k].frame == 2999 && logged < 80) { char line[700]; int o = 0, nd = 0; for (int j = 0; j < tab[k].n && j < (int)n && o < 600; j++) if (memcmp(&tab[k].w[j], &now[j], 4)) nd++, o += snprintf(line + o, sizeof line - o, " [%d] %g>%g", j, tab[k].w[j], now[j]); - if (nd) logged++, rt_log("mvbanks: vs %p c%d size %#llx: %d words%s", (const void *)dc->B.st.vs, bank, + if (nd) logged++, rt_log("mvbanks: vs %p c%d size %#llx: %d words%s", (const void *)cx->B.st.vs, bank, (unsigned long long)u->size, nd, line); tab[k].frame = 0; } @@ -4708,17 +4729,18 @@ static int mo_prev_cached(int index, const p_bind_buf *u, VkDescriptorBufferInfo } static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0) { + DrawCtx *const cx = dc; mvobj_banks(ex); const VkExecTarget *z = &ex->depth; - if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || dc->B.st.samples > 1 || - dc->B.st.raster_discard) + if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || cx->B.st.samples > 1 || + cx->B.st.raster_discard) return; // solid geometry only, which tests and writes the scene depth: effects, particles and screen quads write none, and // drawn again they would put their motion on the surfaces behind them (DLSS takes those from the surface) - if (!dc->B.st.ds.dt || !dc->B.st.ds.dw) return; + if (!cx->B.st.ds.dt || !cx->B.st.ds.dw) return; // a vertex shader that reads a user bank besides the camera's (c4) may move on its own const GfxShaderRes *ur; - int nu = gfx_shader_resources(dc->B.st.vs, GFX_RES_UBO, &ur), reads = 0; + int nu = gfx_shader_resources(cx->B.st.vs, GFX_RES_UBO, &ur), reads = 0; for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4; if (!reads) return; MO.cand++; @@ -4730,20 +4752,20 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi // the variant: the translation of the same code and answers, kept per shader once found; it names the banks the // position depends on (0: none, the shader is skipped from then on) static struct { const GfxShader *of, *vs; uint32_t banks; } vmemo[1024]; - uint32_t vi = (uint32_t)((uintptr_t)dc->B.st.vs * 0x9E3779B97F4A7C15ull >> 54); - if (vmemo[vi].of != dc->B.st.vs) { + uint32_t vi = (uint32_t)((uintptr_t)cx->B.st.vs * 0x9E3779B97F4A7C15ull >> 54); + if (vmemo[vi].of != cx->B.st.vs) { int has, pending = 0; - dc->sf_mvprev = 1; + cx->sf_mvprev = 1; const GfxShader *v = shader_for(ex, NVN_STAGE_VS, &has, &pending, 0); - dc->sf_mvprev = 0; + cx->sf_mvprev = 0; if (!v || pending) { MO.building++; return; } GfxShaderIo io[32]; - uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(dc->B.st.vs, io); + uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(cx->B.st.vs, io); uint32_t banks = extra ? gfx_shader_mvprev_banks(v) & 0x1fff8u : 0; - vmemo[vi].of = dc->B.st.vs, vmemo[vi].vs = v, vmemo[vi].banks = banks; + vmemo[vi].of = cx->B.st.vs, vmemo[vi].vs = v, vmemo[vi].banks = banks; vk_mvprev_want((banks & ~(1u << 4)) >> 3); // binding index = bank - 3 } const GfxShader *vs = vmemo[vi].vs; @@ -4784,7 +4806,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi exact = e && *e ? strtol(e, NULL, 16) : -1; } if (exact >= 0 && banks != (uint32_t)exact) return; - mo_mask_note(banks, dc->B.st.vs); + mo_mask_note(banks, cx->B.st.vs); // last frame's camera and blocks; held still when none of the position's blocks changed (most of the scenery) VkDescriptorBufferInfo cam, prevb[17]; float jit[2]; @@ -4823,7 +4845,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi } if (t0) MO.ns_part[1] += kern_now_ns() - tp, MO.n_part[1]++, tp = kern_now_ns(); GfxShaderIo io[32]; - uint32_t extra = gfx_shader_outputs(vs, io) & ~gfx_shader_outputs(dc->B.st.vs, io); + uint32_t extra = gfx_shader_outputs(vs, io) & ~gfx_shader_outputs(cx->B.st.vs, io); const GfxShaderRes *r; if (gfx_shader_resources(vs, GFX_RES_IMG, &r)) { MO.storage++; @@ -4834,7 +4856,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi MO.no_fs++; return; } - VkPipeState ms = dc->B.st; + VkPipeState ms = cx->B.st; ms.vs = vs, ms.fs = fs, ms.feedback = 0; memset(ms.color_fmt, 0, sizeof ms.color_fmt); ms.color_fmt[0] = NVN_FORMAT_RG16F; @@ -4846,7 +4868,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi // scene depth passes, anything nearer hides it (NX2NAVI_MVOBJ_EQUAL=1: EQUAL, for comparison) static int eq = -1; if (eq < 0) eq = rt_knob("MVOBJ_EQUAL", "0"); - int df = dc->B.st.ds.df; + int df = cx->B.st.ds.df; df = eq ? NVN_DEPTH_FUNC_EQUAL : df == NVN_DEPTH_FUNC_LESS ? NVN_DEPTH_FUNC_LEQUAL : df == NVN_DEPTH_FUNC_GREATER ? NVN_DEPTH_FUNC_GEQUAL : df == NVN_DEPTH_FUNC_LEQUAL || df == NVN_DEPTH_FUNC_GEQUAL ? df : NVN_DEPTH_FUNC_EQUAL; @@ -4896,13 +4918,13 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi MQ.par = (VkDescriptorBufferInfo){pbuf, poff, 16}; memcpy(MQ.par_in, par_in, sizeof par_in); } - memset(dc->mo_ovr, 0, sizeof dc->mo_ovr); + memset(cx->mo_ovr, 0, sizeof cx->mo_ovr); for (int bk = 3; bk <= 16; bk++) // last frame's block, or this frame's where none was found - dc->mo_ovr[NVN_STAGE_VS][bk + 15] = bk == 4 ? cam : have >> bk & 1 ? prevb[bk] : ubo_info(ex, NVN_STAGE_VS, bk); - dc->mo_ovr[NVN_STAGE_FS][3] = MQ.par; - dc->mo_ovr_on = 1; + cx->mo_ovr[NVN_STAGE_VS][bk + 15] = bk == 4 ? cam : have >> bk & 1 ? prevb[bk] : ubo_info(ex, NVN_STAGE_VS, bk); + cx->mo_ovr[NVN_STAGE_FS][3] = MQ.par; + cx->mo_ovr_on = 1; int bound = mo_desc(ex, info, q); - dc->mo_ovr_on = 0; + cx->mo_ovr_on = 0; if (!bound) { MO.storage++; return; @@ -5273,6 +5295,7 @@ static int mfb_dispatch(VkExec *ex, const GfxShader *cs, int stage, VkBuffer rec // A mesh draw of `count` workgroups, or of the count in the game's record at ind (4 bytes, GPU-written). static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { + DrawCtx *const cx = dc; if (ex->render_disabled) { vk_cond_draws_skip += ex->has_render_cond; draw_skipped(ex, 1, SK_DISABLED); @@ -5319,7 +5342,7 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { if (!mfb_dispatch(ex, c->cs_task, NVN_STAGE_TASK, rec, at)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; } vk_exec_barrier(ex); vk_meshfb_scan(ex->cb, at, c->ti.payload_bytes / 4, c->ti.threads, slot_words, verts); - dc->B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own + cx->B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own ex->recorded += 4; vk_exec_barrier(ex); } @@ -5328,8 +5351,8 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { vk_meshfb_note(ex->cb, at); ex->recorded++; vk_exec_barrier(ex); - dc->c0_binst = 0; - dc->mfb_cur_vs = vs; + cx->c0_binst = 0; + cx->mfb_cur_vs = vs; int prim = c->mi.ipp == 1 ? NVN_DRAW_PRIMITIVE_POINTS : c->mi.ipp == 2 ? NVN_DRAW_PRIMITIVE_LINES : NVN_DRAW_PRIMITIVE_TRIANGLES; if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 2)) return; vk_exec_frame.draws++; @@ -6072,6 +6095,7 @@ static void untranslated_dispatch_fallback(VkExec *ex) { } static void op_dispatch(VkExec *ex, const p_dispatch *d) { + DrawCtx *const cx = dc; if (d->x <= 0 || d->y <= 0 || d->z <= 0) return; if (!ex->prog) return warn_once(W_PROGRAM, "a dispatch without a bound program is skipped"); if (prog_lead_on()) prog_lead(ex->prog, 2); @@ -6109,7 +6133,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS vk_exec_barrier_dispatch(ex); // ends the pass; earlier writes are visible to the shader bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); - dc->c0_dim[0] = (uint32_t)d->x, dc->c0_dim[1] = (uint32_t)d->y, dc->c0_dim[2] = (uint32_t)d->z; + cx->c0_dim[0] = (uint32_t)d->x, cx->c0_dim[1] = (uint32_t)d->y, cx->c0_dim[2] = (uint32_t)d->z; twin_dispatch = 1; { // the replay's binding path is the CPU cost the frame profile needs to separate from recording uint64_t bt = bt_now(); @@ -6128,6 +6152,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { // DispatchComputeIndirect: vkCmdDispatchIndirect from the pool buffer. The shader reads the group counts from c0 0x200, // so they are copied there on the GPU, into a c0 range no other dispatch uses, before the dispatch. static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { + DrawCtx *const cx = dc; if (!ex->prog) return warn_once(W_PROGRAM, "a dispatch without a bound program is skipped"); VkMemSpan ip; if (!vk_mem_resolve(d->ind, sizeof(VkDispatchIndirectCommand), &ip) || ip.offset % 4) @@ -6146,8 +6171,8 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vk_exec_barrier_dispatch(ex); VK_MEM_USE(ip); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); - dc->c0_dim[0] = dc->c0_dim[1] = dc->c0_dim[2] = 0; - dc->B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads + cx->c0_dim[0] = cx->c0_dim[1] = cx->c0_dim[2] = 0; + cx->B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads twin_dispatch = 1; { // the replay's binding path is the CPU cost the frame profile needs to separate from recording uint64_t bt = bt_now(); @@ -6156,17 +6181,17 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vk_exec_frame.bind_calls++; } twin_dispatch = 0; - if (dc->B.c0_valid[2]) { // the shader binds c0 + if (cx->B.c0_valid[2]) { // the shader binds c0 // NVN2 programs read the group counts from c0 byte 0 (c0_info); at 0x200 their bank holds pushed constants // or the driver's tables, which the copy would overwrite const uint8_t *pctl, *pcode; uint32_t plen; int nvn2 = find_stage(ex->prog, NVN_STAGE_CS, &pctl, &pcode, &plen) && ctl_nvn2(pctl); - VkBufferCopy cp = {ip.offset, dc->B.c0_off[2] + (nvn2 ? 0 : 0x200), sizeof(VkDispatchIndirectCommand)}; - vkCmdCopyBuffer(ex->cb, ip.buffer, dc->B.c0_buf[2], 1, &cp); + VkBufferCopy cp = {ip.offset, cx->B.c0_off[2] + (nvn2 ? 0 : 0x200), sizeof(VkDispatchIndirectCommand)}; + vkCmdCopyBuffer(ex->cb, ip.buffer, cx->B.c0_buf[2], 1, &cp); ex->recorded++; vk_exec_barrier(ex); - dc->B.c0_valid[2] = 0; // the range no longer holds B.c0[2] + cx->B.c0_valid[2] = 0; // the range no longer holds B.c0[2] } VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0); vkCmdDispatchIndirect(ex->cb, ip.buffer, ip.offset); @@ -6210,11 +6235,12 @@ static void bind_handles(uint64_t *dst, int cap, const p_bind_handles *b) { } static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { + DrawCtx *const cx = dc; switch (op) { case OP_BIND_PROGRAM: { const p_bind_program *b = payload; ex->prog = b->prog, ex->prog_stages = b->stages; - dc->bind_gen++; + cx->bind_gen++; break; } case OP_BIND_STATE: { @@ -6263,7 +6289,7 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { rt_log("vk: uniform buffer bind to index %d (bank %d), stage %d, %llu bytes: beyond Maxwell's banks, dropped", b->index, b->index + 3, b->stage, (unsigned long long)b->size); } - dc->bind_gen++; + cx->bind_gen++; break; } case OP_BIND_SSBO: { @@ -6276,14 +6302,14 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { rt_log("vk: storage buffer bind, stage %d slot %d: %#llx, %llu bytes", b->stage, b->index, (unsigned long long)b->addr, (unsigned long long)b->size); } - dc->bind_gen++; + cx->bind_gen++; break; } case OP_BIND_TEX: { const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 8 ? ex->tex[b->stage] : NULL, 128, b); ex->tex_serial++; - dc->bind_gen++; + cx->bind_gen++; break; } case OP_BIND_SAMP: { @@ -6295,14 +6321,14 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 8 ? ex->img[b->stage] : NULL, 32, b); ex->tex_serial++; // the shader memo's handle check (feedback_scan reads only textures: a rescan at most) - dc->bind_gen++; + cx->bind_gen++; break; } case OP_UPDATE_UB: op_update_ub(ex, payload); break; case OP_PUSH_CC: { const p_push_cc *p = payload; if (p->offset + (uint64_t)p->size <= sizeof ex->cs_push) memcpy(ex->cs_push + p->offset, p->data, p->size); - dc->bind_gen++; // storage buffers may come from these words (push_ptr) + cx->bind_gen++; // storage buffers may come from these words (push_ptr) break; } case OP_DRAW: op_draw(ex, payload); break; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 44dc998d..62e24591 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -35,6 +35,11 @@ static CmdSlot *slots; static int nslots; static int cur_slot; // the slot vk_sync_submit records into now (an index: take_slot may move slots) static uint64_t cur_fence; // the value of the submission being recorded +// The command buffers of the part being recorded that come before slots[cur_slot] (vk_sync_append): earlier pieces of +// the replay's own and those the replay workers recorded, in queue order. +#define PART_CBS 254 +static VkCommandBuffer part_cbs[PART_CBS]; +static int npart_cbs; // Shared with the completion thread (st_lock). static PlatMutex st_lock = PLAT_MUTEX_INIT; @@ -69,7 +74,7 @@ static void results_run(uint64_t upto); static struct { int n; VkSubmitInfo2 si[HOLD_CAP]; - VkCommandBufferSubmitInfo cbi[HOLD_CAP][2]; + VkCommandBufferSubmitInfo cbi[HOLD_CAP][PART_CBS + 2]; // the prologue and the part's command buffers VkSemaphoreSubmitInfo sig[HOLD_CAP][2]; uint64_t last; // the newest timeline value a held submission signals, 0 none uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried @@ -694,6 +699,77 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { return cb; } +// The command buffers of the part that ends now, in queue order, into cbi: the prologue (when there is one), the +// part's earlier pieces (vk_sync_append) and, with `with_last`, its last command buffer. Empties the list. +static uint32_t part_list(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, VkCommandBuffer last, int with_last) { + uint32_t n = 0; + VkCommandBufferSubmitInfo one = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}; + if (pro) one.commandBuffer = pro, cbi[n++] = one; + for (int i = 0; i < npart_cbs; i++) one.commandBuffer = part_cbs[i], cbi[n++] = one; + if (with_last) one.commandBuffer = last, cbi[n++] = one; + npart_cbs = 0; + return n; +} + +// The replay workers (vk_exec.c "replay workers"): the replay's command buffer being recorded ends and joins the +// part's list, then the n command buffers the workers recorded, in that order, and a new command buffer of the same +// part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers' commands run +// between the replay's commands before and after. vk_sync_append_room: whether the list has room for n more. +int vk_sync_append_room(int n) { return npart_cbs + n + 1 <= PART_CBS; } + +VkCommandBuffer vk_sync_append(const VkCommandBuffer *cbs, int n) { + if (!vk_sync_append_room(n)) rt_fatal("vk: a submission part of more than %d command buffers", PART_CBS); + VkCtx *c = vk_ctx(); + VkCommandBuffer cb = slots[cur_slot].cb; + if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of it is in cb, ended by its recorder + part_cbs[npart_cbs++] = cb; + for (int i = 0; i < n; i++) part_cbs[npart_cbs++] = cbs[i]; + CmdSlot *s = take_slot(c, cur_fence); // the ended one keeps its slot until the value is reached + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(s->cb, &bi); + rec_begin(s->cb, cur_fence); + return s->cb; +} + +// The replay workers' command buffers: each worker has a command pool of its own (a pool belongs to one thread at a +// time) and claims command buffers from it for the submission being recorded, as the replay claims its slots. Called +// on the worker while the submitting thread holds sub_lock and waits for it, so cur_fence stays put. +#define SYNC_WORKERS 16 +static struct { VkCommandPool pool; CmdSlot *slots; int n; } WP[SYNC_WORKERS]; + +VkCommandBuffer vk_sync_worker_begin(int w) { + VkCtx *c = vk_ctx(); + if (w < 0 || w >= SYNC_WORKERS) rt_fatal("vk: replay worker %d of %d", w, SYNC_WORKERS); + typeof(WP[0]) *p = &WP[w]; + if (!p->pool) { + VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, + .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, + .queueFamilyIndex = c->gfx_family}; + if (vkCreateCommandPool(c->device, &pi, NULL, &p->pool) != VK_SUCCESS) rt_fatal("vk: replay worker command pool"); + } + uint64_t gpu = 0; + vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu); + CmdSlot *s = NULL; + for (int i = 0; i < p->n && !s; i++) + if (p->slots[i].fence <= gpu) s = &p->slots[i]; + if (!s) { + CmdSlot *m = realloc(p->slots, sizeof *m * (size_t)(p->n + 1)); + if (!m) rt_fatal("vk: out of memory (command buffers)"); + p->slots = m, s = &m[p->n++]; + VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = p->pool, + .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; + if (vkAllocateCommandBuffers(c->device, &ai, &s->cb) != VK_SUCCESS) rt_fatal("vk: command buffer allocation"); + } + s->fence = cur_fence; + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(s->cb, &bi); + return s->cb; +} + +void vk_sync_worker_end(VkCommandBuffer cb) { vkEndCommandBuffer(cb); } + static VkCommandBuffer split_part(int exec); VkCommandBuffer vk_sync_split(void) { return split_part(0); } // The part that ends here is the last with the submission's own commands: it signals the executed timeline. @@ -707,10 +783,9 @@ static VkCommandBuffer split_part(int exec) { if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder vk_upload_flush(0); // the uploads this part asked for precede it in queue order VkCommandBuffer pro = prologue(c, cur_fence); - VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, - {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}}; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1, - .pCommandBufferInfos = pro ? cbi : cbi + 1}; + VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = part_list(cbi, pro, cb, 1), + .pCommandBufferInfos = cbi}; VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; if (exec && exec_tl && exec_sig < cur_fence) @@ -785,8 +860,8 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { vk_upload_flush(0); VkCommandBuffer pro = prologue(c, fence); s = &slots[cur_slot]; - VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, - {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}}; + VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; + uint32_t ncbi = part_list(cbi, pro, s->cb, recorded); // the timeline, and the executed timeline unless a part of this submission signalled it already (split_part) VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}, @@ -794,9 +869,8 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}}; int both = exec_tl && exec_sig < fence; if (both) exec_sig = fence; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, - .commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u), - .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = both ? 2u : 1u, + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncbi, + .pCommandBufferInfos = cbi, .signalSemaphoreInfoCount = both ? 2u : 1u, .pSignalSemaphoreInfos = sig}; ts_submitted(); queue_submit(c, &si, fence); // held or handed to the queue (submit batching) @@ -980,6 +1054,11 @@ void vk_sync_shutdown(void) { plat_mutex_lock(&sub_lock); VkCtx *c = vk_ctx(); if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL); + for (int w = 0; w < SYNC_WORKERS; w++) { + if (c && WP[w].pool) vkDestroyCommandPool(c->device, WP[w].pool, NULL); + free(WP[w].slots); + memset(&WP[w], 0, sizeof WP[w]); + } if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL); exec_tl = VK_NULL_HANDLE; perf_gpu_stats = NULL;