diff --git a/runtime/vk.h b/runtime/vk.h index ad1d10ec..7419b0ea 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -531,6 +531,15 @@ const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view); void vk_texture_lock(void); void vk_texture_unlock(void); const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view); +// The replay workers (vk_exec.c "replay workers"): a chunk's lookup that would make or take back a texture entry +// before the chunk's turn returns nothing and says so here; the chunk replays the op again in turn +// (vk_texture_deferred_take clears it). vk_exec_turn_due: the calling thread records a chunk whose earlier chunks may not +// be done yet; vk_exec_turn waits until they are (no-op elsewhere). +int vk_texture_deferred(void); +int vk_texture_deferred_take(void); +int vk_exec_turn_due(void); +int vk_exec_in_chunk(void); // the calling thread records a chunk of a parallel group +void vk_exec_turn(void); VkSampler vk_sampler_by_id_locked(int id); // The slot's sampler with nearest filtering and no anisotropy: a depth texture read without a compare lands on one texel // on the Switch although the game binds a linear sampler. VK_NULL_HANDLE diff --git a/runtime/vk_contact.c b/runtime/vk_contact.c index ba08c435..35bf6fd4 100644 --- a/runtime/vk_contact.c +++ b/runtime/vk_contact.c @@ -112,6 +112,8 @@ static void mem_barrier(VkCommandBuffer cb) { vkCmdPipelineBarrier2(cb, &d); } +int vk_contact_pending(void) { return C.pending; } + void vk_contact_flush(VkExec *ex) { if (!C.pending) return; C.pending = 0; diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index edd3c9b8..cdeda538 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -1540,8 +1540,16 @@ void vk_dlaa_jitter_check(VkExec *ex) { static int scene_color(VkFormat f); +static void dlaa_draw_locked(VkExec *ex); void vk_dlaa_draw(VkExec *ex) { if (!dl_active() || D.ended) return; + static PlatMutex lock = PLAT_MUTEX_INIT; // the replay workers' scene draws count at once + plat_mutex_lock(&lock); + dlaa_draw_locked(ex); + plat_mutex_unlock(&lock); +} + +static void dlaa_draw_locked(VkExec *ex) { static int survey = -1; // NX2NAVI_DLAA_SURVEY=1 (test): what is drawn into the armed light target if (survey < 0) survey = rt_knob("DLAA_SURVEY", "0"); static uint64_t dfn[16]; // scene draws writing depth, by depth test (reversed depth: GREATER/GEQUAL) diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 9b00c4aa..b8fbf533 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -47,6 +47,32 @@ static int x_ready; // it records none: vk_exec_inline_begin hands out its command buffer. static _Thread_local VkExec *x_cur; +// ---- parallel groups (NX2NAVI_REPLAY_WORKERS; "replay workers" below). A chunk: a part of a group's ops that one +// thread records into a command buffer of its own, from the replay state the front pass left at its first op. +#define GROUP_CHUNKS 17 // chunks a group is cut into at most: RW_MAX workers and the submitting thread +typedef struct { + VkExec ex; // the state at its first op, then the state it records with + const op_hdr *h; // its first op + uint32_t nops; // the ops it records (JUMPs not counted) + uint32_t cond0, ncond; // its render conditions' decisions in G.cond + int idx; // in its group: 0 runs the group's pass hooks + VkCommandBuffer cb; + int recorded; + volatile int32_t done; +} Chunk; +static struct { + Chunk c[GROUP_CHUNKS]; + int n; + uint8_t *cond; // the decisions of the group's render conditions, in stream order + uint32_t ncond, ccond; + volatile int32_t next; // the next chunk to take (huge between groups) + volatile int32_t hooks; // chunk 0 began its first pass (or ended): the others may start + volatile uint32_t gen; // bumped when a group is handed out +} G = {.next = 1 << 30}; +static _Thread_local Chunk *rw_chunk; // the chunk the calling thread records, NULL outside one +static _Thread_local int rw_ordered; // every earlier chunk of its group is done +static PlatMutex tfr_lock = PLAT_MUTEX_INIT; // target frames (vk_target_frames.h) noted by several chunks at once + // ---- NX2NAVI_TRACE=gfx frame counters VkExecFrame vk_exec_frame; @@ -752,8 +778,13 @@ void vk_exec_barrier_log(uint64_t frame) { void vk_exec_pass_begin(VkExec *ex) { if (ex->in_pass) return; resolve_rt(ex); - vk_contact_pass_begin(ex); - vk_dlaa_pass(ex); // DLAA runs before the bloom chain's first pass (vk_dlaa.c) + if (vk_texture_deferred()) return; // a chunk's lookup waits for its turn (chunk_record): the op is replayed again + // A parallel group's targets do not change: its pass hooks, which follow the frame pass by pass, run in its first + // chunk, which the others start after (chunk_record). + if (!rw_chunk || !rw_chunk->idx) { + vk_contact_pass_begin(ex); + vk_dlaa_pass(ex); // DLAA runs before the bloom chain's first pass (vk_dlaa.c) + } // Learned targets (VkTexImage.fb_learned) are attached in the feedback layout VkImage want[9]; VkImageAspectFlags wasp[9]; @@ -792,7 +823,7 @@ void vk_exec_pass_begin(VkExec *ex) { } // The start mark goes between the barrier and vkCmdBeginRendering, its end right after vkCmdEndRendering // (vk_exec_pass_end): outside the pass, and without breaking it. - VkPassMark *m = ex->mark = vk_pass_mark_begin(ex->cb, VK_MARK_PASS, ex->fence); + VkPassMark *m = ex->mark = rw_chunk ? NULL : vk_pass_mark_begin(ex->cb, VK_MARK_PASS, ex->fence); // one ring if (m) { m->step = ex->mark_passes++; m->w = ex->pass_w, m->h = ex->pass_h, m->d = ex->pass_layers; @@ -814,6 +845,7 @@ void vk_exec_pass_begin(VkExec *ex) { vk_exec_frame.passes++; ex->recorded++; ex->in_pass = 1; + if (rw_chunk && !rw_chunk->idx) __atomic_store_n(&G.hooks, 1, __ATOMIC_RELEASE); } void vk_exec_pass_end(VkExec *ex) { @@ -835,6 +867,7 @@ void vk_exec_pass_end(VkExec *ex) { bc_copy_back(ex, &ex->color[i]); if (vk_dump_armed) vk_dump_pass_end(ex); uint64_t stamp = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1; + if (rw_chunk) plat_mutex_lock(&tfr_lock); for (int i = 0; i < 8; i++) if (ex->color[i].image) { TfrTarget ft = tfr_of(&ex->color[i]); @@ -844,6 +877,7 @@ void vk_exec_pass_end(VkExec *ex) { TfrTarget ft = tfr_of(&ex->depth); vk_texture_gpu_written(ex->depth.tex), tfr_note(&ft, stamp); } + if (rw_chunk) plat_mutex_unlock(&tfr_lock); } VkCommandBuffer vk_exec_inline_begin(int *reopen) { @@ -947,6 +981,7 @@ static void clear_color(VkExec *ex, const p_clear_color *c) { if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : vk_cond_clears_run++; if (ex->render_disabled) return; resolve_rt(ex); + if (vk_texture_deferred()) return; int i = c->index; if (i < 0 || i >= (int)ex->pass_ncolor || !ex->color[i].image) { warn_once(W_CLEAR_TARGET, "colour clear of target %d, which is not bound or not renderable: skipped", i); @@ -980,6 +1015,7 @@ static void clear_ds(VkExec *ex, const p_clear_ds *c) { if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : (vk_cond_clears_run++, vk_cond_depth_run++); if (ex->render_disabled) return; resolve_rt(ex); + if (vk_texture_deferred()) return; const VkExecTarget *t = &ex->depth; if (!t->image) return; VkImageAspectFlags asp = 0; @@ -1482,7 +1518,7 @@ void vk_exec_ahead(VkExec *ex) { rt_log("vk: a draw that finds its pipeline missing has the rest of its submission read ahead for the stage " "libraries the later draws need: %s (NX2NAVI_PIPE_AHEAD)", AH.on ? "on" : "off"); } - if (!AH.on || !AH.live || evo_in || AH.seq == vk_exec_submit_seq) return; + if (!AH.on || !AH.live || evo_in || rw_chunk || AH.seq == vk_exec_submit_seq) return; // not in a chunk AH.seq = vk_exec_submit_seq; uint64_t t0 = kern_now_ns(); uint32_t ops = 0; @@ -1685,24 +1721,30 @@ static void op_one(VkExec *ex, const op_hdr *h) { } } -// ---- replay workers (NX2NAVI_REPLAY_WORKERS, default 0; stage 1 of recording a submission on several threads) -// A submission's draws come in runs: between two ops that have to stay in the replay's order on its own thread -// (dispatches, copies, texture and buffer clears, events, the game's barriers, calls and the end of a command buffer) -// there are only state, binds, draws and the ops a draw run carries (counter reports and resets, render conditions, -// inline uniform updates, render target changes and attachment clears). Hyrule Field, standing (NX2NAVI_REPLAY_RUNS, -// 2026-11-11): about 3,500 draws a frame in 82 runs, 96% of them in runs of 16 draws or more and 67% in runs of 256 -// or more, with a counter report or reset beside nearly every draw and a render condition beside every second one. -// With NX2NAVI_REPLAY_WORKERS set, a run of RUN_MIN draws or more is recorded by a worker thread: it takes a copy of -// the replay state, records the run into a command buffer of its own pool (vk_sync_worker_begin) with a draw context -// of its own (vk_exec_draw.c DrawCtx), ends it behind a full barrier, and hands the state the run left back. The -// replay's command buffer up to the run ended behind a full barrier too, and vk_sync_append puts the worker's command -// buffer after it in the same part, so the GPU runs everything in the order of the op stream. Stage 1 has one worker -// and the submitting thread waits for each run: the order of every side effect is the replay's own, and what it shows -// is whether a run recorded apart, from a copy of the state, gives the same pictures. Later stages record the runs -// of a submission on several workers at once. +// ---- replay workers (NX2NAVI_REPLAY_WORKERS, default 0: the replay records every op on the submitting thread) +// A submission's draws come in runs between ops that have to stay in the replay's order on its own thread (dispatches, +// copies, texture and buffer clears, events, the game's barriers, calls). Hyrule Field, standing (NX2NAVI_REPLAY_RUNS, +// 2026-11-11): about 3,500 draws a frame in 82 runs, 96% of them in runs of 16 draws or more and 67% in runs of 256 or +// more, with a timestamp report beside nearly every draw and a render condition beside every second one. +// A run is cut into groups: a group ends where the render targets change after its first draw or clear, and before an +// inline uniform update, a mesh draw and a samples-passed counter op, which the submitting thread replays itself in +// the order of the stream like everything outside groups. A group of 2 * CHUNK_DRAWS draws or more is recorded by +// several threads at once: +// - the front pass, on the submitting thread, walks the group in the order of the stream before anything of it is +// recorded: it applies the state and binds, settles the CPU side of counter reports and decides the render conditions +// (vk_exec_xfer_front), and copies the replay state at the first op of each chunk; +// - the chunks are taken in order by the workers and the submitting thread, each recording into a command buffer of its +// own pool with a draw context of its own, from its copy of the state, behind the full barrier that ended the +// replay's command buffer before the group, and ending behind a full barrier of its own. vk_sync_append puts them +// after the replay's command buffer in the same part, in the order of the stream, and the replay goes on in a new one. +// The frame's pass hooks (vk_contact_pass_begin, vk_dlaa_pass) follow the frame pass by pass: they run in chunk 0, +// which reaches the group's pass first, and the others start after that pass began; groups follow one another. +// What still depends on the order between chunks waits for its turn (vk_exec_turn: every earlier chunk of the group +// done): a texture lookup that would make or take back an entry (vk_texture.c entry_of) returns nothing and the chunk +// replays the op again once in turn, and so do the contact shadows' pass, per-object motion and the HDR pass's source. #define RW_MAX 16 -#define RUN_MIN 16 // draws a run needs to go to a worker -#define RUN_PEEK 65536u // ops the submitting thread reads ahead to find a run's draws, at most +#define CHUNK_DRAWS 32 // draws a chunk takes at least +#define GROUP_PEEK 65536u // ops read ahead for a group, at most // The ops a run may hold; JUMP is followed. static int run_op(uint32_t op) { @@ -1719,57 +1761,50 @@ static int run_op(uint32_t op) { } } -static int is_draw(uint32_t op) { - return op == OP_DRAW || op == OP_DRAW_IDX || op == OP_DRAW_IDX_IND || op == OP_DRAW_MESH || op == OP_DRAW_MESH_IND; -} - typedef struct { PlatThread *t; - int idx; + int idx; // also its command pool (vk_sync_worker_begin); the submitting thread's is RW.n void *dc; // its draw context (vk_exec_draw_ctx_new) - volatile int32_t go, done; // a run is handed over / recorded volatile int32_t sleeping; // waits for wake rather than spinning PlatEvent wake; - VkExec ex; // the replay state it records the run with, and leaves - const op_hdr *h; // the run's first op; then the op that ended it - int depth; // the call depth of the run (for vk_exec_ahead) } RwWorker; static struct { int n; // workers; 0 off, -1 not read yet RwWorker *w[RW_MAX]; - const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op and found too few draws - uint64_t runs, run_draws, short_runs, wait_ns; // since the last perf line + const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small + uint64_t groups, group_draws, chunks, serial_draws, wait_ns; // since the last perf line + volatile uint64_t turns, turn_ns, redos; } RW = {.n = -1}; -static void rw_walk(RwWorker *w); +static void group_take(int pool); static void rw_main(void *arg) { RwWorker *w = arg; plat_thread_priority(PLAT_PRIO_HIGHEST); // as the recording thread (rec_claim) vk_exec_draw_ctx_use(w->dc); + uint32_t seen = 0; for (;;) { - // spins a while after a run (the next one is usually a few microseconds away), then sleeps - for (unsigned i = 0; !__atomic_load_n(&w->go, __ATOMIC_ACQUIRE); i++) { - if (i < 200000) { + // spins about 100 us after a group (the next one of the frame is usually closer), then sleeps + uint64_t t0 = kern_now_ns(); + for (unsigned i = 0; __atomic_load_n(&G.gen, __ATOMIC_ACQUIRE) == seen; i++) { + if ((i & 63) || kern_now_ns() - t0 < 100000) { plat_pause(); continue; } __atomic_store_n(&w->sleeping, 1, __ATOMIC_SEQ_CST); - if (!__atomic_load_n(&w->go, __ATOMIC_SEQ_CST)) plat_event_wait_ns(&w->wake, 2000000); + if (__atomic_load_n(&G.gen, __ATOMIC_SEQ_CST) == seen) plat_event_wait_ns(&w->wake, 2000000); __atomic_store_n(&w->sleeping, 0, __ATOMIC_RELAXED); - i = 0; + t0 = kern_now_ns(); } - __atomic_store_n(&w->go, 0, __ATOMIC_RELAXED); - rw_walk(w); - __atomic_store_n(&w->done, 1, __ATOMIC_RELEASE); + seen = __atomic_load_n(&G.gen, __ATOMIC_ACQUIRE); + group_take(w->idx); } } static void rw_init(void) { int n = cfg_int("NX2NAVI_REPLAY_WORKERS"); RW.n = n < 0 ? 0 : n > RW_MAX ? RW_MAX : n; - if (RW.n > 1) RW.n = 1; // stage 1: one worker for (int k = 0; k < RW.n; k++) { RwWorker *w = calloc(1, sizeof *w); if (!w) rt_fatal("vk: out of memory (replay worker)"); @@ -1780,87 +1815,195 @@ static void rw_init(void) { if (!(w->t = plat_thread_new(rw_main, w, name))) rt_fatal("vk: cannot start a replay worker"); RW.w[k] = w; } - if (RW.n) rt_log("vk: runs of %d draws or more are recorded by %d replay worker%s (NX2NAVI_REPLAY_WORKERS); the " - "submitting thread waits for each run", RUN_MIN, RW.n, RW.n > 1 ? "s" : ""); + if (RW.n) rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s and the submitting thread " + "(NX2NAVI_REPLAY_WORKERS)", 2 * CHUNK_DRAWS, RW.n, RW.n > 1 ? "s" : ""); } -// The draws from h up to the op that ends its run (*end), reading at most RUN_PEEK ops. -static uint32_t run_peek(const op_hdr *h, const op_hdr **end) { +int vk_exec_turn_due(void) { return rw_chunk && !rw_ordered; } +int vk_exec_in_chunk(void) { return rw_chunk != NULL; } + +void vk_exec_turn(void) { + Chunk *c = rw_chunk; + if (!c || rw_ordered) return; + uint64_t t0 = kern_now_ns(); + for (int i = 0; i < c->idx; i++) + while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause(); + rw_ordered = 1; + __atomic_add_fetch(&RW.turns, 1, __ATOMIC_RELAXED); + __atomic_add_fetch(&RW.turn_ns, kern_now_ns() - t0, __ATOMIC_RELAXED); +} + +// The image views a command buffer made for its targets go when its submission is done. +static void views_release(VkExec *ex, uint64_t fence) { + if (!ex->nviews) return; + ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); + if (!l) rt_fatal("vk: out of memory"); + l->n = ex->nviews; + for (int i = 0; i < ex->nviews; i++) l->v[i] = ((ViewEnt *)ex->views)[i].view; + vk_completion_add(fence, free_views, l); + ex->nviews = 0; +} + +// The draws of the group that starts at h (it ends at *end), reading at most GROUP_PEEK ops. *serial: it holds a +// mesh draw (those are replayed on the submitting thread). +static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end) { uint32_t draws = 0; - for (uint32_t ops = 0; ops < RUN_PEEK; ops++) { + int begun = 0; + const p_set_rt *rt = &ex->rt; + for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the walk reports it + const void *a = nvn_op_payload(h); if (h->op == OP_JUMP) { - h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target; + h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; } - if (!run_op(h->op)) break; - draws += (uint32_t)is_draw(h->op); + if (!run_op(h->op) || h->op == OP_UPDATE_UB || h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND || + vk_exec_xfer_serial(h->op, a)) + break; + if (h->op == OP_SET_RT && memcmp(rt, a, sizeof *rt)) { + if (begun) break; // another pass: the next group + rt = a; + } + if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) draws++, begun = 1; + if (h->op == OP_CLEAR_COLOR || h->op == OP_CLEAR_DS) begun = 1; h = nvn_op_next(h); } *end = h; return draws; } -// The worker's side: records the run from w->h into a command buffer of its own; w->h becomes the op that ended it. -static void rw_walk(RwWorker *w) { - VkExec *ex = &w->ex; - x_cur = ex; - ex->dc = w->dc; - ex->cb = vk_sync_worker_begin(w->idx); - // the replay's command buffer before the run ended behind a full barrier, with every image in GENERAL - ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL; - ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0; - vk_exec_draw_new_cb(); - const op_hdr *h = w->h; - for (;;) { - if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the walk reports it - if (h->op == OP_JUMP) { - h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target; - continue; - } - if (!run_op(h->op)) break; - AH.at[w->depth] = h, AH.depth = w->depth; // the submitting thread waits: where vk_exec_ahead reads on from - if (RR.on > 0) runs_op(h->op); - if (!op_linear(h->op)) vk_exec_brk++; - op_one(ex, h); - h = nvn_op_next(h); - } - vk_exec_barrier(ex); // the replay's next command buffer starts after the run, every image back in GENERAL - vk_sync_worker_end(ex->cb); - w->h = h; - x_cur = NULL; +static Chunk *chunk_open(const VkExec *ex, const op_hdr *h, int idx) { + Chunk *c = &G.c[idx]; + memcpy(&c->ex, ex, sizeof *ex); + c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx; + return c; } -// The submitting thread's side, at op h of the walk: when a run of RUN_MIN draws or more starts here, a worker records -// it and the walk goes on with the op that ended it, which is returned; otherwise h. -static const op_hdr *run_start(VkExec *ex, const op_hdr *h, int depth) { +// Records chunk c with the command pool `pool` (a worker's index; the submitting thread's is RW.n). +static void chunk_record(Chunk *c, int pool) { + if (c->idx) + while (!__atomic_load_n(&G.hooks, __ATOMIC_ACQUIRE)) plat_pause(); + VkExec *ex = &c->ex, *was = x_cur; + if (pool < RW.n) ex->dc = RW.w[pool]->dc; // the submitting thread records with its own (the state's) + x_cur = ex, rw_chunk = c, rw_ordered = !c->idx; + vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond); + ex->cb = vk_sync_worker_begin(pool); + // the replay's command buffer before the group ended behind a full barrier, with every image in GENERAL + ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL; + ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0, ex->fb_fs = NULL; + ex->views = NULL, ex->nviews = ex->cap_views = 0; + vk_exec_draw_new_cb(); + const op_hdr *o = c->h; + for (uint32_t k = 0; k < c->nops;) { + if (o->op == OP_JUMP) { + o = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(o))->target; + continue; + } + if (!op_linear(o->op)) vk_exec_brk++; + op_one(ex, o); + if (vk_texture_deferred_take()) { // a lookup would have made or taken back a texture entry: in turn, again + vk_exec_pass_end(ex); + ex->rt_resolved = 0, ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + vk_exec_turn(); + __atomic_add_fetch(&RW.redos, 1, __ATOMIC_RELAXED); + op_one(ex, o); + } + k++, o = nvn_op_next(o); + } + if (!c->idx) __atomic_store_n(&G.hooks, 1, __ATOMIC_RELEASE); // a chunk 0 that began no pass + vk_exec_barrier(ex); // the next command buffer starts after it, every image back in GENERAL + vk_sync_worker_end(ex->cb); + views_release(ex, ex->fence); + free(ex->views); + ex->views = NULL; + c->cb = ex->cb, c->recorded = ex->recorded; + vk_exec_xfer_worker(0, NULL, 0); + x_cur = was, rw_chunk = NULL; + __atomic_store_n(&c->done, 1, __ATOMIC_RELEASE); +} + +static void group_take(int pool) { + for (;;) { + int i = __atomic_fetch_add(&G.next, 1, __ATOMIC_ACQ_REL); + if (i >= __atomic_load_n(&G.n, __ATOMIC_ACQUIRE)) return; + chunk_record(&G.c[i], pool); + } +} + +// The submitting thread, at op h of the walk: when a group large enough to split starts here, it is recorded by the +// workers and this thread, and the walk goes on with the op that ended it, which is returned; otherwise h (the walk +// replays the group itself, without reading ahead again before its end). +static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { const op_hdr *end; - uint32_t draws = run_peek(h, &end); - if (draws < RUN_MIN || !vk_sync_append_room(1)) { - RW.skip_to[depth] = end, RW.short_runs += draws != 0; + uint32_t draws = group_peek(ex, h, &end); + if (end == h) return h; // an op the submitting thread replays itself + int n = (int)(draws / CHUNK_DRAWS); + if (n > RW.n + 1) n = RW.n + 1; + if (n > GROUP_CHUNKS) n = GROUP_CHUNKS; + if (n < 2 || vk_dump_armed || !vk_sync_append_room(n)) { + RW.skip_to[depth] = end, RW.serial_draws += draws; return h; } uint64_t t0 = kern_now_ns(); - vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL - RwWorker *w = RW.w[0]; - memcpy(&w->ex, ex, sizeof *ex); - w->h = h, w->depth = depth; - __atomic_store_n(&w->done, 0, __ATOMIC_RELAXED); - __atomic_store_n(&w->go, 1, __ATOMIC_SEQ_CST); - if (__atomic_load_n(&w->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&w->wake); - while (!__atomic_load_n(&w->done, __ATOMIC_ACQUIRE)) plat_pause(); - // the state the run left goes on in a new command buffer of the replay's, after the worker's in the part - int recorded = ex->recorded + w->ex.recorded; - VkCommandBuffer wcb = w->ex.cb; - struct DrawCtx *own = ex->dc; - memcpy(ex, &w->ex, sizeof *ex); - ex->dc = own; - ex->cb = vk_sync_append(&wcb, 1); - ex->recorded = ex->barrier_at = recorded; // the worker's command buffer ended with a full barrier - ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL, ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0; + vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL + vk_exec_draw_group_begin(ex); + // the front pass + uint32_t per = (draws + (uint32_t)n - 1) / (uint32_t)n, cd = 0; + int nc = 1, cut = 0; + G.ncond = 0; + Chunk *c = chunk_open(ex, h, 0); + for (const op_hdr *o = h; o != end;) { + const void *a = nvn_op_payload(o); + if (o->op == OP_JUMP) { + o = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; + continue; + } + if (cut) c = chunk_open(ex, o, nc++), cd = 0, cut = 0; + switch (o->op) { + case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: + if (++cd == per && nc < n) cut = 1; + break; + case OP_CLEAR_COLOR: case OP_CLEAR_DS: break; // recorded by the chunk + case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: { + int on = vk_exec_xfer_front(ex, o->op, a); + if (o->op == OP_RENDER_COND) { + if (G.ncond == G.ccond) { + G.ccond = G.ccond ? G.ccond * 2 : 1024; + if (!(G.cond = realloc(G.cond, G.ccond))) rt_fatal("vk: out of memory (render conditions)"); + } + G.cond[G.ncond++] = (uint8_t)on; + c->ncond++; + } + break; + } + default: op_one(ex, o); break; // state, binds, render targets + } + c->nops++; + o = nvn_op_next(o); + } + // handed out: the workers and this thread take the chunks in order + for (int i = 0; i < nc; i++) G.c[i].done = 0; + G.hooks = 0; + __atomic_store_n(&G.n, nc, __ATOMIC_RELEASE); + __atomic_store_n(&G.next, 0, __ATOMIC_RELEASE); + __atomic_add_fetch(&G.gen, 1, __ATOMIC_SEQ_CST); + for (int k = 0; k < RW.n; k++) + if (__atomic_load_n(&RW.w[k]->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&RW.w[k]->wake); + group_take(RW.n); + for (int i = 0; i < nc; i++) + while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause(); + __atomic_store_n(&G.next, 1 << 30, __ATOMIC_RELEASE); + // the state the front pass left goes on in a new command buffer of the replay's, after the chunks' in the part + VkCommandBuffer cbs[GROUP_CHUNKS]; + int rec = 0; + for (int i = 0; i < nc; i++) cbs[i] = G.c[i].cb, rec += G.c[i].recorded; + ex->cb = vk_sync_append(cbs, nc); + ex->recorded += rec, ex->barrier_at = ex->recorded; // the last chunk ended with a full barrier + ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL, ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0, ex->fb_fs = NULL; vk_exec_draw_new_cb(); - RW.runs++, RW.run_draws += draws, RW.wait_ns += kern_now_ns() - t0; - return w->h; + RW.groups++, RW.group_draws += draws, RW.chunks += (uint64_t)nc, RW.wait_ns += kern_now_ns() - t0; + return end; } static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { @@ -1868,9 +2011,9 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { for (;;) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); - if (RW.skip_to[depth] == h) RW.skip_to[depth] = NULL; // the short run read ahead is behind + if (RW.skip_to[depth] == h) RW.skip_to[depth] = NULL; // the small group read ahead is behind if (RW.n > 0 && !RW.skip_to[depth] && !evo_in && h->op != OP_JUMP && run_op(h->op)) { - const op_hdr *stop = run_start(ex, h, depth); + const op_hdr *stop = group_start(ex, h, depth); if (stop != h) { if (split_draws && split_wanted()) maybe_split(ex); h = stop; @@ -2034,14 +2177,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_draw_new_cb(); } if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory - if (ex->nviews) { - ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); - if (!l) rt_fatal("vk: out of memory"); - l->n = ex->nviews; - for (int i = 0; i < ex->nviews; i++) l->v[i] = ((ViewEnt *)ex->views)[i].view; - vk_completion_add(fence, free_views, l); - ex->nviews = 0; - } + views_release(ex, fence); ex->cb = VK_NULL_HANDLE; if (n) n_subs++; if ((rt_trace & RT_TRACE_PERF) && n_subs >= 240) { @@ -2065,12 +2201,18 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t char cbs[1600]; vk_sync_cb_stats(cbs, sizeof cbs); rt_log("perf: completion thread since the last line, by callback (distance from vk_completion_add):%s", cbs); - if (RW.n > 0) - rt_log("perf: replay workers: %.1f runs per submission of %.0f draws each, %.2f ms each with the hand-over; " - "%.1f shorter runs per submission stayed on the submitting thread", (double)RW.runs / (double)n_subs, - RW.runs ? (double)RW.run_draws / (double)RW.runs : 0.0, - RW.runs ? (double)RW.wait_ns / 1e6 / (double)RW.runs : 0.0, (double)RW.short_runs / (double)n_subs); - RW.runs = RW.run_draws = RW.short_runs = RW.wait_ns = 0; + if (RW.n > 0) { + uint64_t turns = __atomic_exchange_n(&RW.turns, 0, __ATOMIC_RELAXED); + uint64_t turn_ns = __atomic_exchange_n(&RW.turn_ns, 0, __ATOMIC_RELAXED); + uint64_t redos = __atomic_exchange_n(&RW.redos, 0, __ATOMIC_RELAXED); + rt_log("perf: replay workers: per submission %.1f groups of %.0f draws in %.1f chunks, %.2f ms each; %.0f draws " + "in smaller groups; waits for an earlier chunk %.1f (%.3f ms), ops replayed again in turn %.2f", + (double)RW.groups / (double)n_subs, RW.groups ? (double)RW.group_draws / (double)RW.groups : 0.0, + RW.groups ? (double)RW.chunks / (double)RW.groups : 0.0, + RW.groups ? (double)RW.wait_ns / 1e6 / (double)RW.groups : 0.0, (double)RW.serial_draws / (double)n_subs, + (double)turns / (double)n_subs, (double)turn_ns / 1e6 / (double)n_subs, (double)redos / (double)n_subs); + } + RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; n_subs = n_parts = n_idle_parts = n_pass_splits = 0; } diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 75d60777..bd507eb6 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -154,6 +154,16 @@ void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence); // record: vk_exec_draw.c DrawCtx), and the one the calling thread records with from now on (NULL: the replay's). void *vk_exec_draw_ctx_new(void); void vk_exec_draw_ctx_use(void *ctx); +// vk_exec_draw.c, before the chunks of a parallel group are recorded (submitting thread): the stand-ins are made and the +// texture pool set this submission took is brought up to date, which the chunks then only bind. +void vk_exec_draw_group_begin(VkExec *ex); +// vk_exec_xfer.c, parallel groups: vk_exec_xfer_serial says an op stays on the submitting thread (a samples-passed reset +// or report); vk_exec_xfer_front is the front pass's CPU side of a counter report, render enable or render condition +// (returns the condition's decision, 1 renders); vk_exec_xfer_worker turns the worker side on for the calling thread with +// its chunk's decisions, or off. +int vk_exec_xfer_serial(uint32_t op, const void *payload); +int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload); +void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond); // vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex // buffers): the replay continues in a new one (vk_sync_split). void vk_exec_draw_new_cb(void); @@ -302,6 +312,7 @@ void vk_aa_composed(int ok); // vk_contact.c: a pass about to begin (its targets resolved, no pass open), and a dispatch or copy about to be recorded void vk_contact_pass_begin(VkExec *ex); void vk_contact_flush(VkExec *ex); +int vk_contact_pending(void); // a contact shadows' pass waits for the next transfer outside a pass // vk_exec_xfer.c, samples-passed counting: a draw about to be recorded in the open pass (begins an occlusion query // when counting and none is active), and the open pass about to end (ends the active query). void vk_exec_oq_draw(VkExec *ex); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 760337f3..bd91c049 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -275,6 +275,7 @@ typedef struct DrawCtx { VkDescriptorBufferInfo mo_ovr[2][32]; // [vertex, fragment][slot]; buffer 0: not replaced struct { int valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM; // the shader memo (shader_memo_on) struct { int valid; VkPipeState s; const VkPipeInfo *info; } PM; // the pipeline memo (pipe_memo_on) + void *fm; // feedback_scan's memo per texture handle (FmEnt[FM_N]), made at first use VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views VkDescSlot bind_d[MAX_DESC]; VkBufferView bind_bv[MAX_DESC]; @@ -291,6 +292,10 @@ void *vk_exec_draw_ctx_new(void) { void vk_exec_draw_ctx_use(void *c) { dc = c ? c : &dc_main; } +static PlatMutex shader_lock = PLAT_MUTEX_INIT; // shader_tab: the replay workers look shaders up at once +static PlatMutex rs_lock = PLAT_MUTEX_INIT; // rs_lists +static PlatMutex pool_mt = PLAT_MUTEX_INIT; // pool_sync from a chunk of a parallel group + static void cb_done(void *arg) { // completion thread CbRes *r = arg; VkCtx *c = vk_ctx(); @@ -1066,6 +1071,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded set = shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)]; + plat_mutex_lock(&shader_lock); for (int w = 0; w < SHADER_WAYS && set[w].used; w++) { ShaderEnt *e = &set[w]; if (e->serial != serial || e->stage != st || memcmp(&e->a, &a, sizeof a)) continue; @@ -1074,8 +1080,11 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin memmove(&set[1], &set[0], w * sizeof *set); set[0] = hit; } - return set[0].s; + const GfxShader *found = set[0].s; + plat_mutex_unlock(&shader_lock); + return found; } + plat_mutex_unlock(&shader_lock); } int gs = st == NVN_STAGE_VS ? GFX_STAGE_VS : st == NVN_STAGE_FS ? GFX_STAGE_FS : st == NVN_STAGE_TASK ? GFX_STAGE_TS : st == NVN_STAGE_MESH ? GFX_STAGE_MS : GFX_STAGE_CS; @@ -1089,8 +1098,10 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st, (unsigned long long)serial); else if (set) { // in front, the least recently used way drops out + plat_mutex_lock(&shader_lock); memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set); set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s}; + plat_mutex_unlock(&shader_lock); } return s; } @@ -1607,7 +1618,7 @@ static int td_mode(void) { // texture bindings change the whole-scan memo below, but the handles they bind were mostly looked up before, in this // pass or an earlier one. Direct-mapped. #define FM_N 8192 -static struct { uint64_t h, tok; uint32_t gen; SubRange r; } FM[FM_N]; +typedef struct { uint64_t h, tok; uint32_t gen; SubRange r; } FmEnt; // DrawCtx.fm: one table per recording thread static SubRange sampled_sub(uint64_t h) { // texture lock held view_t v; @@ -1624,6 +1635,8 @@ static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) { int n = gfx_shader_resources(fs, GFX_RES_TEX, &res); uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup int memo = td_mode(), locked = 0; + FmEnt *fm = ex->dc->fm; + if (!fm && !(fm = ex->dc->fm = calloc(FM_N, sizeof *fm))) rt_fatal("vk: out of memory (feedback memo)"); for (int r = 0; r < n; r++) { // Only bound slots (0..127) are looked at. The texture and sampler pool arrays (res.pool: 32768 and 4096 // descriptors, bindless handles) have slot -1: walking them ran up to about 100,000 empty iterations per scan, @@ -1635,7 +1648,7 @@ static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) { int slot = res[r].slot + k; uint64_t h = ex->tex[NVN_STAGE_FS][slot]; if (!h) continue; - typeof(FM[0]) *f = &FM[(h * 0x9E3779B97F4A7C15ull) >> 51]; + FmEnt *f = &fm[(h * 0x9E3779B97F4A7C15ull) >> 51]; if (memo && f->h == h && slot_still(&f->gen, f->tok, h, gen)) { if (memo == 2) { if (!locked) vk_texture_lock(), locked = 1; @@ -2439,6 +2452,26 @@ static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) { static void pool_sync_entries(VkExec *ex); +static void pool_sync(VkExec *ex); + +// A chunk of a parallel group (vk_exec.c "replay workers") binds the set as vk_exec_draw_group_begin left it, up to date +// when the group began; the chunk that takes this submission's first set takes it in turn, under pool_mt. Elsewhere the +// replay syncs the pools at every draw that binds them, as before. +static void pool_sync_draw(VkExec *ex) { + if (!vk_exec_in_chunk()) return pool_sync(ex); + if (__atomic_load_n(&PL.cur_fence, __ATOMIC_ACQUIRE) == ex->fence) return; + vk_exec_turn(); + plat_mutex_lock(&pool_mt); + pool_sync(ex); + plat_mutex_unlock(&pool_mt); +} + +void vk_exec_draw_group_begin(VkExec *ex) { + dummies_init(); + if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); + if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) pool_sync(ex); // the set this submission took +} + static void pool_sync(VkExec *ex) { if (!PL.layout) vk_pool_set_layout(); uint64_t from = 0, pre = PL.logn; @@ -3066,6 +3099,7 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { DrawCtx *const cx = ex->dc; if (info == cx->rs_last) return cx->rs_last_l; + plat_mutex_lock(&rs_lock); if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info); RsList *l = nvn_hm_get(&rs_lists, &info); if (!l) { @@ -3115,6 +3149,7 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { memcpy(l->r, tmp, sizeof *tmp * n); nvn_hm_put(&rs_lists, &info, l); } + plat_mutex_unlock(&rs_lock); cx->rs_last = info, cx->rs_last_l = l; return l; } @@ -3501,7 +3536,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } } if (info->pool) { // set 2: the texture and sampler pools - pool_sync(ex); + pool_sync_draw(ex); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 2, 1, &PL.set, 0, NULL); cx->RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again } @@ -4193,11 +4228,13 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int vk_exec_frame.bind_ns += (uint64_t)(bt_now() - bt); vk_exec_frame.bind_calls++; } + if (vk_texture_deferred()) return 0; // a chunk's lookup waits for its turn: the op is replayed again if (fbl & ex->fb_dirty) feedback_barrier(ex); feedback_written(ex, &s, fbl); if (!mesh) bind_vertex_buffers(ex); if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh && !fb); if (vk_hdr_track) vk_hdr_draw(ex, info); + if (vk_texture_deferred()) return 0; vk_exec_oq_draw(ex); DRS.issued[mesh]++; draw_stats_tick(); @@ -4742,6 +4779,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi int nu = gfx_shader_resources(cx->B.st.vs, GFX_RES_UBO, &ur), reads = 0; for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4; if (!reads) return; + vk_exec_turn(); // a replay worker's chunk: the motion list follows the stream MO.cand++; uint64_t tp = t0 ? kern_now_ns() : 0; if (t0) MO.ns_part[0] += tp - t0, MO.n_part[0]++; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 40dc3d6a..c88c8e5e 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -222,6 +222,18 @@ static uint64_t oq_count; // completion thread: samples since the typedef struct { uint64_t addr, seq; int reset, lost; uint32_t n; uint32_t q[]; } OqSegment; +// The replay workers (vk_exec.c "parallel groups") record draws of one group at once. A samples-passed reset or report +// never falls inside such a group (vk_exec_xfer_serial), so counting does not change while the workers record: each +// keeps the query open in its own pass (XW.open), takes ring slots atomically and adds the slots it closes to the +// segment the next report takes, under oq_seg_lock. XW.cond: the decisions of its chunk's render conditions, made by the +// front pass in the order of the stream (vk_exec_xfer_front). +static _Thread_local struct { int on, open; const uint8_t *cond; uint32_t ncond, i; } XW = {.open = -1}; +static PlatMutex oq_seg_lock = PLAT_MUTEX_INIT; + +void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond) { + XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0; +} + static int oq_ready(void) { if (oq_state) return oq_state > 0; VkCtx *v = vk_ctx(); @@ -248,31 +260,36 @@ static int oq_ready(void) { } static void oq_close(VkExec *ex) { - if (oq_open < 0) return; - vkCmdEndQuery(ex->cb, oq_pool, (uint32_t)oq_open); + int *open = XW.on ? &XW.open : &oq_open; + if (*open < 0) return; + vkCmdEndQuery(ex->cb, oq_pool, (uint32_t)*open); ex->recorded++; + if (XW.on) plat_mutex_lock(&oq_seg_lock); if (oq_nseg == oq_capseg) { oq_capseg = oq_capseg ? oq_capseg * 2 : 64; oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg); if (!oq_seg) rt_fatal("vk: out of memory"); } - oq_seg[oq_nseg++] = (uint32_t)oq_open; - oq_open = -1; + oq_seg[oq_nseg++] = (uint32_t)*open; + if (XW.on) plat_mutex_unlock(&oq_seg_lock); + *open = -1; } void vk_exec_oq_draw(VkExec *ex) { - if (!oq_counting || oq_open >= 0) return; - uint32_t q = oq_next % OQ_N; + int *open = XW.on ? &XW.open : &oq_open; + if (!oq_counting || *open >= 0) return; + // a worker takes its slot at once (a busy one is lost, not tried again): the others take theirs meanwhile + uint32_t q = (XW.on ? __atomic_fetch_add(&oq_next, 1, __ATOMIC_RELAXED) : oq_next) % OQ_N; if (__atomic_load_n(&oq_busy[q], __ATOMIC_ACQUIRE)) { warn_once(W_QUERY, "occlusion query ring full: draws left uncounted"); - oq_lost = 1; + __atomic_store_n(&oq_lost, 1, __ATOMIC_RELAXED); return; } - oq_busy[q] = 1; - oq_next++; + __atomic_store_n(&oq_busy[q], 1, __ATOMIC_RELAXED); + if (!XW.on) oq_next++; vkCmdBeginQuery(ex->cb, oq_pool, q, oq_flags); ex->recorded++; - oq_open = (int)q; + *open = (int)q; } void vk_exec_oq_pass_end(VkExec *ex) { oq_close(ex); } @@ -986,7 +1003,55 @@ static void clear_tex(VkExec *ex, const p_clear_tex *p) { vk_texture_gpu_written(t); } +// ---- parallel groups (vk_exec.c): the CPU side in the front pass, the rest in the workers + +// 1 when the op has to be replayed on the submitting thread in the order of the stream: a samples-passed reset or report +// ends a segment of occlusion counts, which the workers' queries are added to. +int vk_exec_xfer_serial(uint32_t op, const void *payload) { + int type; + if (op == OP_REPORT_COUNTER) type = (int)((const p_report_counter *)payload)->type; + else if (op == OP_RESET_COUNTER) type = (int)((const p_reset_counter *)payload)->type; + else return 0; + return type == NVN_COUNTER_TYPE_SAMPLES_PASSED && oq_ready(); +} + +// The front pass, in the order of the stream, before the workers record: a counter report other than samples passed +// (its completion callback and pending entry, which later conditions read), a render enable and a render condition +// (decided here, on the reports replayed so far). Returns the condition's decision: 1 renders. +int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload) { + switch (op) { + case OP_REPORT_COUNTER: report_counter(fx, payload); return 1; // not samples passed (vk_exec_xfer_serial) + case OP_RESET_COUNTER: return 1; // other counters have nothing to reset + case OP_RENDER_ENABLE: fx->render_disabled = !((const p_flag *)payload)->value; return 1; + case OP_RENDER_COND: + fx->render_cond = *(const p_render_cond *)payload; + fx->has_render_cond = 1; + fx->render_disabled = !cond_enabled(payload); + return !fx->render_disabled; + default: rt_fatal("vk: op %s has no front pass", nvn_op_name(op)); + } + return 1; +} + +// A worker: what the front pass left, in its chunk. The contact shadows' pass (vk_contact_flush) is recorded where +// the replay would, once every earlier chunk of the group is done. +static void xfer_worker(VkExec *ex, uint32_t op, const void *payload) { + if (op != OP_RENDER_ENABLE && vk_contact_pending()) vk_exec_turn(), vk_contact_flush(ex); + switch (op) { + case OP_REPORT_COUNTER: case OP_RESET_COUNTER: break; + case OP_RENDER_ENABLE: ex->render_disabled = !((const p_flag *)payload)->value; break; + case OP_RENDER_COND: + ex->render_cond = *(const p_render_cond *)payload; + ex->has_render_cond = 1; + if (XW.i >= XW.ncond) rt_fatal("vk: a replay worker met more render conditions than its front pass decided"); + ex->render_disabled = !XW.cond[XW.i++]; + break; + default: rt_fatal("vk: op %s in a parallel group", nvn_op_name(op)); + } +} + static void op_xfer_impl(VkExec *ex, uint32_t op, const void *payload) { + if (XW.on) return xfer_worker(ex, op, payload); if (op != OP_RENDER_ENABLE && op != OP_SIGNAL_EVENT) vk_contact_flush(ex); switch (op) { case OP_COPY_B2B: copy_b2b(ex, payload); break; diff --git a/runtime/vk_hdr.c b/runtime/vk_hdr.c index 402bea39..108d552b 100644 --- a/runtime/vk_hdr.c +++ b/runtime/vk_hdr.c @@ -773,6 +773,7 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) { for (int i = 0; i < 4; i++) // unlocked first look: most draws go elsewhere (a stale slot only costs the lock below) known |= __atomic_load_n(&recs[i].win, __ATOMIC_RELAXED) == t->image; if (!known) return; + vk_exec_turn(); // a replay worker's chunk: the last such draw of the stream decides, as in the replay VkImage src = VK_NULL_HANDLE; VkFormat fmt = VK_FORMAT_UNDEFINED; int ntex = 0, slot = -1; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 326418b4..2d09aa5f 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2560,7 +2560,24 @@ void vk_mem_shadow_sync(uint64_t fence) { plat_mutex_unlock(&shadow_lock); } +// The replay workers (vk_exec.c) bind and write from several threads at once: a mirror's first use in a submission +// (creation, sync) and the notes of GPU writes go under use_lock. A use the submission already synced takes no lock. +static PlatMutex use_lock = PLAT_MUTEX_INIT; + +static void mem_use_locked(VkMemSpan *sp); void vk_mem_use(VkMemSpan *sp) { + PoolRec *r = sp->pool; + if (r->mstate == MS_ON && r->acc_sub == msub) { + r->vd_use_sub = msub; + sp->buffer = r->mbuf; + return; + } + plat_mutex_lock(&use_lock); + mem_use_locked(sp); + plat_mutex_unlock(&use_lock); +} + +static void mem_use_locked(VkMemSpan *sp) { PoolRec *r = sp->pool; if (r->mstate == MS_REFUSED && r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0 && evict_enabled()) { plat_dec64(&m_refused_n[r->refused_why]); // counted again if refused again @@ -2578,7 +2595,11 @@ void vk_mem_use(VkMemSpan *sp) { void vk_mem_pick(VkMemSpan *sp) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON) return; - if (r->acc_sub != msub) mirror_sync(r); + if (r->acc_sub != msub) { + plat_mutex_lock(&use_lock); + if (r->acc_sub != msub) mirror_sync(r); + plat_mutex_unlock(&use_lock); + } sp->buffer = r->mbuf; } @@ -3195,8 +3216,16 @@ static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) { r->st_wb += size - o; range_add(&wbl, &nwbl, &cwbl, r, sp->offset + o, size - o); // what found no snapshot room } -void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 0); } -void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 1); } +void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { + plat_mutex_lock(&use_lock); + gpu_wrote(sp, size, 0); + plat_mutex_unlock(&use_lock); +} +void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { + plat_mutex_lock(&use_lock); + gpu_wrote(sp, size, 1); + plat_mutex_unlock(&use_lock); +} // Adds [a, b) to l, n sorted ranges that do not touch; returns the new count. Above cap ranges (l holds cap + 1) the two // nearest merge, so a full list copies the fewest bytes that were not flushed; cap 1 keeps one span over all flushes. diff --git a/runtime/vk_mvprev.c b/runtime/vk_mvprev.c index ec1ad618..f2914b39 100644 --- a/runtime/vk_mvprev.c +++ b/runtime/vk_mvprev.c @@ -25,6 +25,7 @@ #include "cfg.h" #include "host.h" +#include "platform/platform.h" #include "vk.h" #include "vk_exec.h" #include "vk_dlaa.h" @@ -131,9 +132,18 @@ void vk_mvprev_draw(VkExec *ex) { P.ns_draw += kern_now_ns() - t0; } +static void mvprev_draw_locked(VkExec *ex); +static PlatMutex draw_lock = PLAT_MUTEX_INIT; // the replay workers' scene draws note blocks at once + static void mvprev_draw_(VkExec *ex) { const VkExecTarget *z = &ex->depth; if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return; + plat_mutex_lock(&draw_lock); + mvprev_draw_locked(ex); + plat_mutex_unlock(&draw_lock); +} + +static void mvprev_draw_locked(VkExec *ex) { int check = P.bytes[0] != NULL; // NX2NAVI_MVPREV_CHECK=1: the summary's checks (3.6 ms a frame) for (int index = 0; index < BANKS; index++) { if (!(P.mask >> index & 1)) continue; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 0e493eb2..9b7414a8 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -3618,10 +3618,25 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { start_line(e, NULL, gone_desc, gone_rel, e->og_start == START_COPY_GONE ? 2 : filled ? 3 : copied); } +// A replay worker's chunk before its turn (vk_exec.c "replay workers"): a lookup that would make or take back an entry +// returns nothing and sets this; the chunk replays the op again once every earlier chunk is done, so entries are made, +// initialised and taken back in the order of the stream, as the replay would. +static _Thread_local int tex_deferred; +int vk_texture_deferred(void) { return tex_deferred; } +int vk_texture_deferred_take(void) { + int d = tex_deferred; + tex_deferred = 0; + return d; +} + static Tex *entry_of(const tex_t *t) { TexKey k; make_key(t, &k); Tex *e = nvn_hm_get(&by_key, &k); + if ((!e || e->retiring || e->released) && vk_exec_turn_due()) { + tex_deferred = 1; + return NULL; + } if (e && e->retiring) { // retired by disuse: the pool refresh leaves it out, any other lookup takes it back if (pool_ctx) return NULL; retire_cancel(e); diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index 280aa9f4..b59d9b98 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -417,7 +417,7 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI if (!U.on || !n) return 0; // Consecutive bindings mostly name the region the last one did: that one is tried before the binary search. The // regions are sorted and disjoint, so the region that holds addr is the one first_above finds. - static int hint; + static _Thread_local int hint; // per thread: the replay workers look up at once int i = hint; if (i >= U.nreg || addr < U.reg[i].lo || addr >= U.reg[i].hi) i = first_above(addr); if (i >= U.nreg) return 0; @@ -431,13 +431,17 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI return 1; } +static PlatMutex drop_lock = PLAT_MUTEX_INIT; // the replay workers' shaders write storage at once + void vk_usnap_drop(uint64_t addr, uint64_t n) { if (!U.mode || !n) return; + plat_mutex_lock(&drop_lock); if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr); U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence}; - if (!U.on) return; - for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++) - if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, U.gen++; + if (U.on) + for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++) + if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, __atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED); + plat_mutex_unlock(&drop_lock); } uint64_t vk_usnap_gen(void) { return U.gen; }