From 9fe3dd33310dc70b013cf08657b937d761e1233b Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:37:50 -0500 Subject: [PATCH] Replay workers record mesh draws, and stop waiting for each other at queries and per-object motion. Mesh draws ended a group and stayed on the submitting thread: 119 a frame in Hyrule Field at about 13 us each, and the groups around them were cut short. They are recorded in the chunks now: the entries filled ahead for the indirect ones are the draw context's, the ring index is taken atomically and the ring is made before the chunks record, each chunk counts the ops that break a run of them on its own, and the lazy lookups of the mesh and shading-rate entry points store their pointer before the flag that says it is there. With the mesh fallback, and while NX2NAVI_GPU_PASSES copies the workgroup counts, they stay on the submitting thread. A chunk's closed occlusion query slots go into a list of its own and into their segments once the group is done, in chunk order, instead of under a lock per query. The per-object motion check decides alone per draw context (its variant memo is per context) and takes its lock only to list a draw. The occlusion query ring has 16384 slots: the chunks' blocks are sized to their draws, and with 4096 the ring came round to slots not reset yet. The perf line splits the submitting thread's time with workers on: ops outside groups by kind, the front pass, its own chunks and the wait for the others, and which ops ended the groups. Hyrule Field, RX 7900 XT, 3440x1440, FSR 4 Native AA, same batch: standing 51.7 fps with 3 workers against 43.6 with the workers off (replay 11.2 against 15.1 ms); along the field route 48.9 against 44.7. Screenshots standing and along the route match the replay without workers. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec.c | 108 +++++++++++++++++++++----- runtime/vk_exec.h | 7 +- runtime/vk_exec_draw.c | 172 ++++++++++++++++++++++++++--------------- runtime/vk_exec_xfer.c | 67 ++++++++++------ runtime/vk_mvprev.c | 2 +- 5 files changed, 249 insertions(+), 107 deletions(-) diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 3391129e..a75bdca0 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -57,6 +57,7 @@ typedef struct { uint32_t cond0, ncond; // its render conditions' decisions in G.cond uint64_t oq; // the occlusion counting at its first op (vk_exec_xfer_chunk_state) uint32_t slot0, nslots; // its block of occlusion query slots (vk_exec_xfer_slots) + uint32_t brk; // its ops that are not state or a draw (vk_exec_brk_now) int idx; // in its group: 0 runs the group's pass hooks VkCommandBuffer cb; int recorded; @@ -72,6 +73,7 @@ static struct { volatile uint32_t gen; // bumped when a group is handed out } G = {.next = 1 << 30}; static _Thread_local Chunk *rw_chunk; // the chunk the calling thread records, NULL outside one +static _Thread_local const op_hdr *rw_op; // in a chunk: the op it records (vk_exec_mesh_run reads on from it) static _Thread_local int rw_ordered; // every earlier chunk of its group is done static PlatMutex tfr_lock = PLAT_MUTEX_INIT; // target frames (vk_target_frames.h) noted by several chunks at once @@ -1628,8 +1630,10 @@ void vk_exec_mesh_run_stops(char *out, size_t n) { // The indirect mesh draws that follow the op being replayed (cur: its payload) with only ops of op_linear between // them, MESH_RUN_OPS at most: their payloads into out, at most max. 0 when the walk is not at cur. +uint32_t vk_exec_brk_now(void) { return rw_chunk ? rw_chunk->brk : vk_exec_brk; } + int vk_exec_mesh_run(const void *cur, const void **out, int max) { - const op_hdr *h = AH.at[AH.depth]; + const op_hdr *h = rw_chunk ? rw_op : AH.at[AH.depth]; // a chunk reads on from its own op (into the next ones) if (!h || nvn_op_payload(h) != cur) return 0; int n = 0; h = nvn_op_next(h); @@ -1729,9 +1733,9 @@ static void op_one(VkExec *ex, const op_hdr *h) { // 2026-11-11): about 3,500 draws a frame in 82 runs, 96% of them in runs of 16 draws or more and 67% in runs of 256 or // more, with a timestamp report beside nearly every draw and a render condition beside every second one. // A run is cut into groups: a group ends where the render targets change after its first draw or clear, and before an -// inline uniform update, a mesh draw and a samples-passed counter op, which the submitting thread replays itself in -// the order of the stream like everything outside groups. A group of 2 * CHUNK_DRAWS draws or more is recorded by -// several threads at once: +// inline uniform update (and a mesh draw with the mesh fallback or while NX2NAVI_GPU_PASSES copies their counts), which +// the submitting thread replays itself in the order of the stream like everything outside groups. A group of +// 2 * CHUNK_DRAWS draws or more is recorded by several threads at once: // - the front pass, on the submitting thread, walks the group in the order of the stream before anything of it is // recorded: it applies the state and binds, settles the CPU side of counter reports and decides the render conditions // (vk_exec_xfer_front), and copies the replay state at the first op of each chunk; @@ -1743,7 +1747,9 @@ static void op_one(VkExec *ex, const op_hdr *h) { // which reaches the group's pass first, and the others start after that pass began; groups follow one another. // What still depends on the order between chunks waits for its turn (vk_exec_turn: every earlier chunk of the group // done): a texture lookup that would make or take back an entry (vk_texture.c entry_of) returns nothing and the chunk -// replays the op again once in turn, and so do the contact shadows' pass, per-object motion and the HDR pass's source. +// replays the op again once in turn, and so do the contact shadows' pass and the HDR pass's source. Per-object motion +// lists its draws under a lock (vk_exec_draw.c mvobj_draw), and the occlusion queries' slots and segments follow the +// stream (vk_exec_xfer.c vk_exec_xfer_slots, vk_exec_xfer_group_merge). #define RW_MAX 16 #define CHUNK_DRAWS 32 // draws a chunk takes at least #define GROUP_PEEK 65536u // ops read ahead for a group, at most @@ -1777,11 +1783,26 @@ static struct { const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small uint64_t groups, group_draws, chunks, serial_draws, wait_ns; // since the last perf line uint64_t end_draws[5]; // draws of groups by what ended them (group_peek's GE_*), since the last perf line + uint64_t small_end[5]; // ... of those too small to split + uint64_t run_end[OP_COUNT_]; // draws of groups that an op outside groups ended, by that op + // the submitting thread, since the last perf line: ops outside groups by kind (SER_*), and in groups the front + // pass, its own chunks and the wait for the others' + uint64_t ser_n[4], ser_ns[4], front_ns, own_ns, spin_ns; volatile uint64_t turns, turn_ns, redos; } RW = {.n = -1}; static void group_take(int pool); +enum { SER_DRAW, SER_MESH, SER_DISPATCH, SER_OTHER }; +static int ser_kind(uint32_t op) { + switch (op) { + case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: return SER_DRAW; + case OP_DRAW_MESH: case OP_DRAW_MESH_IND: return SER_MESH; + case OP_DISPATCH: case OP_DISPATCH_IND: return SER_DISPATCH; + default: return SER_OTHER; + } +} + static void rw_main(void *arg) { RwWorker *w = arg; plat_thread_priority(PLAT_PRIO_HIGHEST); // as the recording thread (rec_claim) @@ -1852,12 +1873,18 @@ static void views_release(VkExec *ex, uint64_t fence) { enum { GE_RUN, GE_TARGET, GE_UB, GE_MESH, GE_LIMIT, GE_N }; static const char *const ge_name[GE_N] = {"run end", "target change", "uniform update", "mesh draw", "read limit"}; +int vk_gpu_passes_capturing(void); // vk_crumbs.c: NX2NAVI_GPU_PASSES is copying this frame's counts + // The draws of the group that starts at h (it ends at *end, for the reason *why), reading at most GROUP_PEEK ops. -static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end, int *why) { +static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end, int *why, uint32_t *end_op) { uint32_t draws = 0; + *end_op = 0; int begun = 0; const p_set_rt *rt = &ex->rt; *why = GE_LIMIT; + // mesh draws stay on the submitting thread with the mesh fallback (its buffers are shared) and while + // NX2NAVI_GPU_PASSES copies their workgroup counts (one list) + int mesh_serial = vk_ctx()->mesh_fallback || vk_gpu_passes_capturing(); for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) { *why = GE_RUN; break; } // the walk reports it const void *a = nvn_op_payload(h); @@ -1865,14 +1892,15 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; } - if (!run_op(h->op)) { *why = GE_RUN; break; } + if (!run_op(h->op)) { *why = GE_RUN, *end_op = h->op; break; } if (h->op == OP_UPDATE_UB) { *why = GE_UB; break; } - if (h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND) { *why = GE_MESH; break; } + int mesh = h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND; + if (mesh && mesh_serial) { *why = GE_MESH; break; } if (h->op == OP_SET_RT && memcmp(rt, a, sizeof *rt)) { if (begun) { *why = GE_TARGET; break; } // another pass: the next group rt = a; } - if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) draws++, begun = 1; + if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND || mesh) draws++, begun = 1; if (h->op == OP_CLEAR_COLOR || h->op == OP_CLEAR_DS) begun = 1; h = nvn_op_next(h); } @@ -1884,7 +1912,7 @@ static Chunk *chunk_open(const VkExec *ex, const op_hdr *h, int idx) { Chunk *c = &G.c[idx]; memcpy(&c->ex, ex, sizeof *ex); c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx, c->oq = vk_exec_xfer_chunk_state(); - c->nslots = 0; + c->nslots = 0, c->brk = 0; return c; } @@ -1895,7 +1923,7 @@ static void chunk_record(Chunk *c, int pool) { VkExec *ex = &c->ex, *was = x_cur; if (pool < RW.n) ex->dc = RW.w[pool]->dc; // the submitting thread records with its own (the state's) x_cur = ex, rw_chunk = c, rw_ordered = !c->idx; - vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond, c->oq, c->slot0, c->nslots); + vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond, c->oq, c->slot0, c->nslots, c->idx); ex->cb = vk_sync_worker_begin(pool); // the replay's command buffer before the group ended behind a full barrier, with every image in GENERAL ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL; @@ -1908,7 +1936,8 @@ static void chunk_record(Chunk *c, int pool) { o = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(o))->target; continue; } - if (!op_linear(o->op)) vk_exec_brk++; + if (!op_linear(o->op)) c->brk++; + rw_op = o; op_one(ex, o); if (vk_texture_deferred_take()) { // a lookup would have made or taken back a texture entry: in turn, again vk_exec_pass_end(ex); @@ -1927,7 +1956,7 @@ static void chunk_record(Chunk *c, int pool) { free(ex->views); ex->views = NULL; c->cb = ex->cb, c->recorded = ex->recorded; - vk_exec_xfer_worker(0, NULL, 0, 0, 0, 0); + vk_exec_xfer_worker(0, NULL, 0, 0, 0, 0, 0); x_cur = was, rw_chunk = NULL; __atomic_store_n(&c->done, 1, __ATOMIC_RELEASE); } @@ -1946,14 +1975,16 @@ static void group_take(int pool) { static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { const op_hdr *end; int why; - uint32_t draws = group_peek(ex, h, &end, &why); + uint32_t end_op; + uint32_t draws = group_peek(ex, h, &end, &why, &end_op); + if (why == GE_RUN && end_op < OP_COUNT_) RW.run_end[end_op] += draws; if (end == h) return h; // an op the submitting thread replays itself RW.end_draws[why] += draws; int n = (int)(draws / CHUNK_DRAWS); if (n > RW.n + 1) n = RW.n + 1; if (n > GROUP_CHUNKS) n = GROUP_CHUNKS; if (n < 2 || vk_dump_armed || !vk_sync_append_room(n)) { - RW.skip_to[depth] = end, RW.serial_draws += draws; + RW.skip_to[depth] = end, RW.serial_draws += draws, RW.small_end[why] += draws; return h; } uint64_t t0 = kern_now_ns(); @@ -1974,7 +2005,7 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { } if (cut) c = chunk_open(ex, o, nc++), cd = 0, cut = 0; switch (o->op) { - case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: + case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: case OP_DRAW_MESH: case OP_DRAW_MESH_IND: if (vk_exec_xfer_counting() && !ex->render_disabled) c->nslots++; // at most one query each if (++cd == per && nc < n) cut = 1; break; @@ -2003,12 +2034,17 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { G.hooks = 0; __atomic_store_n(&G.n, nc, __ATOMIC_RELEASE); __atomic_store_n(&G.next, 0, __ATOMIC_RELEASE); + uint64_t tf = kern_now_ns(); __atomic_add_fetch(&G.gen, 1, __ATOMIC_SEQ_CST); for (int k = 0; k < RW.n; k++) if (__atomic_load_n(&RW.w[k]->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&RW.w[k]->wake); group_take(RW.n); + uint64_t to = kern_now_ns(); for (int i = 0; i < nc; i++) while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause(); + uint64_t tw = kern_now_ns(); + RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.spin_ns += tw - to; + vk_exec_xfer_group_merge(nc); // the chunks' closed query slots into their segments __atomic_store_n(&G.next, 1 << 30, __ATOMIC_RELEASE); // the state the front pass left goes on in a new command buffer of the replay's, after the chunks' in the part VkCommandBuffer cbs[GROUP_CHUNKS]; @@ -2057,7 +2093,14 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { } break; } - default: op_one(ex, h); break; + default: + if (RW.n > 0) { // the perf line's split of the replay outside groups + int k = ser_kind(h->op); + uint64_t t = kern_now_ns(); + op_one(ex, h); + RW.ser_n[k]++, RW.ser_ns[k] += kern_now_ns() - t; + } else op_one(ex, h); + break; } if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); if (split_draws && split_wanted()) maybe_split(ex); @@ -2229,13 +2272,40 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t RW.groups ? (double)RW.chunks / (double)RW.groups : 0.0, RW.groups ? (double)RW.wait_ns / 1e6 / (double)RW.groups : 0.0, (double)RW.serial_draws / (double)n_subs, (double)turns / (double)n_subs, (double)turn_ns / 1e6 / (double)n_subs, (double)redos / (double)n_subs); - char e[256]; + char e[512]; int o = 0; for (int k = 0; k < GE_N; k++) o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.0f", k ? "," : "", ge_name[k], (double)RW.end_draws[k] / (double)n_subs); + o += snprintf(e + o, sizeof e - (size_t)o, "; of them in groups too small to split:"); + for (int k = 0; k < GE_N; k++) + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.0f", k ? "," : "", ge_name[k], + (double)RW.small_end[k] / (double)n_subs); rt_log("perf: replay workers: draws per submission by what ended their group:%s", e); - memset(RW.end_draws, 0, sizeof RW.end_draws); + o = 0; + for (int k = 0; k < 6; k++) { // the ops outside groups that ended the most, by their groups' draws + int best = 0; + for (int op = 1; op < OP_COUNT_; op++) + if (RW.run_end[op] > RW.run_end[best]) best = op; + if (!RW.run_end[best]) break; + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.0f", k ? "," : "", nvn_op_name((uint32_t)best), + (double)RW.run_end[best] / (double)n_subs); + RW.run_end[best] = 0; + } + memset(RW.run_end, 0, sizeof RW.run_end); + if (o) rt_log("perf: replay workers: draws per submission of groups a run end ended, by the op:%s", e); + memset(RW.end_draws, 0, sizeof RW.end_draws), memset(RW.small_end, 0, sizeof RW.small_end); + double ns = 1e6 * (double)n_subs; + rt_log("perf: replay workers: the submitting thread per submission: outside groups draws %.0f (%.3f ms), mesh " + "draws %.0f (%.3f ms), dispatches %.0f (%.3f ms), other ops %.0f (%.3f ms); in groups the front pass " + "%.3f ms, its own chunks %.3f ms, waiting for the others %.3f ms", + (double)RW.ser_n[SER_DRAW] / (double)n_subs, (double)RW.ser_ns[SER_DRAW] / ns, + (double)RW.ser_n[SER_MESH] / (double)n_subs, (double)RW.ser_ns[SER_MESH] / ns, + (double)RW.ser_n[SER_DISPATCH] / (double)n_subs, (double)RW.ser_ns[SER_DISPATCH] / ns, + (double)RW.ser_n[SER_OTHER] / (double)n_subs, (double)RW.ser_ns[SER_OTHER] / ns, + (double)RW.front_ns / ns, (double)RW.own_ns / ns, (double)RW.spin_ns / ns); + memset(RW.ser_n, 0, sizeof RW.ser_n), memset(RW.ser_ns, 0, sizeof RW.ser_ns); + RW.front_ns = RW.own_ns = RW.spin_ns = 0; } RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index e3696117..de4dd75d 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -164,7 +164,11 @@ void vk_exec_draw_group_begin(VkExec *ex); void vk_exec_xfer_group(int on); int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload); uint64_t vk_exec_xfer_chunk_state(void); -void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots); +void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots, + int chunk); +// After every chunk of the group is done (submitting thread): the slots the chunks closed go into their segments, in +// the order of the chunks. +void vk_exec_xfer_group_merge(int nchunks); // The front pass: whether a draw here counts samples (a query slot), and n slots of the ring in the order of the stream // for one chunk (the first is returned; vk_exec_xfer_worker hands them to the chunk's worker). int vk_exec_xfer_counting(void); @@ -290,6 +294,7 @@ uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after t void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs); extern uint64_t vk_exec_submit_seq; // replay thread: submissions replayed so far extern uint32_t vk_exec_brk; // replay thread: bumped by every op that is not state or a draw (vk_exec.c op_linear) +uint32_t vk_exec_brk_now(void); // vk_exec_brk, or in a parallel group's chunk the chunk's own count int vk_exec_mesh_run(const void *cur, const void **out, int max); void vk_exec_mesh_run_stops(char *out, size_t n); // NX2NAVI_EVO_TRACE=1 (diagnostics, vk_exec.c "event order trace"): one token per step of the replay's event order. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index c72863d1..f54189bb 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -276,6 +276,10 @@ typedef struct DrawCtx { struct { int valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM; // the shader memo (shader_memo_on) struct { int valid; VkPipeState s; const VkPipeInfo *info; } PM; // the pipeline memo (pipe_memo_on) void *fm; // feedback_scan's memo per texture handle (FmEnt[FM_N]), made at first use + void *mo_vm; // mvobj_pick's memo of the motion variant per vertex shader, made at first use + struct { const void *d; uint64_t ind; VkDeviceSize at; } ma[64]; // mesh_ring_entry: entries filled ahead + int ma_n, ma_i; // ... how many, and the next one a draw takes + uint32_t ma_brk; // ... vk_exec_brk_now when they were filled VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views VkDescSlot bind_d[MAX_DESC]; VkBufferView bind_bv[MAX_DESC]; @@ -2470,8 +2474,11 @@ static void pool_sync_draw(VkExec *ex) { plat_mutex_unlock(&pool_mt); } +static int mesh_ring_make(void); + void vk_exec_draw_group_begin(VkExec *ex) { dummies_init(); + if (vk_ctx()->has_mesh_shader && !vk_ctx()->mesh_fallback) mesh_ring_make(); // the chunks' indirect mesh draws if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) pool_sync(ex); // the set this submission took } @@ -4636,21 +4643,28 @@ static void mo_emit(VkCommandBuffer cb) { } } -static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0); -// The chunks of a parallel group (vk_exec.c) add to the frame's list at once: its memos, statistics and list go under -// mo_lock there. The order of the list does not change the picture: its draws write motion without blending, tested -// against the scene depth they do not write. +static int mvobj_pick(VkExec *ex, const GfxShader **vs_out, uint32_t *banks_out, uint64_t t0); +static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0, + const GfxShader *vs, uint32_t banks); +// The chunks of a parallel group (vk_exec.c) add to the frame's list at once: whether a draw is listed at all is decided +// by its own draw context (mvobj_pick, its memo per context), and only the frame's list, its memos and statistics go +// under mo_lock there: under one lock for the whole check, the chunks' scene draws waited for each other. The order of +// the list does not change the picture: its draws write motion without blending, tested against the scene depth they +// do not write. static PlatMutex mo_lock = PLAT_MUTEX_INIT; static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off) { if (MO.failed || !vk_mvprev_on() || !ex->dc->B.st_valid) return; static int timed = -1; if (timed < 0) timed = rt_knob("MVOBJ_TIMES", "0"); // NX2NAVI_MVOBJ_TIMES=1: the summary's times (the clock reads cost) - int lk = vk_exec_in_chunk(); - if (lk) plat_mutex_lock(&mo_lock); uint64_t t0 = timed ? kern_now_ns() : 0; - mvobj_draw_(ex, d, ib, ib_off, t0); - if (t0) MO.ns_check += kern_now_ns() - t0; + const GfxShader *vs; + uint32_t banks; + int listed = mvobj_pick(ex, &vs, &banks, t0); + int lk = listed && vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&mo_lock); + if (listed) mvobj_draw_(ex, d, ib, ib_off, t0, vs, banks); + if (t0) __atomic_add_fetch(&MO.ns_check, kern_now_ns() - t0, __ATOMIC_RELAXED); if (lk) plat_mutex_unlock(&mo_lock); } @@ -4776,30 +4790,34 @@ static int mo_prev_cached(int index, const p_bind_buf *u, VkDescriptorBufferInfo return r; } -static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0) { +// Whether the draw goes on to the frame's list (mvobj_draw_), with its motion variant and the banks its position reads; +// this draw context alone decides (no lock in a chunk): the draw's state, and the variant memo per context. +static int mvobj_pick(VkExec *ex, const GfxShader **vs_out, uint32_t *banks_out, uint64_t t0) { DrawCtx *const cx = ex->dc; - mvobj_banks(ex); + if (!vk_exec_in_chunk()) mvobj_banks(ex); // a test's log of every scene draw (its tables are not shared) const VkExecTarget *z = &ex->depth; if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || cx->B.st.samples > 1 || cx->B.st.raster_discard) - return; + return 0; // solid geometry only, which tests and writes the scene depth: effects, particles and screen quads write none, and // drawn again they would put their motion on the surfaces behind them (DLSS takes those from the surface) - if (!cx->B.st.ds.dt || !cx->B.st.ds.dw) return; + if (!cx->B.st.ds.dt || !cx->B.st.ds.dw) return 0; // a vertex shader that reads a user bank besides the camera's (c4) may move on its own const GfxShaderRes *ur; int nu = gfx_shader_resources(cx->B.st.vs, GFX_RES_UBO, &ur), reads = 0; for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4; - if (!reads) return; - MO.cand++; - uint64_t tp = t0 ? kern_now_ns() : 0; - if (t0) MO.ns_part[0] += tp - t0, MO.n_part[0]++; - uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); - if (MQ.frame != frame || MQ.fence != ex->fence || MQ.depth_fmt != z->format) mo_begin(ex, frame); + if (!reads) return 0; + __atomic_add_fetch(&MO.cand, 1, __ATOMIC_RELAXED); + if (t0) { + __atomic_add_fetch(&MO.ns_part[0], kern_now_ns() - t0, __ATOMIC_RELAXED); + __atomic_add_fetch(&MO.n_part[0], 1, __ATOMIC_RELAXED); + } if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); // the variant: the translation of the same code and answers, kept per shader once found; it names the banks the // position depends on (0: none, the shader is skipped from then on) - static struct { const GfxShader *of, *vs; uint32_t banks; } vmemo[1024]; + typedef struct { const GfxShader *of, *vs; uint32_t banks; } VMemo; + VMemo *vmemo = cx->mo_vm; + if (!vmemo && !(vmemo = cx->mo_vm = calloc(1024, sizeof *vmemo))) rt_fatal("vk: out of memory (motion memo)"); uint32_t vi = (uint32_t)((uintptr_t)cx->B.st.vs * 0x9E3779B97F4A7C15ull >> 54); if (vmemo[vi].of != cx->B.st.vs) { int has, pending = 0; @@ -4807,8 +4825,8 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi const GfxShader *v = shader_for(ex, NVN_STAGE_VS, &has, &pending, 0); cx->sf_mvprev = 0; if (!v || pending) { - MO.building++; - return; + __atomic_add_fetch(&MO.building, 1, __ATOMIC_RELAXED); + return 0; } GfxShaderIo io[32]; uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(cx->B.st.vs, io); @@ -4819,8 +4837,8 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi const GfxShader *vs = vmemo[vi].vs; uint32_t banks = vmemo[vi].banks; if (!banks || !(banks & ~(1u << 4))) { - MO.no_out++; - return; + __atomic_add_fetch(&MO.no_out, 1, __ATOMIC_RELAXED); + return 0; } // a position without the scene camera (c4) has a view of its own (c8 c9 c10 [c13]: its c8 is at a new address every // frame, last frame's is never found): drawn again it would get this frame's view and no camera motion @@ -4828,8 +4846,8 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi static int noc4 = -1; if (noc4 < 0) noc4 = rt_knob("MVOBJ_NOC4", "0"); // NX2NAVI_MVOBJ_NOC4=1 (test): drawn again all the same if (!(banks >> 4 & 1) && !noc4) { - MO.no_out++; - return; + __atomic_add_fetch(&MO.no_out, 1, __ATOMIC_RELAXED); + return 0; } // only positions that read an object's own blocks: bones (c5), place (c7) or c6. The rest (terrain and grass: c14 // c15 c16, with or without c9) measured no better than the camera's motion (NX2NAVI_MV_CHECK, turning, those draws @@ -4838,8 +4856,8 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi static int all = -1; if (all < 0) all = rt_knob("MVOBJ_ALL", "0"); if (!all && !(banks & (1u << 5 | 1u << 6 | 1u << 7))) { - MO.no_out++; - return; + __atomic_add_fetch(&MO.no_out, 1, __ATOMIC_RELAXED); + return 0; } // NX2NAVI_MVOBJ_ONLY= (test): only draws whose position reads no bank outside the mask (bit n: cn) static long only = -2; @@ -4847,13 +4865,26 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi const char *e = cfg_get("NX2NAVI_MVOBJ_ONLY"); only = e && *e ? strtol(e, NULL, 16) : -1; } - if (only >= 0 && banks & ~(uint32_t)only & ~(1u << 4)) return; + if (only >= 0 && banks & ~(uint32_t)only & ~(1u << 4)) return 0; static long exact = -2; // NX2NAVI_MVOBJ_EXACT= (test): only draws with exactly these position banks if (exact == -2) { const char *e = cfg_get("NX2NAVI_MVOBJ_EXACT"); exact = e && *e ? strtol(e, NULL, 16) : -1; } - if (exact >= 0 && banks != (uint32_t)exact) return; + if (exact >= 0 && banks != (uint32_t)exact) return 0; + *vs_out = vs, *banks_out = banks; + return 1; +} + +// A draw mvobj_pick took, onto the frame's list (in a chunk under mo_lock): the list begins at the first such draw of +// the frame's submission (mo_emit drops a list of another frame, submission or depth format). +static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0, + const GfxShader *vs, uint32_t banks) { + DrawCtx *const cx = ex->dc; + const VkExecTarget *z = &ex->depth; + uint64_t tp = t0 ? kern_now_ns() : 0; + uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (MQ.frame != frame || MQ.fence != ex->fence || MQ.depth_fmt != z->format) mo_begin(ex, frame); mo_mask_note(banks, cx->B.st.vs); // last frame's camera and blocks; held still when none of the position's blocks changed (most of the scenery) VkDescriptorBufferInfo cam, prevb[17]; @@ -5103,12 +5134,25 @@ int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth // NVN2 mesh draws (DrawMeshTasks): count task workgroups, or mesh workgroups for a program without a task stage, through // VK_EXT_mesh_shader. The stages are the translator's TaskEXT / MeshEXT modules (tools/xlate/MeshSpirv.cs). +// The lazy lookups below store the pointer before the flag that says it was looked up: the chunks of a parallel +// group draw meshes at once, and one that saw the flag first went on with no pointer and skipped its draws. static PFN_vkCmdDrawMeshTasksEXT draw_mesh_fn(void) { static PFN_vkCmdDrawMeshTasksEXT fn; static int looked; - if (!looked) { - looked = 1; + if (!__atomic_load_n(&looked, __ATOMIC_ACQUIRE)) { if (vk_ctx()->has_mesh_shader) fn = (PFN_vkCmdDrawMeshTasksEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksEXT"); + __atomic_store_n(&looked, 1, __ATOMIC_RELEASE); + } + return fn; +} + +static PFN_vkCmdDrawMeshTasksIndirectEXT draw_mesh_ind_fn(void) { + static PFN_vkCmdDrawMeshTasksIndirectEXT fn; + static int looked; + if (!__atomic_load_n(&looked, __ATOMIC_ACQUIRE)) { + if (vk_ctx()->has_mesh_shader) + fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT"); + __atomic_store_n(&looked, 1, __ATOMIC_RELEASE); } return fn; } @@ -5120,12 +5164,12 @@ static PFN_vkCmdDrawMeshTasksEXT draw_mesh_fn(void) { static void mesh_shading_rate(VkExec *ex) { static PFN_vkCmdSetFragmentShadingRateKHR fn; static int looked, from; - if (!looked) { - looked = 1; + if (!__atomic_load_n(&looked, __ATOMIC_ACQUIRE)) { // as draw_mesh_fn: the pointer before the flag const char *e = cfg_get("NX2NAVI_FOLIAGE_SHADOW_VRS"); from = e && *e ? atoi(e) : 2; if (vk_ctx()->has_vrs) fn = (PFN_vkCmdSetFragmentShadingRateKHR)vkGetDeviceProcAddr(vk_ctx()->device, "vkCmdSetFragmentShadingRateKHR"); + __atomic_store_n(&looked, 1, __ATOMIC_RELEASE); } if (!fn) return; rec_shading_rate(fn, ex->cb, !ex->color[0].image && ex->depth.image && (int)ex->depth.base_layer >= from ? 2 : 1); @@ -5489,12 +5533,25 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { // only by the draw it was filled for, at the same record address, while no other op was replayed since // (vk_exec_brk). In the village 42 of the 45 draws a frame are served that way: 3 gaps with these copies a frame // where there were 41 (0.04 ms of GPU time for 0.22), and about 40 passes fewer to begin and end. +// The entries filled ahead are the draw context's (DrawCtx.ma): a chunk of a parallel group (vk_exec.c) fills them for +// the draws that follow its own (vk_exec_mesh_run reads on from the op it records), and an entry it filled for a draw +// another chunk records is left unused. enum { MESH_AHEAD = 64 }; -static struct { const p_draw_mesh_ind *d; uint64_t ind; VkDeviceSize at; } mesh_ahead[MESH_AHEAD]; -static int mesh_ahead_n, mesh_ahead_i; -static uint32_t mesh_ahead_brk; +_Static_assert(MESH_AHEAD == sizeof dc_main.ma / sizeof dc_main.ma[0], "DrawCtx.ma holds MESH_AHEAD entries"); static uint64_t mesh_ahead_taken, mesh_ahead_own; // since the start: draws served by an entry filled ahead; the others +// The ring, made before the chunks of a group record (vk_exec_draw_group_begin) or at the first indirect mesh draw. +static int mesh_ring_make(void) { + if (mesh_ring) return 1; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING, + .usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + VkBuffer b; + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &b, &mesh_ring_alloc, NULL) != VK_SUCCESS) return 0; + __atomic_store_n(&mesh_ring, b, __ATOMIC_RELEASE); + return 1; +} + static int mesh_ahead_on(void) { static int mode = -1; if (mode < 0) mode = rt_knob("MESH_AHEAD", "1"); @@ -5502,6 +5559,7 @@ static int mesh_ahead_on(void) { } static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { + DrawCtx *const cx = ex->dc; if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) { char stops[160]; vk_exec_mesh_run_stops(stops, sizeof stops); @@ -5509,32 +5567,24 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan "their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1), (unsigned long long)mesh_ahead_taken, stops); } - if (mesh_ahead_i < mesh_ahead_n) { - if (mesh_ahead[mesh_ahead_i].d == d && mesh_ahead[mesh_ahead_i].ind == d->ind && mesh_ahead_brk == vk_exec_brk) { - *at = mesh_ahead[mesh_ahead_i++].at; - mesh_ahead_taken++; + if (cx->ma_i < cx->ma_n) { + if (cx->ma[cx->ma_i].d == d && cx->ma[cx->ma_i].ind == d->ind && cx->ma_brk == vk_exec_brk_now()) { + *at = cx->ma[cx->ma_i++].at; + __atomic_add_fetch(&mesh_ahead_taken, 1, __ATOMIC_RELAXED); return 1; } - mesh_ahead_n = mesh_ahead_i = 0; // the walk went another way: the entries left are never drawn from + cx->ma_n = cx->ma_i = 0; // the walk went another way: the entries left are never drawn from } - mesh_ahead_own++; - if (!mesh_ring) { - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING, - .usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; - VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; - if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &mesh_ring_alloc, NULL) != VK_SUCCESS) { - mesh_ring = VK_NULL_HANDLE; - return 0; - } - } - *at = 12ull * (mesh_ring_next++ % MESH_RING); + __atomic_add_fetch(&mesh_ahead_own, 1, __ATOMIC_RELAXED); + if (!__atomic_load_n(&mesh_ring, __ATOMIC_ACQUIRE) && (vk_exec_in_chunk() || !mesh_ring_make())) return 0; + *at = 12ull * (__atomic_fetch_add(&mesh_ring_next, 1, __ATOMIC_RELAXED) % MESH_RING); vk_exec_barrier(ex); // ends the pass (the next draw opens it again) and orders the culling's writes before the copy vk_gpu_gap_note(VK_GAP_MESH_RING, 12); vkCmdFillBuffer(ex->cb, mesh_ring, *at + 4, 8, 1); VkBufferCopy cp = {src->offset, *at, 4}; vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); mesh_count_note(ex, src); - mesh_ahead_n = mesh_ahead_i = 0; + cx->ma_n = cx->ma_i = 0; if (mesh_ahead_on()) { const void *run[MESH_AHEAD]; int n = vk_exec_mesh_run(d, run, MESH_AHEAD); @@ -5543,17 +5593,17 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan VkMemSpan ip; if (!vk_mem_resolve(f->ind, 12, &ip) || ip.offset % 4) break; // that draw reports it when the walk gets there VK_MEM_USE(ip); - VkDeviceSize fa = 12ull * (mesh_ring_next++ % MESH_RING); + VkDeviceSize fa = 12ull * (__atomic_fetch_add(&mesh_ring_next, 1, __ATOMIC_RELAXED) % MESH_RING); vk_gpu_gap_note(VK_GAP_MESH_RING, 12); vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1); VkBufferCopy fc = {ip.offset, fa, 4}; vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc); mesh_count_note(ex, &ip); ex->recorded += 2; - mesh_ahead[mesh_ahead_n].d = f, mesh_ahead[mesh_ahead_n].ind = f->ind, mesh_ahead[mesh_ahead_n].at = fa; - mesh_ahead_n++; + cx->ma[cx->ma_n].d = f, cx->ma[cx->ma_n].ind = f->ind, cx->ma[cx->ma_n].at = fa; + cx->ma_n++; } - mesh_ahead_brk = vk_exec_brk; + cx->ma_brk = vk_exec_brk_now(); } VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, @@ -5568,13 +5618,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan // DrawMeshTasksIndirect: one draw from its ring entry (mesh_ring_entry); x2-x4 of the call vary like leftovers, so they // are not read as a count or stride. static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { - static PFN_vkCmdDrawMeshTasksIndirectEXT fn; - static int looked; - if (!looked) { - looked = 1; - if (vk_ctx()->has_mesh_shader) - fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT"); - } + PFN_vkCmdDrawMeshTasksIndirectEXT fn = draw_mesh_ind_fn(); int fallback = vk_ctx()->mesh_fallback; if (!fn && !fallback) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); static int on = -1; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index df63b238..e3dde4fd 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -204,7 +204,10 @@ static Pending *pending_slot(uint64_t addr, int insert) { // report into guest memory. Slots come from one ring; the completion thread reads and host-resets them // (hostQueryReset, Vulkan 1.2), so the replay records no reset commands. A full ring (no slot read back yet) leaves the // draw uncounted and the segment's count at least 1, as does a query still unavailable after its fence. Without hostQueryReset the counts stay 0. -#define OQ_N 4096 +// The ring holds about four frames of Hyrule Field's queries (about 3,500 a frame): the replay workers' chunks take +// blocks of slots sized to their draws (vk_exec_xfer_slots), more than they use, and with 4096 the ring came round to +// slots the completion thread had not reset yet, leaving draws uncounted. +#define OQ_N 16384 #define LOG_ONCE(...) do { static volatile char once_; if (!__atomic_exchange_n(&once_, 1, __ATOMIC_RELAXED)) rt_log(__VA_ARGS__); } while (0) static VkQueryPool oq_pool; static int oq_state; // 0 not set up, 1 counting available, -1 unavailable @@ -228,21 +231,24 @@ typedef struct { uint64_t addr, seq; int reset; volatile int32_t lost; uint32_t // the counting as the stream does from its chunk's first op (vk_exec_xfer_chunk_state): XW.counting and the segment its // closed queries belong to, the next one at or after its position in the stream (XW.seg; past the group's last, the // pending list oq_seg, which a later reset or report takes). It keeps the query open in its own pass (XW.open), takes -// ring slots from its chunk's block and adds the slots it closes under oq_seg_lock. XW.cond: the decisions of its chunk's -// render conditions, made by the front pass in the order of the stream. +// ring slots from its chunk's block and keeps the slots it closes with their segment in its chunk's list (CP), which +// vk_exec_xfer_group_merge puts into the segments once the group is done, chunk after chunk: under a lock per slot, the +// workers waited for each other at every query. XW.cond: the decisions of its chunk's render conditions, made by the +// front pass in the order of the stream. // The blocks (vk_exec_xfer_slots): the front pass gives each chunk as many slots as it has draws that count samples and // render, one after the other, so the slots follow the stream as the replay's own do and the completion thread reads // and resets them in long runs (oq_take). Taken from the ring at once, the slots of a group's chunks interleaved: four // times the result calls and a reset per slot, 3.5 ms more of the wait after a frame's first submission (RX 7900 XT, // Hyrule Field, 3 workers). A chunk that needs more than its block takes them from the ring. static _Thread_local struct { - int on, open, counting, reported, keep; + int on, open, counting, reported, keep, chunk; uint32_t seg, qnext, qend; const uint8_t *cond; uint32_t ncond, i; } XW = {.open = -1}; -static PlatMutex oq_seg_lock = PLAT_MUTEX_INIT; static struct { int on; OqSegment **s; uint32_t n, cap; } GS; // the front pass's segments of the group +#define CP_CHUNKS 32 +static struct { uint32_t *seg, *q, n, cap; } CP[CP_CHUNKS]; // per chunk index: closed slots and their segment void vk_exec_xfer_group(int on) { // on: a new group's front pass begins; off: it ended, the list stays if (on) GS.n = 0; @@ -261,8 +267,10 @@ uint64_t vk_exec_xfer_chunk_state(void) { return (uint64_t)GS.n << 8 | (uint64_t)(oq_counting & 1) | (uint64_t)(oq_reported & 1) << 1 | (uint64_t)(oq_keep & 1) << 2; } -void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots) { - XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0; +void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots, + int chunk) { + if (chunk < 0 || chunk >= CP_CHUNKS) rt_fatal("vk: replay worker chunk %d of at most %d", chunk, CP_CHUNKS); + XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0, XW.chunk = chunk; XW.qnext = slot0, XW.qend = slot0 + nslots; XW.counting = (int)(state & 1), XW.reported = (int)(state >> 1 & 1), XW.keep = (int)(state >> 2 & 1); XW.seg = (uint32_t)(state >> 8); @@ -270,22 +278,37 @@ void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t s // Adds closed slot q to the segment a worker's queries belong to now, or the pending list. static void seg_add(uint32_t q) { - plat_mutex_lock(&oq_seg_lock); - if (XW.seg < GS.n) { - OqSegment *s = GS.s[XW.seg]; - if (s->n == s->cap) { - s->cap = s->cap ? s->cap * 2 : 16; - if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); - } - s->q[s->n++] = q; - } else { - if (oq_nseg == oq_capseg) { - oq_capseg = oq_capseg ? oq_capseg * 2 : 64; - if (!(oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg))) rt_fatal("vk: out of memory"); - } - oq_seg[oq_nseg++] = q; + __typeof__(CP[0]) *l = &CP[XW.chunk]; + if (l->n == l->cap) { + l->cap = l->cap ? l->cap * 2 : 256; + if (!(l->seg = realloc(l->seg, sizeof *l->seg * l->cap)) || !(l->q = realloc(l->q, sizeof *l->q * l->cap))) + rt_fatal("vk: out of memory"); + } + l->seg[l->n] = XW.seg, l->q[l->n++] = q; +} + +void vk_exec_xfer_group_merge(int nchunks) { + for (int c = 0; c < nchunks && c < CP_CHUNKS; c++) { + __typeof__(CP[0]) *l = &CP[c]; + for (uint32_t i = 0; i < l->n; i++) { + uint32_t q = l->q[i]; + if (l->seg[i] < GS.n) { + OqSegment *s = GS.s[l->seg[i]]; + if (s->n == s->cap) { + s->cap = s->cap ? s->cap * 2 : 16; + if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); + } + s->q[s->n++] = q; + } else { + if (oq_nseg == oq_capseg) { + oq_capseg = oq_capseg ? oq_capseg * 2 : 64; + if (!(oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg))) rt_fatal("vk: out of memory"); + } + oq_seg[oq_nseg++] = q; + } + } + l->n = 0; } - plat_mutex_unlock(&oq_seg_lock); } static int oq_ready(void) { diff --git a/runtime/vk_mvprev.c b/runtime/vk_mvprev.c index 2b65767b..682fd6d9 100644 --- a/runtime/vk_mvprev.c +++ b/runtime/vk_mvprev.c @@ -81,7 +81,7 @@ int vk_mvprev_on(void) { return P.on > 0; } void vk_mvprev_want(uint32_t index_mask) { static int fixed = -1; if (fixed < 0) fixed = !rt_knob("MVPREV_GROW", "1"); // NX2NAVI_MVPREV_GROW=0: c5, c7 and c14 only (test) - if (!fixed) P.mask |= index_mask & ((1u << BANKS) - 1); + if (!fixed) __atomic_fetch_or(&P.mask, index_mask & ((1u << BANKS) - 1), __ATOMIC_RELAXED); // chunks at once } uint64_t vk_mvprev_step(int stage, int index, uint64_t size) {