diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 3391129e..a75bdca0 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -57,6 +57,7 @@ typedef struct { uint32_t cond0, ncond; // its render conditions' decisions in G.cond uint64_t oq; // the occlusion counting at its first op (vk_exec_xfer_chunk_state) uint32_t slot0, nslots; // its block of occlusion query slots (vk_exec_xfer_slots) + uint32_t brk; // its ops that are not state or a draw (vk_exec_brk_now) int idx; // in its group: 0 runs the group's pass hooks VkCommandBuffer cb; int recorded; @@ -72,6 +73,7 @@ static struct { volatile uint32_t gen; // bumped when a group is handed out } G = {.next = 1 << 30}; static _Thread_local Chunk *rw_chunk; // the chunk the calling thread records, NULL outside one +static _Thread_local const op_hdr *rw_op; // in a chunk: the op it records (vk_exec_mesh_run reads on from it) static _Thread_local int rw_ordered; // every earlier chunk of its group is done static PlatMutex tfr_lock = PLAT_MUTEX_INIT; // target frames (vk_target_frames.h) noted by several chunks at once @@ -1628,8 +1630,10 @@ void vk_exec_mesh_run_stops(char *out, size_t n) { // The indirect mesh draws that follow the op being replayed (cur: its payload) with only ops of op_linear between // them, MESH_RUN_OPS at most: their payloads into out, at most max. 0 when the walk is not at cur. +uint32_t vk_exec_brk_now(void) { return rw_chunk ? rw_chunk->brk : vk_exec_brk; } + int vk_exec_mesh_run(const void *cur, const void **out, int max) { - const op_hdr *h = AH.at[AH.depth]; + const op_hdr *h = rw_chunk ? rw_op : AH.at[AH.depth]; // a chunk reads on from its own op (into the next ones) if (!h || nvn_op_payload(h) != cur) return 0; int n = 0; h = nvn_op_next(h); @@ -1729,9 +1733,9 @@ static void op_one(VkExec *ex, const op_hdr *h) { // 2026-11-11): about 3,500 draws a frame in 82 runs, 96% of them in runs of 16 draws or more and 67% in runs of 256 or // more, with a timestamp report beside nearly every draw and a render condition beside every second one. // A run is cut into groups: a group ends where the render targets change after its first draw or clear, and before an -// inline uniform update, a mesh draw and a samples-passed counter op, which the submitting thread replays itself in -// the order of the stream like everything outside groups. A group of 2 * CHUNK_DRAWS draws or more is recorded by -// several threads at once: +// inline uniform update (and a mesh draw with the mesh fallback or while NX2NAVI_GPU_PASSES copies their counts), which +// the submitting thread replays itself in the order of the stream like everything outside groups. A group of +// 2 * CHUNK_DRAWS draws or more is recorded by several threads at once: // - the front pass, on the submitting thread, walks the group in the order of the stream before anything of it is // recorded: it applies the state and binds, settles the CPU side of counter reports and decides the render conditions // (vk_exec_xfer_front), and copies the replay state at the first op of each chunk; @@ -1743,7 +1747,9 @@ static void op_one(VkExec *ex, const op_hdr *h) { // which reaches the group's pass first, and the others start after that pass began; groups follow one another. // What still depends on the order between chunks waits for its turn (vk_exec_turn: every earlier chunk of the group // done): a texture lookup that would make or take back an entry (vk_texture.c entry_of) returns nothing and the chunk -// replays the op again once in turn, and so do the contact shadows' pass, per-object motion and the HDR pass's source. +// replays the op again once in turn, and so do the contact shadows' pass and the HDR pass's source. Per-object motion +// lists its draws under a lock (vk_exec_draw.c mvobj_draw), and the occlusion queries' slots and segments follow the +// stream (vk_exec_xfer.c vk_exec_xfer_slots, vk_exec_xfer_group_merge). #define RW_MAX 16 #define CHUNK_DRAWS 32 // draws a chunk takes at least #define GROUP_PEEK 65536u // ops read ahead for a group, at most @@ -1777,11 +1783,26 @@ static struct { const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small uint64_t groups, group_draws, chunks, serial_draws, wait_ns; // since the last perf line uint64_t end_draws[5]; // draws of groups by what ended them (group_peek's GE_*), since the last perf line + uint64_t small_end[5]; // ... of those too small to split + uint64_t run_end[OP_COUNT_]; // draws of groups that an op outside groups ended, by that op + // the submitting thread, since the last perf line: ops outside groups by kind (SER_*), and in groups the front + // pass, its own chunks and the wait for the others' + uint64_t ser_n[4], ser_ns[4], front_ns, own_ns, spin_ns; volatile uint64_t turns, turn_ns, redos; } RW = {.n = -1}; static void group_take(int pool); +enum { SER_DRAW, SER_MESH, SER_DISPATCH, SER_OTHER }; +static int ser_kind(uint32_t op) { + switch (op) { + case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: return SER_DRAW; + case OP_DRAW_MESH: case OP_DRAW_MESH_IND: return SER_MESH; + case OP_DISPATCH: case OP_DISPATCH_IND: return SER_DISPATCH; + default: return SER_OTHER; + } +} + static void rw_main(void *arg) { RwWorker *w = arg; plat_thread_priority(PLAT_PRIO_HIGHEST); // as the recording thread (rec_claim) @@ -1852,12 +1873,18 @@ static void views_release(VkExec *ex, uint64_t fence) { enum { GE_RUN, GE_TARGET, GE_UB, GE_MESH, GE_LIMIT, GE_N }; static const char *const ge_name[GE_N] = {"run end", "target change", "uniform update", "mesh draw", "read limit"}; +int vk_gpu_passes_capturing(void); // vk_crumbs.c: NX2NAVI_GPU_PASSES is copying this frame's counts + // The draws of the group that starts at h (it ends at *end, for the reason *why), reading at most GROUP_PEEK ops. -static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end, int *why) { +static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end, int *why, uint32_t *end_op) { uint32_t draws = 0; + *end_op = 0; int begun = 0; const p_set_rt *rt = &ex->rt; *why = GE_LIMIT; + // mesh draws stay on the submitting thread with the mesh fallback (its buffers are shared) and while + // NX2NAVI_GPU_PASSES copies their workgroup counts (one list) + int mesh_serial = vk_ctx()->mesh_fallback || vk_gpu_passes_capturing(); for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) { *why = GE_RUN; break; } // the walk reports it const void *a = nvn_op_payload(h); @@ -1865,14 +1892,15 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; } - if (!run_op(h->op)) { *why = GE_RUN; break; } + if (!run_op(h->op)) { *why = GE_RUN, *end_op = h->op; break; } if (h->op == OP_UPDATE_UB) { *why = GE_UB; break; } - if (h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND) { *why = GE_MESH; break; } + int mesh = h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND; + if (mesh && mesh_serial) { *why = GE_MESH; break; } if (h->op == OP_SET_RT && memcmp(rt, a, sizeof *rt)) { if (begun) { *why = GE_TARGET; break; } // another pass: the next group rt = a; } - if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) draws++, begun = 1; + if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND || mesh) draws++, begun = 1; if (h->op == OP_CLEAR_COLOR || h->op == OP_CLEAR_DS) begun = 1; h = nvn_op_next(h); } @@ -1884,7 +1912,7 @@ static Chunk *chunk_open(const VkExec *ex, const op_hdr *h, int idx) { Chunk *c = &G.c[idx]; memcpy(&c->ex, ex, sizeof *ex); c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx, c->oq = vk_exec_xfer_chunk_state(); - c->nslots = 0; + c->nslots = 0, c->brk = 0; return c; } @@ -1895,7 +1923,7 @@ static void chunk_record(Chunk *c, int pool) { VkExec *ex = &c->ex, *was = x_cur; if (pool < RW.n) ex->dc = RW.w[pool]->dc; // the submitting thread records with its own (the state's) x_cur = ex, rw_chunk = c, rw_ordered = !c->idx; - vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond, c->oq, c->slot0, c->nslots); + vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond, c->oq, c->slot0, c->nslots, c->idx); ex->cb = vk_sync_worker_begin(pool); // the replay's command buffer before the group ended behind a full barrier, with every image in GENERAL ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL; @@ -1908,7 +1936,8 @@ static void chunk_record(Chunk *c, int pool) { o = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(o))->target; continue; } - if (!op_linear(o->op)) vk_exec_brk++; + if (!op_linear(o->op)) c->brk++; + rw_op = o; op_one(ex, o); if (vk_texture_deferred_take()) { // a lookup would have made or taken back a texture entry: in turn, again vk_exec_pass_end(ex); @@ -1927,7 +1956,7 @@ static void chunk_record(Chunk *c, int pool) { free(ex->views); ex->views = NULL; c->cb = ex->cb, c->recorded = ex->recorded; - vk_exec_xfer_worker(0, NULL, 0, 0, 0, 0); + vk_exec_xfer_worker(0, NULL, 0, 0, 0, 0, 0); x_cur = was, rw_chunk = NULL; __atomic_store_n(&c->done, 1, __ATOMIC_RELEASE); } @@ -1946,14 +1975,16 @@ static void group_take(int pool) { static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { const op_hdr *end; int why; - uint32_t draws = group_peek(ex, h, &end, &why); + uint32_t end_op; + uint32_t draws = group_peek(ex, h, &end, &why, &end_op); + if (why == GE_RUN && end_op < OP_COUNT_) RW.run_end[end_op] += draws; if (end == h) return h; // an op the submitting thread replays itself RW.end_draws[why] += draws; int n = (int)(draws / CHUNK_DRAWS); if (n > RW.n + 1) n = RW.n + 1; if (n > GROUP_CHUNKS) n = GROUP_CHUNKS; if (n < 2 || vk_dump_armed || !vk_sync_append_room(n)) { - RW.skip_to[depth] = end, RW.serial_draws += draws; + RW.skip_to[depth] = end, RW.serial_draws += draws, RW.small_end[why] += draws; return h; } uint64_t t0 = kern_now_ns(); @@ -1974,7 +2005,7 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { } if (cut) c = chunk_open(ex, o, nc++), cd = 0, cut = 0; switch (o->op) { - case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: + case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: case OP_DRAW_MESH: case OP_DRAW_MESH_IND: if (vk_exec_xfer_counting() && !ex->render_disabled) c->nslots++; // at most one query each if (++cd == per && nc < n) cut = 1; break; @@ -2003,12 +2034,17 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { G.hooks = 0; __atomic_store_n(&G.n, nc, __ATOMIC_RELEASE); __atomic_store_n(&G.next, 0, __ATOMIC_RELEASE); + uint64_t tf = kern_now_ns(); __atomic_add_fetch(&G.gen, 1, __ATOMIC_SEQ_CST); for (int k = 0; k < RW.n; k++) if (__atomic_load_n(&RW.w[k]->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&RW.w[k]->wake); group_take(RW.n); + uint64_t to = kern_now_ns(); for (int i = 0; i < nc; i++) while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause(); + uint64_t tw = kern_now_ns(); + RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.spin_ns += tw - to; + vk_exec_xfer_group_merge(nc); // the chunks' closed query slots into their segments __atomic_store_n(&G.next, 1 << 30, __ATOMIC_RELEASE); // the state the front pass left goes on in a new command buffer of the replay's, after the chunks' in the part VkCommandBuffer cbs[GROUP_CHUNKS]; @@ -2057,7 +2093,14 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { } break; } - default: op_one(ex, h); break; + default: + if (RW.n > 0) { // the perf line's split of the replay outside groups + int k = ser_kind(h->op); + uint64_t t = kern_now_ns(); + op_one(ex, h); + RW.ser_n[k]++, RW.ser_ns[k] += kern_now_ns() - t; + } else op_one(ex, h); + break; } if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); if (split_draws && split_wanted()) maybe_split(ex); @@ -2229,13 +2272,40 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t RW.groups ? (double)RW.chunks / (double)RW.groups : 0.0, RW.groups ? (double)RW.wait_ns / 1e6 / (double)RW.groups : 0.0, (double)RW.serial_draws / (double)n_subs, (double)turns / (double)n_subs, (double)turn_ns / 1e6 / (double)n_subs, (double)redos / (double)n_subs); - char e[256]; + char e[512]; int o = 0; for (int k = 0; k < GE_N; k++) o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.0f", k ? "," : "", ge_name[k], (double)RW.end_draws[k] / (double)n_subs); + o += snprintf(e + o, sizeof e - (size_t)o, "; of them in groups too small to split:"); + for (int k = 0; k < GE_N; k++) + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.0f", k ? "," : "", ge_name[k], + (double)RW.small_end[k] / (double)n_subs); rt_log("perf: replay workers: draws per submission by what ended their group:%s", e); - memset(RW.end_draws, 0, sizeof RW.end_draws); + o = 0; + for (int k = 0; k < 6; k++) { // the ops outside groups that ended the most, by their groups' draws + int best = 0; + for (int op = 1; op < OP_COUNT_; op++) + if (RW.run_end[op] > RW.run_end[best]) best = op; + if (!RW.run_end[best]) break; + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.0f", k ? "," : "", nvn_op_name((uint32_t)best), + (double)RW.run_end[best] / (double)n_subs); + RW.run_end[best] = 0; + } + memset(RW.run_end, 0, sizeof RW.run_end); + if (o) rt_log("perf: replay workers: draws per submission of groups a run end ended, by the op:%s", e); + memset(RW.end_draws, 0, sizeof RW.end_draws), memset(RW.small_end, 0, sizeof RW.small_end); + double ns = 1e6 * (double)n_subs; + rt_log("perf: replay workers: the submitting thread per submission: outside groups draws %.0f (%.3f ms), mesh " + "draws %.0f (%.3f ms), dispatches %.0f (%.3f ms), other ops %.0f (%.3f ms); in groups the front pass " + "%.3f ms, its own chunks %.3f ms, waiting for the others %.3f ms", + (double)RW.ser_n[SER_DRAW] / (double)n_subs, (double)RW.ser_ns[SER_DRAW] / ns, + (double)RW.ser_n[SER_MESH] / (double)n_subs, (double)RW.ser_ns[SER_MESH] / ns, + (double)RW.ser_n[SER_DISPATCH] / (double)n_subs, (double)RW.ser_ns[SER_DISPATCH] / ns, + (double)RW.ser_n[SER_OTHER] / (double)n_subs, (double)RW.ser_ns[SER_OTHER] / ns, + (double)RW.front_ns / ns, (double)RW.own_ns / ns, (double)RW.spin_ns / ns); + memset(RW.ser_n, 0, sizeof RW.ser_n), memset(RW.ser_ns, 0, sizeof RW.ser_ns); + RW.front_ns = RW.own_ns = RW.spin_ns = 0; } RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index e3696117..de4dd75d 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -164,7 +164,11 @@ void vk_exec_draw_group_begin(VkExec *ex); void vk_exec_xfer_group(int on); int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload); uint64_t vk_exec_xfer_chunk_state(void); -void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots); +void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots, + int chunk); +// After every chunk of the group is done (submitting thread): the slots the chunks closed go into their segments, in +// the order of the chunks. +void vk_exec_xfer_group_merge(int nchunks); // The front pass: whether a draw here counts samples (a query slot), and n slots of the ring in the order of the stream // for one chunk (the first is returned; vk_exec_xfer_worker hands them to the chunk's worker). int vk_exec_xfer_counting(void); @@ -290,6 +294,7 @@ uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after t void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs); extern uint64_t vk_exec_submit_seq; // replay thread: submissions replayed so far extern uint32_t vk_exec_brk; // replay thread: bumped by every op that is not state or a draw (vk_exec.c op_linear) +uint32_t vk_exec_brk_now(void); // vk_exec_brk, or in a parallel group's chunk the chunk's own count int vk_exec_mesh_run(const void *cur, const void **out, int max); void vk_exec_mesh_run_stops(char *out, size_t n); // NX2NAVI_EVO_TRACE=1 (diagnostics, vk_exec.c "event order trace"): one token per step of the replay's event order. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index c72863d1..f54189bb 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -276,6 +276,10 @@ typedef struct DrawCtx { struct { int valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM; // the shader memo (shader_memo_on) struct { int valid; VkPipeState s; const VkPipeInfo *info; } PM; // the pipeline memo (pipe_memo_on) void *fm; // feedback_scan's memo per texture handle (FmEnt[FM_N]), made at first use + void *mo_vm; // mvobj_pick's memo of the motion variant per vertex shader, made at first use + struct { const void *d; uint64_t ind; VkDeviceSize at; } ma[64]; // mesh_ring_entry: entries filled ahead + int ma_n, ma_i; // ... how many, and the next one a draw takes + uint32_t ma_brk; // ... vk_exec_brk_now when they were filled VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views VkDescSlot bind_d[MAX_DESC]; VkBufferView bind_bv[MAX_DESC]; @@ -2470,8 +2474,11 @@ static void pool_sync_draw(VkExec *ex) { plat_mutex_unlock(&pool_mt); } +static int mesh_ring_make(void); + void vk_exec_draw_group_begin(VkExec *ex) { dummies_init(); + if (vk_ctx()->has_mesh_shader && !vk_ctx()->mesh_fallback) mesh_ring_make(); // the chunks' indirect mesh draws if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) pool_sync(ex); // the set this submission took } @@ -4636,21 +4643,28 @@ static void mo_emit(VkCommandBuffer cb) { } } -static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0); -// The chunks of a parallel group (vk_exec.c) add to the frame's list at once: its memos, statistics and list go under -// mo_lock there. The order of the list does not change the picture: its draws write motion without blending, tested -// against the scene depth they do not write. +static int mvobj_pick(VkExec *ex, const GfxShader **vs_out, uint32_t *banks_out, uint64_t t0); +static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0, + const GfxShader *vs, uint32_t banks); +// The chunks of a parallel group (vk_exec.c) add to the frame's list at once: whether a draw is listed at all is decided +// by its own draw context (mvobj_pick, its memo per context), and only the frame's list, its memos and statistics go +// under mo_lock there: under one lock for the whole check, the chunks' scene draws waited for each other. The order of +// the list does not change the picture: its draws write motion without blending, tested against the scene depth they +// do not write. static PlatMutex mo_lock = PLAT_MUTEX_INIT; static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off) { if (MO.failed || !vk_mvprev_on() || !ex->dc->B.st_valid) return; static int timed = -1; if (timed < 0) timed = rt_knob("MVOBJ_TIMES", "0"); // NX2NAVI_MVOBJ_TIMES=1: the summary's times (the clock reads cost) - int lk = vk_exec_in_chunk(); - if (lk) plat_mutex_lock(&mo_lock); uint64_t t0 = timed ? kern_now_ns() : 0; - mvobj_draw_(ex, d, ib, ib_off, t0); - if (t0) MO.ns_check += kern_now_ns() - t0; + const GfxShader *vs; + uint32_t banks; + int listed = mvobj_pick(ex, &vs, &banks, t0); + int lk = listed && vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&mo_lock); + if (listed) mvobj_draw_(ex, d, ib, ib_off, t0, vs, banks); + if (t0) __atomic_add_fetch(&MO.ns_check, kern_now_ns() - t0, __ATOMIC_RELAXED); if (lk) plat_mutex_unlock(&mo_lock); } @@ -4776,30 +4790,34 @@ static int mo_prev_cached(int index, const p_bind_buf *u, VkDescriptorBufferInfo return r; } -static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0) { +// Whether the draw goes on to the frame's list (mvobj_draw_), with its motion variant and the banks its position reads; +// this draw context alone decides (no lock in a chunk): the draw's state, and the variant memo per context. +static int mvobj_pick(VkExec *ex, const GfxShader **vs_out, uint32_t *banks_out, uint64_t t0) { DrawCtx *const cx = ex->dc; - mvobj_banks(ex); + if (!vk_exec_in_chunk()) mvobj_banks(ex); // a test's log of every scene draw (its tables are not shared) const VkExecTarget *z = &ex->depth; if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || cx->B.st.samples > 1 || cx->B.st.raster_discard) - return; + return 0; // solid geometry only, which tests and writes the scene depth: effects, particles and screen quads write none, and // drawn again they would put their motion on the surfaces behind them (DLSS takes those from the surface) - if (!cx->B.st.ds.dt || !cx->B.st.ds.dw) return; + if (!cx->B.st.ds.dt || !cx->B.st.ds.dw) return 0; // a vertex shader that reads a user bank besides the camera's (c4) may move on its own const GfxShaderRes *ur; int nu = gfx_shader_resources(cx->B.st.vs, GFX_RES_UBO, &ur), reads = 0; for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4; - if (!reads) return; - MO.cand++; - uint64_t tp = t0 ? kern_now_ns() : 0; - if (t0) MO.ns_part[0] += tp - t0, MO.n_part[0]++; - uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); - if (MQ.frame != frame || MQ.fence != ex->fence || MQ.depth_fmt != z->format) mo_begin(ex, frame); + if (!reads) return 0; + __atomic_add_fetch(&MO.cand, 1, __ATOMIC_RELAXED); + if (t0) { + __atomic_add_fetch(&MO.ns_part[0], kern_now_ns() - t0, __ATOMIC_RELAXED); + __atomic_add_fetch(&MO.n_part[0], 1, __ATOMIC_RELAXED); + } if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); // the variant: the translation of the same code and answers, kept per shader once found; it names the banks the // position depends on (0: none, the shader is skipped from then on) - static struct { const GfxShader *of, *vs; uint32_t banks; } vmemo[1024]; + typedef struct { const GfxShader *of, *vs; uint32_t banks; } VMemo; + VMemo *vmemo = cx->mo_vm; + if (!vmemo && !(vmemo = cx->mo_vm = calloc(1024, sizeof *vmemo))) rt_fatal("vk: out of memory (motion memo)"); uint32_t vi = (uint32_t)((uintptr_t)cx->B.st.vs * 0x9E3779B97F4A7C15ull >> 54); if (vmemo[vi].of != cx->B.st.vs) { int has, pending = 0; @@ -4807,8 +4825,8 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi const GfxShader *v = shader_for(ex, NVN_STAGE_VS, &has, &pending, 0); cx->sf_mvprev = 0; if (!v || pending) { - MO.building++; - return; + __atomic_add_fetch(&MO.building, 1, __ATOMIC_RELAXED); + return 0; } GfxShaderIo io[32]; uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(cx->B.st.vs, io); @@ -4819,8 +4837,8 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi const GfxShader *vs = vmemo[vi].vs; uint32_t banks = vmemo[vi].banks; if (!banks || !(banks & ~(1u << 4))) { - MO.no_out++; - return; + __atomic_add_fetch(&MO.no_out, 1, __ATOMIC_RELAXED); + return 0; } // a position without the scene camera (c4) has a view of its own (c8 c9 c10 [c13]: its c8 is at a new address every // frame, last frame's is never found): drawn again it would get this frame's view and no camera motion @@ -4828,8 +4846,8 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi static int noc4 = -1; if (noc4 < 0) noc4 = rt_knob("MVOBJ_NOC4", "0"); // NX2NAVI_MVOBJ_NOC4=1 (test): drawn again all the same if (!(banks >> 4 & 1) && !noc4) { - MO.no_out++; - return; + __atomic_add_fetch(&MO.no_out, 1, __ATOMIC_RELAXED); + return 0; } // only positions that read an object's own blocks: bones (c5), place (c7) or c6. The rest (terrain and grass: c14 // c15 c16, with or without c9) measured no better than the camera's motion (NX2NAVI_MV_CHECK, turning, those draws @@ -4838,8 +4856,8 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi static int all = -1; if (all < 0) all = rt_knob("MVOBJ_ALL", "0"); if (!all && !(banks & (1u << 5 | 1u << 6 | 1u << 7))) { - MO.no_out++; - return; + __atomic_add_fetch(&MO.no_out, 1, __ATOMIC_RELAXED); + return 0; } // NX2NAVI_MVOBJ_ONLY= (test): only draws whose position reads no bank outside the mask (bit n: cn) static long only = -2; @@ -4847,13 +4865,26 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi const char *e = cfg_get("NX2NAVI_MVOBJ_ONLY"); only = e && *e ? strtol(e, NULL, 16) : -1; } - if (only >= 0 && banks & ~(uint32_t)only & ~(1u << 4)) return; + if (only >= 0 && banks & ~(uint32_t)only & ~(1u << 4)) return 0; static long exact = -2; // NX2NAVI_MVOBJ_EXACT= (test): only draws with exactly these position banks if (exact == -2) { const char *e = cfg_get("NX2NAVI_MVOBJ_EXACT"); exact = e && *e ? strtol(e, NULL, 16) : -1; } - if (exact >= 0 && banks != (uint32_t)exact) return; + if (exact >= 0 && banks != (uint32_t)exact) return 0; + *vs_out = vs, *banks_out = banks; + return 1; +} + +// A draw mvobj_pick took, onto the frame's list (in a chunk under mo_lock): the list begins at the first such draw of +// the frame's submission (mo_emit drops a list of another frame, submission or depth format). +static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0, + const GfxShader *vs, uint32_t banks) { + DrawCtx *const cx = ex->dc; + const VkExecTarget *z = &ex->depth; + uint64_t tp = t0 ? kern_now_ns() : 0; + uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (MQ.frame != frame || MQ.fence != ex->fence || MQ.depth_fmt != z->format) mo_begin(ex, frame); mo_mask_note(banks, cx->B.st.vs); // last frame's camera and blocks; held still when none of the position's blocks changed (most of the scenery) VkDescriptorBufferInfo cam, prevb[17]; @@ -5103,12 +5134,25 @@ int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth // NVN2 mesh draws (DrawMeshTasks): count task workgroups, or mesh workgroups for a program without a task stage, through // VK_EXT_mesh_shader. The stages are the translator's TaskEXT / MeshEXT modules (tools/xlate/MeshSpirv.cs). +// The lazy lookups below store the pointer before the flag that says it was looked up: the chunks of a parallel +// group draw meshes at once, and one that saw the flag first went on with no pointer and skipped its draws. static PFN_vkCmdDrawMeshTasksEXT draw_mesh_fn(void) { static PFN_vkCmdDrawMeshTasksEXT fn; static int looked; - if (!looked) { - looked = 1; + if (!__atomic_load_n(&looked, __ATOMIC_ACQUIRE)) { if (vk_ctx()->has_mesh_shader) fn = (PFN_vkCmdDrawMeshTasksEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksEXT"); + __atomic_store_n(&looked, 1, __ATOMIC_RELEASE); + } + return fn; +} + +static PFN_vkCmdDrawMeshTasksIndirectEXT draw_mesh_ind_fn(void) { + static PFN_vkCmdDrawMeshTasksIndirectEXT fn; + static int looked; + if (!__atomic_load_n(&looked, __ATOMIC_ACQUIRE)) { + if (vk_ctx()->has_mesh_shader) + fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT"); + __atomic_store_n(&looked, 1, __ATOMIC_RELEASE); } return fn; } @@ -5120,12 +5164,12 @@ static PFN_vkCmdDrawMeshTasksEXT draw_mesh_fn(void) { static void mesh_shading_rate(VkExec *ex) { static PFN_vkCmdSetFragmentShadingRateKHR fn; static int looked, from; - if (!looked) { - looked = 1; + if (!__atomic_load_n(&looked, __ATOMIC_ACQUIRE)) { // as draw_mesh_fn: the pointer before the flag const char *e = cfg_get("NX2NAVI_FOLIAGE_SHADOW_VRS"); from = e && *e ? atoi(e) : 2; if (vk_ctx()->has_vrs) fn = (PFN_vkCmdSetFragmentShadingRateKHR)vkGetDeviceProcAddr(vk_ctx()->device, "vkCmdSetFragmentShadingRateKHR"); + __atomic_store_n(&looked, 1, __ATOMIC_RELEASE); } if (!fn) return; rec_shading_rate(fn, ex->cb, !ex->color[0].image && ex->depth.image && (int)ex->depth.base_layer >= from ? 2 : 1); @@ -5489,12 +5533,25 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { // only by the draw it was filled for, at the same record address, while no other op was replayed since // (vk_exec_brk). In the village 42 of the 45 draws a frame are served that way: 3 gaps with these copies a frame // where there were 41 (0.04 ms of GPU time for 0.22), and about 40 passes fewer to begin and end. +// The entries filled ahead are the draw context's (DrawCtx.ma): a chunk of a parallel group (vk_exec.c) fills them for +// the draws that follow its own (vk_exec_mesh_run reads on from the op it records), and an entry it filled for a draw +// another chunk records is left unused. enum { MESH_AHEAD = 64 }; -static struct { const p_draw_mesh_ind *d; uint64_t ind; VkDeviceSize at; } mesh_ahead[MESH_AHEAD]; -static int mesh_ahead_n, mesh_ahead_i; -static uint32_t mesh_ahead_brk; +_Static_assert(MESH_AHEAD == sizeof dc_main.ma / sizeof dc_main.ma[0], "DrawCtx.ma holds MESH_AHEAD entries"); static uint64_t mesh_ahead_taken, mesh_ahead_own; // since the start: draws served by an entry filled ahead; the others +// The ring, made before the chunks of a group record (vk_exec_draw_group_begin) or at the first indirect mesh draw. +static int mesh_ring_make(void) { + if (mesh_ring) return 1; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING, + .usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + VkBuffer b; + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &b, &mesh_ring_alloc, NULL) != VK_SUCCESS) return 0; + __atomic_store_n(&mesh_ring, b, __ATOMIC_RELEASE); + return 1; +} + static int mesh_ahead_on(void) { static int mode = -1; if (mode < 0) mode = rt_knob("MESH_AHEAD", "1"); @@ -5502,6 +5559,7 @@ static int mesh_ahead_on(void) { } static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { + DrawCtx *const cx = ex->dc; if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) { char stops[160]; vk_exec_mesh_run_stops(stops, sizeof stops); @@ -5509,32 +5567,24 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan "their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1), (unsigned long long)mesh_ahead_taken, stops); } - if (mesh_ahead_i < mesh_ahead_n) { - if (mesh_ahead[mesh_ahead_i].d == d && mesh_ahead[mesh_ahead_i].ind == d->ind && mesh_ahead_brk == vk_exec_brk) { - *at = mesh_ahead[mesh_ahead_i++].at; - mesh_ahead_taken++; + if (cx->ma_i < cx->ma_n) { + if (cx->ma[cx->ma_i].d == d && cx->ma[cx->ma_i].ind == d->ind && cx->ma_brk == vk_exec_brk_now()) { + *at = cx->ma[cx->ma_i++].at; + __atomic_add_fetch(&mesh_ahead_taken, 1, __ATOMIC_RELAXED); return 1; } - mesh_ahead_n = mesh_ahead_i = 0; // the walk went another way: the entries left are never drawn from + cx->ma_n = cx->ma_i = 0; // the walk went another way: the entries left are never drawn from } - mesh_ahead_own++; - if (!mesh_ring) { - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 12ull * MESH_RING, - .usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; - VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; - if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &mesh_ring, &mesh_ring_alloc, NULL) != VK_SUCCESS) { - mesh_ring = VK_NULL_HANDLE; - return 0; - } - } - *at = 12ull * (mesh_ring_next++ % MESH_RING); + __atomic_add_fetch(&mesh_ahead_own, 1, __ATOMIC_RELAXED); + if (!__atomic_load_n(&mesh_ring, __ATOMIC_ACQUIRE) && (vk_exec_in_chunk() || !mesh_ring_make())) return 0; + *at = 12ull * (__atomic_fetch_add(&mesh_ring_next, 1, __ATOMIC_RELAXED) % MESH_RING); vk_exec_barrier(ex); // ends the pass (the next draw opens it again) and orders the culling's writes before the copy vk_gpu_gap_note(VK_GAP_MESH_RING, 12); vkCmdFillBuffer(ex->cb, mesh_ring, *at + 4, 8, 1); VkBufferCopy cp = {src->offset, *at, 4}; vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); mesh_count_note(ex, src); - mesh_ahead_n = mesh_ahead_i = 0; + cx->ma_n = cx->ma_i = 0; if (mesh_ahead_on()) { const void *run[MESH_AHEAD]; int n = vk_exec_mesh_run(d, run, MESH_AHEAD); @@ -5543,17 +5593,17 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan VkMemSpan ip; if (!vk_mem_resolve(f->ind, 12, &ip) || ip.offset % 4) break; // that draw reports it when the walk gets there VK_MEM_USE(ip); - VkDeviceSize fa = 12ull * (mesh_ring_next++ % MESH_RING); + VkDeviceSize fa = 12ull * (__atomic_fetch_add(&mesh_ring_next, 1, __ATOMIC_RELAXED) % MESH_RING); vk_gpu_gap_note(VK_GAP_MESH_RING, 12); vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1); VkBufferCopy fc = {ip.offset, fa, 4}; vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc); mesh_count_note(ex, &ip); ex->recorded += 2; - mesh_ahead[mesh_ahead_n].d = f, mesh_ahead[mesh_ahead_n].ind = f->ind, mesh_ahead[mesh_ahead_n].at = fa; - mesh_ahead_n++; + cx->ma[cx->ma_n].d = f, cx->ma[cx->ma_n].ind = f->ind, cx->ma[cx->ma_n].at = fa; + cx->ma_n++; } - mesh_ahead_brk = vk_exec_brk; + cx->ma_brk = vk_exec_brk_now(); } VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, @@ -5568,13 +5618,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan // DrawMeshTasksIndirect: one draw from its ring entry (mesh_ring_entry); x2-x4 of the call vary like leftovers, so they // are not read as a count or stride. static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { - static PFN_vkCmdDrawMeshTasksIndirectEXT fn; - static int looked; - if (!looked) { - looked = 1; - if (vk_ctx()->has_mesh_shader) - fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vk_dpa(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT"); - } + PFN_vkCmdDrawMeshTasksIndirectEXT fn = draw_mesh_ind_fn(); int fallback = vk_ctx()->mesh_fallback; if (!fn && !fallback) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); static int on = -1; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index df63b238..e3dde4fd 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -204,7 +204,10 @@ static Pending *pending_slot(uint64_t addr, int insert) { // report into guest memory. Slots come from one ring; the completion thread reads and host-resets them // (hostQueryReset, Vulkan 1.2), so the replay records no reset commands. A full ring (no slot read back yet) leaves the // draw uncounted and the segment's count at least 1, as does a query still unavailable after its fence. Without hostQueryReset the counts stay 0. -#define OQ_N 4096 +// The ring holds about four frames of Hyrule Field's queries (about 3,500 a frame): the replay workers' chunks take +// blocks of slots sized to their draws (vk_exec_xfer_slots), more than they use, and with 4096 the ring came round to +// slots the completion thread had not reset yet, leaving draws uncounted. +#define OQ_N 16384 #define LOG_ONCE(...) do { static volatile char once_; if (!__atomic_exchange_n(&once_, 1, __ATOMIC_RELAXED)) rt_log(__VA_ARGS__); } while (0) static VkQueryPool oq_pool; static int oq_state; // 0 not set up, 1 counting available, -1 unavailable @@ -228,21 +231,24 @@ typedef struct { uint64_t addr, seq; int reset; volatile int32_t lost; uint32_t // the counting as the stream does from its chunk's first op (vk_exec_xfer_chunk_state): XW.counting and the segment its // closed queries belong to, the next one at or after its position in the stream (XW.seg; past the group's last, the // pending list oq_seg, which a later reset or report takes). It keeps the query open in its own pass (XW.open), takes -// ring slots from its chunk's block and adds the slots it closes under oq_seg_lock. XW.cond: the decisions of its chunk's -// render conditions, made by the front pass in the order of the stream. +// ring slots from its chunk's block and keeps the slots it closes with their segment in its chunk's list (CP), which +// vk_exec_xfer_group_merge puts into the segments once the group is done, chunk after chunk: under a lock per slot, the +// workers waited for each other at every query. XW.cond: the decisions of its chunk's render conditions, made by the +// front pass in the order of the stream. // The blocks (vk_exec_xfer_slots): the front pass gives each chunk as many slots as it has draws that count samples and // render, one after the other, so the slots follow the stream as the replay's own do and the completion thread reads // and resets them in long runs (oq_take). Taken from the ring at once, the slots of a group's chunks interleaved: four // times the result calls and a reset per slot, 3.5 ms more of the wait after a frame's first submission (RX 7900 XT, // Hyrule Field, 3 workers). A chunk that needs more than its block takes them from the ring. static _Thread_local struct { - int on, open, counting, reported, keep; + int on, open, counting, reported, keep, chunk; uint32_t seg, qnext, qend; const uint8_t *cond; uint32_t ncond, i; } XW = {.open = -1}; -static PlatMutex oq_seg_lock = PLAT_MUTEX_INIT; static struct { int on; OqSegment **s; uint32_t n, cap; } GS; // the front pass's segments of the group +#define CP_CHUNKS 32 +static struct { uint32_t *seg, *q, n, cap; } CP[CP_CHUNKS]; // per chunk index: closed slots and their segment void vk_exec_xfer_group(int on) { // on: a new group's front pass begins; off: it ended, the list stays if (on) GS.n = 0; @@ -261,8 +267,10 @@ uint64_t vk_exec_xfer_chunk_state(void) { return (uint64_t)GS.n << 8 | (uint64_t)(oq_counting & 1) | (uint64_t)(oq_reported & 1) << 1 | (uint64_t)(oq_keep & 1) << 2; } -void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots) { - XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0; +void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots, + int chunk) { + if (chunk < 0 || chunk >= CP_CHUNKS) rt_fatal("vk: replay worker chunk %d of at most %d", chunk, CP_CHUNKS); + XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0, XW.chunk = chunk; XW.qnext = slot0, XW.qend = slot0 + nslots; XW.counting = (int)(state & 1), XW.reported = (int)(state >> 1 & 1), XW.keep = (int)(state >> 2 & 1); XW.seg = (uint32_t)(state >> 8); @@ -270,22 +278,37 @@ void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t s // Adds closed slot q to the segment a worker's queries belong to now, or the pending list. static void seg_add(uint32_t q) { - plat_mutex_lock(&oq_seg_lock); - if (XW.seg < GS.n) { - OqSegment *s = GS.s[XW.seg]; - if (s->n == s->cap) { - s->cap = s->cap ? s->cap * 2 : 16; - if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); - } - s->q[s->n++] = q; - } else { - if (oq_nseg == oq_capseg) { - oq_capseg = oq_capseg ? oq_capseg * 2 : 64; - if (!(oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg))) rt_fatal("vk: out of memory"); - } - oq_seg[oq_nseg++] = q; + __typeof__(CP[0]) *l = &CP[XW.chunk]; + if (l->n == l->cap) { + l->cap = l->cap ? l->cap * 2 : 256; + if (!(l->seg = realloc(l->seg, sizeof *l->seg * l->cap)) || !(l->q = realloc(l->q, sizeof *l->q * l->cap))) + rt_fatal("vk: out of memory"); + } + l->seg[l->n] = XW.seg, l->q[l->n++] = q; +} + +void vk_exec_xfer_group_merge(int nchunks) { + for (int c = 0; c < nchunks && c < CP_CHUNKS; c++) { + __typeof__(CP[0]) *l = &CP[c]; + for (uint32_t i = 0; i < l->n; i++) { + uint32_t q = l->q[i]; + if (l->seg[i] < GS.n) { + OqSegment *s = GS.s[l->seg[i]]; + if (s->n == s->cap) { + s->cap = s->cap ? s->cap * 2 : 16; + if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); + } + s->q[s->n++] = q; + } else { + if (oq_nseg == oq_capseg) { + oq_capseg = oq_capseg ? oq_capseg * 2 : 64; + if (!(oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg))) rt_fatal("vk: out of memory"); + } + oq_seg[oq_nseg++] = q; + } + } + l->n = 0; } - plat_mutex_unlock(&oq_seg_lock); } static int oq_ready(void) { diff --git a/runtime/vk_mvprev.c b/runtime/vk_mvprev.c index 2b65767b..682fd6d9 100644 --- a/runtime/vk_mvprev.c +++ b/runtime/vk_mvprev.c @@ -81,7 +81,7 @@ int vk_mvprev_on(void) { return P.on > 0; } void vk_mvprev_want(uint32_t index_mask) { static int fixed = -1; if (fixed < 0) fixed = !rt_knob("MVPREV_GROW", "1"); // NX2NAVI_MVPREV_GROW=0: c5, c7 and c14 only (test) - if (!fixed) P.mask |= index_mask & ((1u << BANKS) - 1); + if (!fixed) __atomic_fetch_or(&P.mask, index_mask & ((1u << BANKS) - 1), __ATOMIC_RELAXED); // chunks at once } uint64_t vk_mvprev_step(int stage, int index, uint64_t size) {