diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index cfdde613..1fc4d3c7 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -1456,10 +1456,11 @@ static int scene_color(VkFormat f); static void dlaa_draw_locked(VkExec *ex); void vk_dlaa_draw(VkExec *ex) { if (!dl_active() || D.ended) return; - static PlatMutex lock = PLAT_MUTEX_INIT; // the replay workers' scene draws count at once - plat_mutex_lock(&lock); + static PlatMutex lock = PLAT_MUTEX_INIT; // the replay workers' scene draws count at once (vk_exec.c chunks) + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&lock); dlaa_draw_locked(ex); - plat_mutex_unlock(&lock); + if (lk) plat_mutex_unlock(&lock); } static void dlaa_draw_locked(VkExec *ex) { diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 35758f64..4abc67ee 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -55,6 +55,7 @@ typedef struct { const op_hdr *h; // its first op uint32_t nops; // the ops it records (JUMPs not counted) uint32_t cond0, ncond; // its render conditions' decisions in G.cond + uint64_t oq; // the occlusion counting at its first op (vk_exec_xfer_chunk_state) int idx; // in its group: 0 runs the group's pass hooks VkCommandBuffer cb; int recorded; @@ -1774,6 +1775,7 @@ static struct { RwWorker *w[RW_MAX]; const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small uint64_t groups, group_draws, chunks, serial_draws, wait_ns; // since the last perf line + uint64_t end_draws[5]; // draws of groups by what ended them (group_peek's GE_*), since the last perf line volatile uint64_t turns, turn_ns, redos; } RW = {.n = -1}; @@ -1844,24 +1846,29 @@ static void views_release(VkExec *ex, uint64_t fence) { ex->nviews = 0; } -// The draws of the group that starts at h (it ends at *end), reading at most GROUP_PEEK ops. *serial: it holds a -// mesh draw (those are replayed on the submitting thread). -static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end) { +// What ended a group (RW.end_draws): the end of the run, a render target change, an inline uniform update, a mesh +// draw, the read-ahead limit. +enum { GE_RUN, GE_TARGET, GE_UB, GE_MESH, GE_LIMIT, GE_N }; +static const char *const ge_name[GE_N] = {"run end", "target change", "uniform update", "mesh draw", "read limit"}; + +// The draws of the group that starts at h (it ends at *end, for the reason *why), reading at most GROUP_PEEK ops. +static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end, int *why) { uint32_t draws = 0; int begun = 0; const p_set_rt *rt = &ex->rt; + *why = GE_LIMIT; for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) { - if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the walk reports it + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) { *why = GE_RUN; break; } // the walk reports it const void *a = nvn_op_payload(h); if (h->op == OP_JUMP) { h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; } - if (!run_op(h->op) || h->op == OP_UPDATE_UB || h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND || - vk_exec_xfer_serial(h->op, a)) - break; + if (!run_op(h->op)) { *why = GE_RUN; break; } + if (h->op == OP_UPDATE_UB) { *why = GE_UB; break; } + if (h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND) { *why = GE_MESH; break; } if (h->op == OP_SET_RT && memcmp(rt, a, sizeof *rt)) { - if (begun) break; // another pass: the next group + if (begun) { *why = GE_TARGET; break; } // another pass: the next group rt = a; } if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) draws++, begun = 1; @@ -1875,7 +1882,7 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end static Chunk *chunk_open(const VkExec *ex, const op_hdr *h, int idx) { Chunk *c = &G.c[idx]; memcpy(&c->ex, ex, sizeof *ex); - c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx; + c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx, c->oq = vk_exec_xfer_chunk_state(); return c; } @@ -1886,7 +1893,7 @@ static void chunk_record(Chunk *c, int pool) { VkExec *ex = &c->ex, *was = x_cur; if (pool < RW.n) ex->dc = RW.w[pool]->dc; // the submitting thread records with its own (the state's) x_cur = ex, rw_chunk = c, rw_ordered = !c->idx; - vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond); + vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond, c->oq); ex->cb = vk_sync_worker_begin(pool); // the replay's command buffer before the group ended behind a full barrier, with every image in GENERAL ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL; @@ -1918,7 +1925,7 @@ static void chunk_record(Chunk *c, int pool) { free(ex->views); ex->views = NULL; c->cb = ex->cb, c->recorded = ex->recorded; - vk_exec_xfer_worker(0, NULL, 0); + vk_exec_xfer_worker(0, NULL, 0, 0); x_cur = was, rw_chunk = NULL; __atomic_store_n(&c->done, 1, __ATOMIC_RELEASE); } @@ -1936,8 +1943,10 @@ static void group_take(int pool) { // replays the group itself, without reading ahead again before its end). static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { const op_hdr *end; - uint32_t draws = group_peek(ex, h, &end); + int why; + uint32_t draws = group_peek(ex, h, &end, &why); if (end == h) return h; // an op the submitting thread replays itself + RW.end_draws[why] += draws; int n = (int)(draws / CHUNK_DRAWS); if (n > RW.n + 1) n = RW.n + 1; if (n > GROUP_CHUNKS) n = GROUP_CHUNKS; @@ -1952,6 +1961,7 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { uint32_t per = (draws + (uint32_t)n - 1) / (uint32_t)n, cd = 0; int nc = 1, cut = 0; G.ncond = 0; + vk_exec_xfer_group(1); Chunk *c = chunk_open(ex, h, 0); for (const op_hdr *o = h; o != end;) { const void *a = nvn_op_payload(o); @@ -1982,6 +1992,7 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { c->nops++; o = nvn_op_next(o); } + vk_exec_xfer_group(0); // the segments stay listed for the workers until the next group // handed out: the workers and this thread take the chunks in order for (int i = 0; i < nc; i++) G.c[i].done = 0; G.hooks = 0; @@ -2206,6 +2217,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t RW.groups ? (double)RW.chunks / (double)RW.groups : 0.0, RW.groups ? (double)RW.wait_ns / 1e6 / (double)RW.groups : 0.0, (double)RW.serial_draws / (double)n_subs, (double)turns / (double)n_subs, (double)turn_ns / 1e6 / (double)n_subs, (double)redos / (double)n_subs); + char e[256]; + int o = 0; + for (int k = 0; k < GE_N; k++) + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.0f", k ? "," : "", ge_name[k], + (double)RW.end_draws[k] / (double)n_subs); + rt_log("perf: replay workers: draws per submission by what ended their group:%s", e); + memset(RW.end_draws, 0, sizeof RW.end_draws); } RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 7b574b56..6aba301d 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -156,13 +156,14 @@ void vk_exec_draw_ctx_use(void *ctx); // vk_exec_draw.c, before the chunks of a parallel group are recorded (submitting thread): the stand-ins are made and the // texture pool set this submission took is brought up to date, which the chunks then only bind. void vk_exec_draw_group_begin(VkExec *ex); -// vk_exec_xfer.c, parallel groups: vk_exec_xfer_serial says an op stays on the submitting thread (a samples-passed reset -// or report); vk_exec_xfer_front is the front pass's CPU side of a counter report, render enable or render condition -// (returns the condition's decision, 1 renders); vk_exec_xfer_worker turns the worker side on for the calling thread with -// its chunk's decisions, or off. -int vk_exec_xfer_serial(uint32_t op, const void *payload); +// vk_exec_xfer.c, parallel groups: vk_exec_xfer_group turns the front pass on (it lists the group's occlusion segments) +// or off; vk_exec_xfer_front replays a counter report or reset, render enable or render condition in the front pass +// (returns the condition's decision, 1 renders); vk_exec_xfer_chunk_state is the occlusion counting at a chunk's first +// op, which vk_exec_xfer_worker hands to the worker side with the chunk's decisions (on), or turns it off. +void vk_exec_xfer_group(int on); int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload); -void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond); +uint64_t vk_exec_xfer_chunk_state(void); +void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state); // vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex // buffers): the replay continues in a new one (vk_sync_split). void vk_exec_draw_new_cb(void); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index f3730817..08521d05 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -292,9 +292,11 @@ void *vk_exec_draw_ctx_new(void) { void vk_exec_draw_ctx_use(void *c) { dc = c ? c : &dc_main; } -static PlatMutex shader_lock = PLAT_MUTEX_INIT; // shader_tab: the replay workers look shaders up at once +// Tables the chunks of a parallel group (vk_exec.c "replay workers") share, locked only while a chunk records: outside +// groups the submitting thread alone replays, and these lookups run per draw. +static PlatMutex shader_lock = PLAT_MUTEX_INIT; // shader_tab static PlatMutex rs_lock = PLAT_MUTEX_INIT; // rs_lists -static PlatMutex pool_mt = PLAT_MUTEX_INIT; // pool_sync from a chunk of a parallel group +static PlatMutex pool_mt = PLAT_MUTEX_INIT; // pool_sync from a chunk static void cb_done(void *arg) { // completion thread CbRes *r = arg; @@ -1051,7 +1053,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded set = shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)]; - plat_mutex_lock(&shader_lock); + int lk = vk_exec_in_chunk(); // only the chunks of a parallel group look up at once + if (lk) plat_mutex_lock(&shader_lock); for (int w = 0; w < SHADER_WAYS && set[w].used; w++) { ShaderEnt *e = &set[w]; if (e->serial != serial || e->stage != st || memcmp(&e->a, &a, sizeof a)) continue; @@ -1061,10 +1064,10 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin set[0] = hit; } const GfxShader *found = set[0].s; - plat_mutex_unlock(&shader_lock); + if (lk) plat_mutex_unlock(&shader_lock); return found; } - plat_mutex_unlock(&shader_lock); + if (lk) plat_mutex_unlock(&shader_lock); } int gs = st == NVN_STAGE_VS ? GFX_STAGE_VS : st == NVN_STAGE_FS ? GFX_STAGE_FS : st == NVN_STAGE_TASK ? GFX_STAGE_TS : st == NVN_STAGE_MESH ? GFX_STAGE_MS : GFX_STAGE_CS; @@ -1078,10 +1081,11 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st, (unsigned long long)serial); else if (set) { // in front, the least recently used way drops out - plat_mutex_lock(&shader_lock); + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&shader_lock); memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set); set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s}; - plat_mutex_unlock(&shader_lock); + if (lk) plat_mutex_unlock(&shader_lock); } return s; } @@ -3070,7 +3074,8 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { DrawCtx *const cx = ex->dc; if (info == cx->rs_last) return cx->rs_last_l; - plat_mutex_lock(&rs_lock); + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&rs_lock); if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info); RsList *l = nvn_hm_get(&rs_lists, &info); if (!l) { @@ -3120,7 +3125,7 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { memcpy(l->r, tmp, sizeof *tmp * n); nvn_hm_put(&rs_lists, &info, l); } - plat_mutex_unlock(&rs_lock); + if (lk) plat_mutex_unlock(&rs_lock); cx->rs_last = info, cx->rs_last_l = l; return l; } @@ -4602,14 +4607,21 @@ static void mo_emit(VkCommandBuffer cb) { } static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0); +// The chunks of a parallel group (vk_exec.c) add to the frame's list at once: its memos, statistics and list go under +// mo_lock there. The order of the list does not change the picture: its draws write motion without blending, tested +// against the scene depth they do not write. +static PlatMutex mo_lock = PLAT_MUTEX_INIT; + static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off) { if (MO.failed || !vk_mvprev_on() || !ex->dc->B.st_valid) return; static int timed = -1; if (timed < 0) timed = rt_knob("MVOBJ_TIMES", "0"); // NX2NAVI_MVOBJ_TIMES=1: the summary's times (the clock reads cost) - if (!timed) return mvobj_draw_(ex, d, ib, ib_off, 0); - uint64_t t0 = kern_now_ns(); + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&mo_lock); + uint64_t t0 = timed ? kern_now_ns() : 0; mvobj_draw_(ex, d, ib, ib_off, t0); - MO.ns_check += kern_now_ns() - t0; + if (t0) MO.ns_check += kern_now_ns() - t0; + if (lk) plat_mutex_unlock(&mo_lock); } // NX2NAVI_MVOBJ_BANKS=1 (test): for scene draws, the words of each other uniform bank (c3, c6, c8-c16) that changed @@ -4749,7 +4761,6 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi int nu = gfx_shader_resources(cx->B.st.vs, GFX_RES_UBO, &ur), reads = 0; for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4; if (!reads) return; - vk_exec_turn(); // a replay worker's chunk: the motion list follows the stream MO.cand++; uint64_t tp = t0 ? kern_now_ns() : 0; if (t0) MO.ns_part[0] += tp - t0, MO.n_part[0]++; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index c88c8e5e..47b519e2 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -220,18 +220,58 @@ static uint32_t oq_nseg, oq_capseg; static int oq_lost; // replay: the segment missed a draw (ring full) static uint64_t oq_count; // completion thread: samples since the last reset -typedef struct { uint64_t addr, seq; int reset, lost; uint32_t n; uint32_t q[]; } OqSegment; +typedef struct { uint64_t addr, seq; int reset; volatile int32_t lost; uint32_t n, cap; uint32_t *q; } OqSegment; -// The replay workers (vk_exec.c "parallel groups") record draws of one group at once. A samples-passed reset or report -// never falls inside such a group (vk_exec_xfer_serial), so counting does not change while the workers record: each -// keeps the query open in its own pass (XW.open), takes ring slots atomically and adds the slots it closes to the -// segment the next report takes, under oq_seg_lock. XW.cond: the decisions of its chunk's render conditions, made by the -// front pass in the order of the stream (vk_exec_xfer_front). -static _Thread_local struct { int on, open; const uint8_t *cond; uint32_t ncond, i; } XW = {.open = -1}; +// The replay workers (vk_exec.c "parallel groups") record the draws of one group at once. The front pass replays a +// group's samples-passed resets and reports in the order of the stream before the workers record (vk_exec_xfer_front): +// each makes its segment, queued for the completion thread in that order, and they are listed in GS. A worker follows +// the counting as the stream does from its chunk's first op (vk_exec_xfer_chunk_state): XW.counting and the segment its +// closed queries belong to, the next one at or after its position in the stream (XW.seg; past the group's last, the +// pending list oq_seg, which a later reset or report takes). It keeps the query open in its own pass (XW.open), takes +// ring slots atomically and adds the slots it closes under oq_seg_lock. XW.cond: the decisions of its chunk's render +// conditions, made by the front pass in the order of the stream. +static _Thread_local struct { + int on, open, counting, reported, keep; + uint32_t seg; + const uint8_t *cond; + uint32_t ncond, i; +} XW = {.open = -1}; static PlatMutex oq_seg_lock = PLAT_MUTEX_INIT; +static struct { int on; OqSegment **s; uint32_t n, cap; } GS; // the front pass's segments of the group -void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond) { +void vk_exec_xfer_group(int on) { // on: a new group's front pass begins; off: it ended, the list stays + if (on) GS.n = 0; + GS.on = on; +} + +uint64_t vk_exec_xfer_chunk_state(void) { + return (uint64_t)GS.n << 8 | (uint64_t)(oq_counting & 1) | (uint64_t)(oq_reported & 1) << 1 | (uint64_t)(oq_keep & 1) << 2; +} + +void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state) { XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0; + XW.counting = (int)(state & 1), XW.reported = (int)(state >> 1 & 1), XW.keep = (int)(state >> 2 & 1); + XW.seg = (uint32_t)(state >> 8); +} + +// Adds closed slot q to the segment a worker's queries belong to now, or the pending list. +static void seg_add(uint32_t q) { + plat_mutex_lock(&oq_seg_lock); + if (XW.seg < GS.n) { + OqSegment *s = GS.s[XW.seg]; + if (s->n == s->cap) { + s->cap = s->cap ? s->cap * 2 : 16; + if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); + } + s->q[s->n++] = q; + } else { + if (oq_nseg == oq_capseg) { + oq_capseg = oq_capseg ? oq_capseg * 2 : 64; + if (!(oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg))) rt_fatal("vk: out of memory"); + } + oq_seg[oq_nseg++] = q; + } + plat_mutex_unlock(&oq_seg_lock); } static int oq_ready(void) { @@ -264,25 +304,27 @@ static void oq_close(VkExec *ex) { if (*open < 0) return; vkCmdEndQuery(ex->cb, oq_pool, (uint32_t)*open); ex->recorded++; - if (XW.on) plat_mutex_lock(&oq_seg_lock); - if (oq_nseg == oq_capseg) { - oq_capseg = oq_capseg ? oq_capseg * 2 : 64; - oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg); - if (!oq_seg) rt_fatal("vk: out of memory"); + if (XW.on) seg_add((uint32_t)*open); + else { + if (oq_nseg == oq_capseg) { + oq_capseg = oq_capseg ? oq_capseg * 2 : 64; + oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg); + if (!oq_seg) rt_fatal("vk: out of memory"); + } + oq_seg[oq_nseg++] = (uint32_t)*open; } - oq_seg[oq_nseg++] = (uint32_t)*open; - if (XW.on) plat_mutex_unlock(&oq_seg_lock); *open = -1; } void vk_exec_oq_draw(VkExec *ex) { int *open = XW.on ? &XW.open : &oq_open; - if (!oq_counting || *open >= 0) return; + if (!(XW.on ? XW.counting : oq_counting) || *open >= 0) return; // a worker takes its slot at once (a busy one is lost, not tried again): the others take theirs meanwhile uint32_t q = (XW.on ? __atomic_fetch_add(&oq_next, 1, __ATOMIC_RELAXED) : oq_next) % OQ_N; if (__atomic_load_n(&oq_busy[q], __ATOMIC_ACQUIRE)) { warn_once(W_QUERY, "occlusion query ring full: draws left uncounted"); - __atomic_store_n(&oq_lost, 1, __ATOMIC_RELAXED); + if (XW.on && XW.seg < GS.n) __atomic_store_n(&GS.s[XW.seg]->lost, 1, __ATOMIC_RELAXED); + else __atomic_store_n(&oq_lost, 1, __ATOMIC_RELAXED); return; } __atomic_store_n(&oq_busy[q], 1, __ATOMIC_RELAXED); @@ -402,16 +444,24 @@ static void oq_cb(void *arg) { write_counter(s->addr, oq_count), pending_written(s->addr, s->seq); } } - free(s); + free(s->q), free(s); } // Hands the segment to the completion thread with a report (addr; 0: no CPU-visible memory) or a reset. static void oq_segment(VkExec *ex, uint64_t addr, int reset) { oq_close(ex); - OqSegment *s = malloc(sizeof *s + sizeof *s->q * oq_nseg); - if (!s) rt_fatal("vk: out of memory"); + OqSegment *s = calloc(1, sizeof *s); + if (!s || (oq_nseg && !(s->q = malloc(sizeof *s->q * oq_nseg)))) rt_fatal("vk: out of memory"); + s->cap = oq_nseg; s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg; - memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg); + if (oq_nseg) memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg); + if (GS.on) { // the front pass: the workers add the queries of the group's ops before it + if (GS.n == GS.cap) { + GS.cap = GS.cap ? GS.cap * 2 : 64; + if (!(GS.s = realloc(GS.s, sizeof *GS.s * GS.cap))) rt_fatal("vk: out of memory"); + } + GS.s[GS.n++] = s; + } oq_nseg = 0, oq_lost = 0; vk_completion_add_result(ex->fence, oq_cb, s); } @@ -1005,23 +1055,13 @@ static void clear_tex(VkExec *ex, const p_clear_tex *p) { // ---- parallel groups (vk_exec.c): the CPU side in the front pass, the rest in the workers -// 1 when the op has to be replayed on the submitting thread in the order of the stream: a samples-passed reset or report -// ends a segment of occlusion counts, which the workers' queries are added to. -int vk_exec_xfer_serial(uint32_t op, const void *payload) { - int type; - if (op == OP_REPORT_COUNTER) type = (int)((const p_report_counter *)payload)->type; - else if (op == OP_RESET_COUNTER) type = (int)((const p_reset_counter *)payload)->type; - else return 0; - return type == NVN_COUNTER_TYPE_SAMPLES_PASSED && oq_ready(); -} - -// The front pass, in the order of the stream, before the workers record: a counter report other than samples passed -// (its completion callback and pending entry, which later conditions read), a render enable and a render condition -// (decided here, on the reports replayed so far). Returns the condition's decision: 1 renders. +// The front pass, in the order of the stream, before the workers record (vk_exec_xfer_group on): a counter report or +// reset (completion callbacks, pending entries, the occlusion segments and counting), a render enable and a render +// condition (decided here, on the reports replayed so far). Returns the condition's decision: 1 renders. int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload) { switch (op) { - case OP_REPORT_COUNTER: report_counter(fx, payload); return 1; // not samples passed (vk_exec_xfer_serial) - case OP_RESET_COUNTER: return 1; // other counters have nothing to reset + case OP_REPORT_COUNTER: report_counter(fx, payload); return 1; + case OP_RESET_COUNTER: reset_counter(fx, payload); return 1; case OP_RENDER_ENABLE: fx->render_disabled = !((const p_flag *)payload)->value; return 1; case OP_RENDER_COND: fx->render_cond = *(const p_render_cond *)payload; @@ -1038,7 +1078,20 @@ int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload) { static void xfer_worker(VkExec *ex, uint32_t op, const void *payload) { if (op != OP_RENDER_ENABLE && vk_contact_pending()) vk_exec_turn(), vk_contact_flush(ex); switch (op) { - case OP_REPORT_COUNTER: case OP_RESET_COUNTER: break; + case OP_REPORT_COUNTER: case OP_RESET_COUNTER: { + int type = op == OP_REPORT_COUNTER ? (int)((const p_report_counter *)payload)->type + : (int)((const p_reset_counter *)payload)->type; + if (type != NVN_COUNTER_TYPE_SAMPLES_PASSED || !oq_ready()) break; + oq_close(ex); // into the segment this op ends; the next queries go to the next one + XW.seg++; + if (op == OP_RESET_COUNTER) XW.counting = 1, XW.reported = 0; + else { + if (XW.reported) XW.keep = 1; // as oq_report + XW.reported = 1; + if (!XW.keep) XW.counting = 0; + } + break; + } case OP_RENDER_ENABLE: ex->render_disabled = !((const p_flag *)payload)->value; break; case OP_RENDER_COND: ex->render_cond = *(const p_render_cond *)payload; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index a7b32d8f..844789c3 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2541,8 +2541,9 @@ void vk_mem_shadow_sync(uint64_t fence) { plat_mutex_unlock(&shadow_lock); } -// The replay workers (vk_exec.c) bind and write from several threads at once: a mirror's first use in a submission -// (creation, sync) and the notes of GPU writes go under use_lock. A use the submission already synced takes no lock. +// The replay workers (vk_exec.c) bind and write from several threads at once: while a chunk records, a mirror's first +// use in a submission (creation, sync) and the notes of GPU writes go under use_lock. A use the submission already +// synced takes no lock, nor does the replay outside a parallel group (it alone records then). static PlatMutex use_lock = PLAT_MUTEX_INIT; static void mem_use_locked(VkMemSpan *sp); @@ -2553,9 +2554,12 @@ void vk_mem_use(VkMemSpan *sp) { sp->buffer = r->mbuf; return; } - plat_mutex_lock(&use_lock); + // a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors) + if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) return; + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&use_lock); mem_use_locked(sp); - plat_mutex_unlock(&use_lock); + if (lk) plat_mutex_unlock(&use_lock); } static void mem_use_locked(VkMemSpan *sp) { @@ -2577,9 +2581,10 @@ void vk_mem_pick(VkMemSpan *sp) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON) return; if (r->acc_sub != msub) { - plat_mutex_lock(&use_lock); + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&use_lock); if (r->acc_sub != msub) mirror_sync(r); - plat_mutex_unlock(&use_lock); + if (lk) plat_mutex_unlock(&use_lock); } sp->buffer = r->mbuf; } @@ -3168,14 +3173,16 @@ static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) { range_add(&wbl, &nwbl, &cwbl, r, sp->offset + o, size - o); // what found no snapshot room } void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { - plat_mutex_lock(&use_lock); + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&use_lock); gpu_wrote(sp, size, 0); - plat_mutex_unlock(&use_lock); + if (lk) plat_mutex_unlock(&use_lock); } void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { - plat_mutex_lock(&use_lock); + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&use_lock); gpu_wrote(sp, size, 1); - plat_mutex_unlock(&use_lock); + if (lk) plat_mutex_unlock(&use_lock); } // Adds [a, b) to l, n sorted ranges that do not touch; returns the new count. Above cap ranges (l holds cap + 1) the two diff --git a/runtime/vk_mvprev.c b/runtime/vk_mvprev.c index f2914b39..2b65767b 100644 --- a/runtime/vk_mvprev.c +++ b/runtime/vk_mvprev.c @@ -138,9 +138,10 @@ static PlatMutex draw_lock = PLAT_MUTEX_INIT; // the replay workers' scene dra static void mvprev_draw_(VkExec *ex) { const VkExecTarget *z = &ex->depth; if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return; - plat_mutex_lock(&draw_lock); + int lk = vk_exec_in_chunk(); // only the chunks of a parallel group (vk_exec.c) note at once + if (lk) plat_mutex_lock(&draw_lock); mvprev_draw_locked(ex); - plat_mutex_unlock(&draw_lock); + if (lk) plat_mutex_unlock(&draw_lock); } static void mvprev_draw_locked(VkExec *ex) { diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index b59d9b98..e9e8b4ab 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -435,13 +435,14 @@ static PlatMutex drop_lock = PLAT_MUTEX_INIT; // the replay workers' shaders w void vk_usnap_drop(uint64_t addr, uint64_t n) { if (!U.mode || !n) return; - plat_mutex_lock(&drop_lock); + int lk = vk_exec_in_chunk(); + if (lk) plat_mutex_lock(&drop_lock); if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr); U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence}; if (U.on) for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++) if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, __atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED); - plat_mutex_unlock(&drop_lock); + if (lk) plat_mutex_unlock(&drop_lock); } uint64_t vk_usnap_gen(void) { return U.gen; }