mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 04:59:57 -04:00
Keep samples-passed counters inside parallel groups, and lock the shared tables only while a chunk records.
Samples-passed resets and reports ended most groups in Hyrule Field (the game reports twice without a reset there, so counting stays on): the front pass now replays them in the order of the stream, makes each occlusion segment and queues its completion callback in that order, and each worker follows the counting from its chunk's first op and adds the queries it closes to the segment the stream gives them. Per-object motion takes a lock instead of waiting for the earlier chunks: its list's order does not change the picture. The locks the chunks share are taken only inside a chunk, as outside groups the submitting thread alone replays; taken always, they cost the replay about 1 ms a frame. The perf line counts the draws by what ended their group. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1456,10 +1456,11 @@ static int scene_color(VkFormat f);
|
||||
static void dlaa_draw_locked(VkExec *ex);
|
||||
void vk_dlaa_draw(VkExec *ex) {
|
||||
if (!dl_active() || D.ended) return;
|
||||
static PlatMutex lock = PLAT_MUTEX_INIT; // the replay workers' scene draws count at once
|
||||
plat_mutex_lock(&lock);
|
||||
static PlatMutex lock = PLAT_MUTEX_INIT; // the replay workers' scene draws count at once (vk_exec.c chunks)
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&lock);
|
||||
dlaa_draw_locked(ex);
|
||||
plat_mutex_unlock(&lock);
|
||||
if (lk) plat_mutex_unlock(&lock);
|
||||
}
|
||||
|
||||
static void dlaa_draw_locked(VkExec *ex) {
|
||||
|
||||
@@ -55,6 +55,7 @@ typedef struct {
|
||||
const op_hdr *h; // its first op
|
||||
uint32_t nops; // the ops it records (JUMPs not counted)
|
||||
uint32_t cond0, ncond; // its render conditions' decisions in G.cond
|
||||
uint64_t oq; // the occlusion counting at its first op (vk_exec_xfer_chunk_state)
|
||||
int idx; // in its group: 0 runs the group's pass hooks
|
||||
VkCommandBuffer cb;
|
||||
int recorded;
|
||||
@@ -1774,6 +1775,7 @@ static struct {
|
||||
RwWorker *w[RW_MAX];
|
||||
const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small
|
||||
uint64_t groups, group_draws, chunks, serial_draws, wait_ns; // since the last perf line
|
||||
uint64_t end_draws[5]; // draws of groups by what ended them (group_peek's GE_*), since the last perf line
|
||||
volatile uint64_t turns, turn_ns, redos;
|
||||
} RW = {.n = -1};
|
||||
|
||||
@@ -1844,24 +1846,29 @@ static void views_release(VkExec *ex, uint64_t fence) {
|
||||
ex->nviews = 0;
|
||||
}
|
||||
|
||||
// The draws of the group that starts at h (it ends at *end), reading at most GROUP_PEEK ops. *serial: it holds a
|
||||
// mesh draw (those are replayed on the submitting thread).
|
||||
static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end) {
|
||||
// What ended a group (RW.end_draws): the end of the run, a render target change, an inline uniform update, a mesh
|
||||
// draw, the read-ahead limit.
|
||||
enum { GE_RUN, GE_TARGET, GE_UB, GE_MESH, GE_LIMIT, GE_N };
|
||||
static const char *const ge_name[GE_N] = {"run end", "target change", "uniform update", "mesh draw", "read limit"};
|
||||
|
||||
// The draws of the group that starts at h (it ends at *end, for the reason *why), reading at most GROUP_PEEK ops.
|
||||
static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end, int *why) {
|
||||
uint32_t draws = 0;
|
||||
int begun = 0;
|
||||
const p_set_rt *rt = &ex->rt;
|
||||
*why = GE_LIMIT;
|
||||
for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) {
|
||||
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the walk reports it
|
||||
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) { *why = GE_RUN; break; } // the walk reports it
|
||||
const void *a = nvn_op_payload(h);
|
||||
if (h->op == OP_JUMP) {
|
||||
h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target;
|
||||
continue;
|
||||
}
|
||||
if (!run_op(h->op) || h->op == OP_UPDATE_UB || h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND ||
|
||||
vk_exec_xfer_serial(h->op, a))
|
||||
break;
|
||||
if (!run_op(h->op)) { *why = GE_RUN; break; }
|
||||
if (h->op == OP_UPDATE_UB) { *why = GE_UB; break; }
|
||||
if (h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND) { *why = GE_MESH; break; }
|
||||
if (h->op == OP_SET_RT && memcmp(rt, a, sizeof *rt)) {
|
||||
if (begun) break; // another pass: the next group
|
||||
if (begun) { *why = GE_TARGET; break; } // another pass: the next group
|
||||
rt = a;
|
||||
}
|
||||
if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) draws++, begun = 1;
|
||||
@@ -1875,7 +1882,7 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end
|
||||
static Chunk *chunk_open(const VkExec *ex, const op_hdr *h, int idx) {
|
||||
Chunk *c = &G.c[idx];
|
||||
memcpy(&c->ex, ex, sizeof *ex);
|
||||
c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx;
|
||||
c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx, c->oq = vk_exec_xfer_chunk_state();
|
||||
return c;
|
||||
}
|
||||
|
||||
@@ -1886,7 +1893,7 @@ static void chunk_record(Chunk *c, int pool) {
|
||||
VkExec *ex = &c->ex, *was = x_cur;
|
||||
if (pool < RW.n) ex->dc = RW.w[pool]->dc; // the submitting thread records with its own (the state's)
|
||||
x_cur = ex, rw_chunk = c, rw_ordered = !c->idx;
|
||||
vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond);
|
||||
vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond, c->oq);
|
||||
ex->cb = vk_sync_worker_begin(pool);
|
||||
// the replay's command buffer before the group ended behind a full barrier, with every image in GENERAL
|
||||
ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL;
|
||||
@@ -1918,7 +1925,7 @@ static void chunk_record(Chunk *c, int pool) {
|
||||
free(ex->views);
|
||||
ex->views = NULL;
|
||||
c->cb = ex->cb, c->recorded = ex->recorded;
|
||||
vk_exec_xfer_worker(0, NULL, 0);
|
||||
vk_exec_xfer_worker(0, NULL, 0, 0);
|
||||
x_cur = was, rw_chunk = NULL;
|
||||
__atomic_store_n(&c->done, 1, __ATOMIC_RELEASE);
|
||||
}
|
||||
@@ -1936,8 +1943,10 @@ static void group_take(int pool) {
|
||||
// replays the group itself, without reading ahead again before its end).
|
||||
static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) {
|
||||
const op_hdr *end;
|
||||
uint32_t draws = group_peek(ex, h, &end);
|
||||
int why;
|
||||
uint32_t draws = group_peek(ex, h, &end, &why);
|
||||
if (end == h) return h; // an op the submitting thread replays itself
|
||||
RW.end_draws[why] += draws;
|
||||
int n = (int)(draws / CHUNK_DRAWS);
|
||||
if (n > RW.n + 1) n = RW.n + 1;
|
||||
if (n > GROUP_CHUNKS) n = GROUP_CHUNKS;
|
||||
@@ -1952,6 +1961,7 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) {
|
||||
uint32_t per = (draws + (uint32_t)n - 1) / (uint32_t)n, cd = 0;
|
||||
int nc = 1, cut = 0;
|
||||
G.ncond = 0;
|
||||
vk_exec_xfer_group(1);
|
||||
Chunk *c = chunk_open(ex, h, 0);
|
||||
for (const op_hdr *o = h; o != end;) {
|
||||
const void *a = nvn_op_payload(o);
|
||||
@@ -1982,6 +1992,7 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) {
|
||||
c->nops++;
|
||||
o = nvn_op_next(o);
|
||||
}
|
||||
vk_exec_xfer_group(0); // the segments stay listed for the workers until the next group
|
||||
// handed out: the workers and this thread take the chunks in order
|
||||
for (int i = 0; i < nc; i++) G.c[i].done = 0;
|
||||
G.hooks = 0;
|
||||
@@ -2206,6 +2217,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
|
||||
RW.groups ? (double)RW.chunks / (double)RW.groups : 0.0,
|
||||
RW.groups ? (double)RW.wait_ns / 1e6 / (double)RW.groups : 0.0, (double)RW.serial_draws / (double)n_subs,
|
||||
(double)turns / (double)n_subs, (double)turn_ns / 1e6 / (double)n_subs, (double)redos / (double)n_subs);
|
||||
char e[256];
|
||||
int o = 0;
|
||||
for (int k = 0; k < GE_N; k++)
|
||||
o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.0f", k ? "," : "", ge_name[k],
|
||||
(double)RW.end_draws[k] / (double)n_subs);
|
||||
rt_log("perf: replay workers: draws per submission by what ended their group:%s", e);
|
||||
memset(RW.end_draws, 0, sizeof RW.end_draws);
|
||||
}
|
||||
RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0;
|
||||
evo_held = evo_resumed = evo_forced = evo_switches = 0;
|
||||
|
||||
@@ -156,13 +156,14 @@ void vk_exec_draw_ctx_use(void *ctx);
|
||||
// vk_exec_draw.c, before the chunks of a parallel group are recorded (submitting thread): the stand-ins are made and the
|
||||
// texture pool set this submission took is brought up to date, which the chunks then only bind.
|
||||
void vk_exec_draw_group_begin(VkExec *ex);
|
||||
// vk_exec_xfer.c, parallel groups: vk_exec_xfer_serial says an op stays on the submitting thread (a samples-passed reset
|
||||
// or report); vk_exec_xfer_front is the front pass's CPU side of a counter report, render enable or render condition
|
||||
// (returns the condition's decision, 1 renders); vk_exec_xfer_worker turns the worker side on for the calling thread with
|
||||
// its chunk's decisions, or off.
|
||||
int vk_exec_xfer_serial(uint32_t op, const void *payload);
|
||||
// vk_exec_xfer.c, parallel groups: vk_exec_xfer_group turns the front pass on (it lists the group's occlusion segments)
|
||||
// or off; vk_exec_xfer_front replays a counter report or reset, render enable or render condition in the front pass
|
||||
// (returns the condition's decision, 1 renders); vk_exec_xfer_chunk_state is the occlusion counting at a chunk's first
|
||||
// op, which vk_exec_xfer_worker hands to the worker side with the chunk's decisions (on), or turns it off.
|
||||
void vk_exec_xfer_group(int on);
|
||||
int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload);
|
||||
void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond);
|
||||
uint64_t vk_exec_xfer_chunk_state(void);
|
||||
void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state);
|
||||
// vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex
|
||||
// buffers): the replay continues in a new one (vk_sync_split).
|
||||
void vk_exec_draw_new_cb(void);
|
||||
|
||||
@@ -292,9 +292,11 @@ void *vk_exec_draw_ctx_new(void) {
|
||||
|
||||
void vk_exec_draw_ctx_use(void *c) { dc = c ? c : &dc_main; }
|
||||
|
||||
static PlatMutex shader_lock = PLAT_MUTEX_INIT; // shader_tab: the replay workers look shaders up at once
|
||||
// Tables the chunks of a parallel group (vk_exec.c "replay workers") share, locked only while a chunk records: outside
|
||||
// groups the submitting thread alone replays, and these lookups run per draw.
|
||||
static PlatMutex shader_lock = PLAT_MUTEX_INIT; // shader_tab
|
||||
static PlatMutex rs_lock = PLAT_MUTEX_INIT; // rs_lists
|
||||
static PlatMutex pool_mt = PLAT_MUTEX_INIT; // pool_sync from a chunk of a parallel group
|
||||
static PlatMutex pool_mt = PLAT_MUTEX_INIT; // pool_sync from a chunk
|
||||
|
||||
static void cb_done(void *arg) { // completion thread
|
||||
CbRes *r = arg;
|
||||
@@ -1051,7 +1053,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
if (serial) {
|
||||
uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded
|
||||
set = shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)];
|
||||
plat_mutex_lock(&shader_lock);
|
||||
int lk = vk_exec_in_chunk(); // only the chunks of a parallel group look up at once
|
||||
if (lk) plat_mutex_lock(&shader_lock);
|
||||
for (int w = 0; w < SHADER_WAYS && set[w].used; w++) {
|
||||
ShaderEnt *e = &set[w];
|
||||
if (e->serial != serial || e->stage != st || memcmp(&e->a, &a, sizeof a)) continue;
|
||||
@@ -1061,10 +1064,10 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
set[0] = hit;
|
||||
}
|
||||
const GfxShader *found = set[0].s;
|
||||
plat_mutex_unlock(&shader_lock);
|
||||
if (lk) plat_mutex_unlock(&shader_lock);
|
||||
return found;
|
||||
}
|
||||
plat_mutex_unlock(&shader_lock);
|
||||
if (lk) plat_mutex_unlock(&shader_lock);
|
||||
}
|
||||
int gs = st == NVN_STAGE_VS ? GFX_STAGE_VS : st == NVN_STAGE_FS ? GFX_STAGE_FS : st == NVN_STAGE_TASK ? GFX_STAGE_TS
|
||||
: st == NVN_STAGE_MESH ? GFX_STAGE_MS : GFX_STAGE_CS;
|
||||
@@ -1078,10 +1081,11 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st,
|
||||
(unsigned long long)serial);
|
||||
else if (set) { // in front, the least recently used way drops out
|
||||
plat_mutex_lock(&shader_lock);
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&shader_lock);
|
||||
memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set);
|
||||
set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s};
|
||||
plat_mutex_unlock(&shader_lock);
|
||||
if (lk) plat_mutex_unlock(&shader_lock);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
@@ -3070,7 +3074,8 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si
|
||||
static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) {
|
||||
DrawCtx *const cx = ex->dc;
|
||||
if (info == cx->rs_last) return cx->rs_last_l;
|
||||
plat_mutex_lock(&rs_lock);
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&rs_lock);
|
||||
if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info);
|
||||
RsList *l = nvn_hm_get(&rs_lists, &info);
|
||||
if (!l) {
|
||||
@@ -3120,7 +3125,7 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) {
|
||||
memcpy(l->r, tmp, sizeof *tmp * n);
|
||||
nvn_hm_put(&rs_lists, &info, l);
|
||||
}
|
||||
plat_mutex_unlock(&rs_lock);
|
||||
if (lk) plat_mutex_unlock(&rs_lock);
|
||||
cx->rs_last = info, cx->rs_last_l = l;
|
||||
return l;
|
||||
}
|
||||
@@ -4602,14 +4607,21 @@ static void mo_emit(VkCommandBuffer cb) {
|
||||
}
|
||||
|
||||
static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0);
|
||||
// The chunks of a parallel group (vk_exec.c) add to the frame's list at once: its memos, statistics and list go under
|
||||
// mo_lock there. The order of the list does not change the picture: its draws write motion without blending, tested
|
||||
// against the scene depth they do not write.
|
||||
static PlatMutex mo_lock = PLAT_MUTEX_INIT;
|
||||
|
||||
static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off) {
|
||||
if (MO.failed || !vk_mvprev_on() || !ex->dc->B.st_valid) return;
|
||||
static int timed = -1;
|
||||
if (timed < 0) timed = rt_knob("MVOBJ_TIMES", "0"); // NX2NAVI_MVOBJ_TIMES=1: the summary's times (the clock reads cost)
|
||||
if (!timed) return mvobj_draw_(ex, d, ib, ib_off, 0);
|
||||
uint64_t t0 = kern_now_ns();
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&mo_lock);
|
||||
uint64_t t0 = timed ? kern_now_ns() : 0;
|
||||
mvobj_draw_(ex, d, ib, ib_off, t0);
|
||||
MO.ns_check += kern_now_ns() - t0;
|
||||
if (t0) MO.ns_check += kern_now_ns() - t0;
|
||||
if (lk) plat_mutex_unlock(&mo_lock);
|
||||
}
|
||||
|
||||
// NX2NAVI_MVOBJ_BANKS=1 (test): for scene draws, the words of each other uniform bank (c3, c6, c8-c16) that changed
|
||||
@@ -4749,7 +4761,6 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi
|
||||
int nu = gfx_shader_resources(cx->B.st.vs, GFX_RES_UBO, &ur), reads = 0;
|
||||
for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4;
|
||||
if (!reads) return;
|
||||
vk_exec_turn(); // a replay worker's chunk: the motion list follows the stream
|
||||
MO.cand++;
|
||||
uint64_t tp = t0 ? kern_now_ns() : 0;
|
||||
if (t0) MO.ns_part[0] += tp - t0, MO.n_part[0]++;
|
||||
|
||||
@@ -220,18 +220,58 @@ static uint32_t oq_nseg, oq_capseg;
|
||||
static int oq_lost; // replay: the segment missed a draw (ring full)
|
||||
static uint64_t oq_count; // completion thread: samples since the last reset
|
||||
|
||||
typedef struct { uint64_t addr, seq; int reset, lost; uint32_t n; uint32_t q[]; } OqSegment;
|
||||
typedef struct { uint64_t addr, seq; int reset; volatile int32_t lost; uint32_t n, cap; uint32_t *q; } OqSegment;
|
||||
|
||||
// The replay workers (vk_exec.c "parallel groups") record draws of one group at once. A samples-passed reset or report
|
||||
// never falls inside such a group (vk_exec_xfer_serial), so counting does not change while the workers record: each
|
||||
// keeps the query open in its own pass (XW.open), takes ring slots atomically and adds the slots it closes to the
|
||||
// segment the next report takes, under oq_seg_lock. XW.cond: the decisions of its chunk's render conditions, made by the
|
||||
// front pass in the order of the stream (vk_exec_xfer_front).
|
||||
static _Thread_local struct { int on, open; const uint8_t *cond; uint32_t ncond, i; } XW = {.open = -1};
|
||||
// The replay workers (vk_exec.c "parallel groups") record the draws of one group at once. The front pass replays a
|
||||
// group's samples-passed resets and reports in the order of the stream before the workers record (vk_exec_xfer_front):
|
||||
// each makes its segment, queued for the completion thread in that order, and they are listed in GS. A worker follows
|
||||
// the counting as the stream does from its chunk's first op (vk_exec_xfer_chunk_state): XW.counting and the segment its
|
||||
// closed queries belong to, the next one at or after its position in the stream (XW.seg; past the group's last, the
|
||||
// pending list oq_seg, which a later reset or report takes). It keeps the query open in its own pass (XW.open), takes
|
||||
// ring slots atomically and adds the slots it closes under oq_seg_lock. XW.cond: the decisions of its chunk's render
|
||||
// conditions, made by the front pass in the order of the stream.
|
||||
static _Thread_local struct {
|
||||
int on, open, counting, reported, keep;
|
||||
uint32_t seg;
|
||||
const uint8_t *cond;
|
||||
uint32_t ncond, i;
|
||||
} XW = {.open = -1};
|
||||
static PlatMutex oq_seg_lock = PLAT_MUTEX_INIT;
|
||||
static struct { int on; OqSegment **s; uint32_t n, cap; } GS; // the front pass's segments of the group
|
||||
|
||||
void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond) {
|
||||
void vk_exec_xfer_group(int on) { // on: a new group's front pass begins; off: it ended, the list stays
|
||||
if (on) GS.n = 0;
|
||||
GS.on = on;
|
||||
}
|
||||
|
||||
uint64_t vk_exec_xfer_chunk_state(void) {
|
||||
return (uint64_t)GS.n << 8 | (uint64_t)(oq_counting & 1) | (uint64_t)(oq_reported & 1) << 1 | (uint64_t)(oq_keep & 1) << 2;
|
||||
}
|
||||
|
||||
void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state) {
|
||||
XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0;
|
||||
XW.counting = (int)(state & 1), XW.reported = (int)(state >> 1 & 1), XW.keep = (int)(state >> 2 & 1);
|
||||
XW.seg = (uint32_t)(state >> 8);
|
||||
}
|
||||
|
||||
// Adds closed slot q to the segment a worker's queries belong to now, or the pending list.
|
||||
static void seg_add(uint32_t q) {
|
||||
plat_mutex_lock(&oq_seg_lock);
|
||||
if (XW.seg < GS.n) {
|
||||
OqSegment *s = GS.s[XW.seg];
|
||||
if (s->n == s->cap) {
|
||||
s->cap = s->cap ? s->cap * 2 : 16;
|
||||
if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory");
|
||||
}
|
||||
s->q[s->n++] = q;
|
||||
} else {
|
||||
if (oq_nseg == oq_capseg) {
|
||||
oq_capseg = oq_capseg ? oq_capseg * 2 : 64;
|
||||
if (!(oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg))) rt_fatal("vk: out of memory");
|
||||
}
|
||||
oq_seg[oq_nseg++] = q;
|
||||
}
|
||||
plat_mutex_unlock(&oq_seg_lock);
|
||||
}
|
||||
|
||||
static int oq_ready(void) {
|
||||
@@ -264,25 +304,27 @@ static void oq_close(VkExec *ex) {
|
||||
if (*open < 0) return;
|
||||
vkCmdEndQuery(ex->cb, oq_pool, (uint32_t)*open);
|
||||
ex->recorded++;
|
||||
if (XW.on) plat_mutex_lock(&oq_seg_lock);
|
||||
if (oq_nseg == oq_capseg) {
|
||||
oq_capseg = oq_capseg ? oq_capseg * 2 : 64;
|
||||
oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg);
|
||||
if (!oq_seg) rt_fatal("vk: out of memory");
|
||||
if (XW.on) seg_add((uint32_t)*open);
|
||||
else {
|
||||
if (oq_nseg == oq_capseg) {
|
||||
oq_capseg = oq_capseg ? oq_capseg * 2 : 64;
|
||||
oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg);
|
||||
if (!oq_seg) rt_fatal("vk: out of memory");
|
||||
}
|
||||
oq_seg[oq_nseg++] = (uint32_t)*open;
|
||||
}
|
||||
oq_seg[oq_nseg++] = (uint32_t)*open;
|
||||
if (XW.on) plat_mutex_unlock(&oq_seg_lock);
|
||||
*open = -1;
|
||||
}
|
||||
|
||||
void vk_exec_oq_draw(VkExec *ex) {
|
||||
int *open = XW.on ? &XW.open : &oq_open;
|
||||
if (!oq_counting || *open >= 0) return;
|
||||
if (!(XW.on ? XW.counting : oq_counting) || *open >= 0) return;
|
||||
// a worker takes its slot at once (a busy one is lost, not tried again): the others take theirs meanwhile
|
||||
uint32_t q = (XW.on ? __atomic_fetch_add(&oq_next, 1, __ATOMIC_RELAXED) : oq_next) % OQ_N;
|
||||
if (__atomic_load_n(&oq_busy[q], __ATOMIC_ACQUIRE)) {
|
||||
warn_once(W_QUERY, "occlusion query ring full: draws left uncounted");
|
||||
__atomic_store_n(&oq_lost, 1, __ATOMIC_RELAXED);
|
||||
if (XW.on && XW.seg < GS.n) __atomic_store_n(&GS.s[XW.seg]->lost, 1, __ATOMIC_RELAXED);
|
||||
else __atomic_store_n(&oq_lost, 1, __ATOMIC_RELAXED);
|
||||
return;
|
||||
}
|
||||
__atomic_store_n(&oq_busy[q], 1, __ATOMIC_RELAXED);
|
||||
@@ -402,16 +444,24 @@ static void oq_cb(void *arg) {
|
||||
write_counter(s->addr, oq_count), pending_written(s->addr, s->seq);
|
||||
}
|
||||
}
|
||||
free(s);
|
||||
free(s->q), free(s);
|
||||
}
|
||||
|
||||
// Hands the segment to the completion thread with a report (addr; 0: no CPU-visible memory) or a reset.
|
||||
static void oq_segment(VkExec *ex, uint64_t addr, int reset) {
|
||||
oq_close(ex);
|
||||
OqSegment *s = malloc(sizeof *s + sizeof *s->q * oq_nseg);
|
||||
if (!s) rt_fatal("vk: out of memory");
|
||||
OqSegment *s = calloc(1, sizeof *s);
|
||||
if (!s || (oq_nseg && !(s->q = malloc(sizeof *s->q * oq_nseg)))) rt_fatal("vk: out of memory");
|
||||
s->cap = oq_nseg;
|
||||
s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg;
|
||||
memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg);
|
||||
if (oq_nseg) memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg);
|
||||
if (GS.on) { // the front pass: the workers add the queries of the group's ops before it
|
||||
if (GS.n == GS.cap) {
|
||||
GS.cap = GS.cap ? GS.cap * 2 : 64;
|
||||
if (!(GS.s = realloc(GS.s, sizeof *GS.s * GS.cap))) rt_fatal("vk: out of memory");
|
||||
}
|
||||
GS.s[GS.n++] = s;
|
||||
}
|
||||
oq_nseg = 0, oq_lost = 0;
|
||||
vk_completion_add_result(ex->fence, oq_cb, s);
|
||||
}
|
||||
@@ -1005,23 +1055,13 @@ static void clear_tex(VkExec *ex, const p_clear_tex *p) {
|
||||
|
||||
// ---- parallel groups (vk_exec.c): the CPU side in the front pass, the rest in the workers
|
||||
|
||||
// 1 when the op has to be replayed on the submitting thread in the order of the stream: a samples-passed reset or report
|
||||
// ends a segment of occlusion counts, which the workers' queries are added to.
|
||||
int vk_exec_xfer_serial(uint32_t op, const void *payload) {
|
||||
int type;
|
||||
if (op == OP_REPORT_COUNTER) type = (int)((const p_report_counter *)payload)->type;
|
||||
else if (op == OP_RESET_COUNTER) type = (int)((const p_reset_counter *)payload)->type;
|
||||
else return 0;
|
||||
return type == NVN_COUNTER_TYPE_SAMPLES_PASSED && oq_ready();
|
||||
}
|
||||
|
||||
// The front pass, in the order of the stream, before the workers record: a counter report other than samples passed
|
||||
// (its completion callback and pending entry, which later conditions read), a render enable and a render condition
|
||||
// (decided here, on the reports replayed so far). Returns the condition's decision: 1 renders.
|
||||
// The front pass, in the order of the stream, before the workers record (vk_exec_xfer_group on): a counter report or
|
||||
// reset (completion callbacks, pending entries, the occlusion segments and counting), a render enable and a render
|
||||
// condition (decided here, on the reports replayed so far). Returns the condition's decision: 1 renders.
|
||||
int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload) {
|
||||
switch (op) {
|
||||
case OP_REPORT_COUNTER: report_counter(fx, payload); return 1; // not samples passed (vk_exec_xfer_serial)
|
||||
case OP_RESET_COUNTER: return 1; // other counters have nothing to reset
|
||||
case OP_REPORT_COUNTER: report_counter(fx, payload); return 1;
|
||||
case OP_RESET_COUNTER: reset_counter(fx, payload); return 1;
|
||||
case OP_RENDER_ENABLE: fx->render_disabled = !((const p_flag *)payload)->value; return 1;
|
||||
case OP_RENDER_COND:
|
||||
fx->render_cond = *(const p_render_cond *)payload;
|
||||
@@ -1038,7 +1078,20 @@ int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload) {
|
||||
static void xfer_worker(VkExec *ex, uint32_t op, const void *payload) {
|
||||
if (op != OP_RENDER_ENABLE && vk_contact_pending()) vk_exec_turn(), vk_contact_flush(ex);
|
||||
switch (op) {
|
||||
case OP_REPORT_COUNTER: case OP_RESET_COUNTER: break;
|
||||
case OP_REPORT_COUNTER: case OP_RESET_COUNTER: {
|
||||
int type = op == OP_REPORT_COUNTER ? (int)((const p_report_counter *)payload)->type
|
||||
: (int)((const p_reset_counter *)payload)->type;
|
||||
if (type != NVN_COUNTER_TYPE_SAMPLES_PASSED || !oq_ready()) break;
|
||||
oq_close(ex); // into the segment this op ends; the next queries go to the next one
|
||||
XW.seg++;
|
||||
if (op == OP_RESET_COUNTER) XW.counting = 1, XW.reported = 0;
|
||||
else {
|
||||
if (XW.reported) XW.keep = 1; // as oq_report
|
||||
XW.reported = 1;
|
||||
if (!XW.keep) XW.counting = 0;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case OP_RENDER_ENABLE: ex->render_disabled = !((const p_flag *)payload)->value; break;
|
||||
case OP_RENDER_COND:
|
||||
ex->render_cond = *(const p_render_cond *)payload;
|
||||
|
||||
@@ -2541,8 +2541,9 @@ void vk_mem_shadow_sync(uint64_t fence) {
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
|
||||
// The replay workers (vk_exec.c) bind and write from several threads at once: a mirror's first use in a submission
|
||||
// (creation, sync) and the notes of GPU writes go under use_lock. A use the submission already synced takes no lock.
|
||||
// The replay workers (vk_exec.c) bind and write from several threads at once: while a chunk records, a mirror's first
|
||||
// use in a submission (creation, sync) and the notes of GPU writes go under use_lock. A use the submission already
|
||||
// synced takes no lock, nor does the replay outside a parallel group (it alone records then).
|
||||
static PlatMutex use_lock = PLAT_MUTEX_INIT;
|
||||
|
||||
static void mem_use_locked(VkMemSpan *sp);
|
||||
@@ -2553,9 +2554,12 @@ void vk_mem_use(VkMemSpan *sp) {
|
||||
sp->buffer = r->mbuf;
|
||||
return;
|
||||
}
|
||||
plat_mutex_lock(&use_lock);
|
||||
// a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors)
|
||||
if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) return;
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&use_lock);
|
||||
mem_use_locked(sp);
|
||||
plat_mutex_unlock(&use_lock);
|
||||
if (lk) plat_mutex_unlock(&use_lock);
|
||||
}
|
||||
|
||||
static void mem_use_locked(VkMemSpan *sp) {
|
||||
@@ -2577,9 +2581,10 @@ void vk_mem_pick(VkMemSpan *sp) {
|
||||
PoolRec *r = sp->pool;
|
||||
if (!r || r->mstate != MS_ON) return;
|
||||
if (r->acc_sub != msub) {
|
||||
plat_mutex_lock(&use_lock);
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&use_lock);
|
||||
if (r->acc_sub != msub) mirror_sync(r);
|
||||
plat_mutex_unlock(&use_lock);
|
||||
if (lk) plat_mutex_unlock(&use_lock);
|
||||
}
|
||||
sp->buffer = r->mbuf;
|
||||
}
|
||||
@@ -3168,14 +3173,16 @@ static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) {
|
||||
range_add(&wbl, &nwbl, &cwbl, r, sp->offset + o, size - o); // what found no snapshot room
|
||||
}
|
||||
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) {
|
||||
plat_mutex_lock(&use_lock);
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&use_lock);
|
||||
gpu_wrote(sp, size, 0);
|
||||
plat_mutex_unlock(&use_lock);
|
||||
if (lk) plat_mutex_unlock(&use_lock);
|
||||
}
|
||||
void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) {
|
||||
plat_mutex_lock(&use_lock);
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&use_lock);
|
||||
gpu_wrote(sp, size, 1);
|
||||
plat_mutex_unlock(&use_lock);
|
||||
if (lk) plat_mutex_unlock(&use_lock);
|
||||
}
|
||||
|
||||
// Adds [a, b) to l, n sorted ranges that do not touch; returns the new count. Above cap ranges (l holds cap + 1) the two
|
||||
|
||||
@@ -138,9 +138,10 @@ static PlatMutex draw_lock = PLAT_MUTEX_INIT; // the replay workers' scene dra
|
||||
static void mvprev_draw_(VkExec *ex) {
|
||||
const VkExecTarget *z = &ex->depth;
|
||||
if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return;
|
||||
plat_mutex_lock(&draw_lock);
|
||||
int lk = vk_exec_in_chunk(); // only the chunks of a parallel group (vk_exec.c) note at once
|
||||
if (lk) plat_mutex_lock(&draw_lock);
|
||||
mvprev_draw_locked(ex);
|
||||
plat_mutex_unlock(&draw_lock);
|
||||
if (lk) plat_mutex_unlock(&draw_lock);
|
||||
}
|
||||
|
||||
static void mvprev_draw_locked(VkExec *ex) {
|
||||
|
||||
@@ -435,13 +435,14 @@ static PlatMutex drop_lock = PLAT_MUTEX_INIT; // the replay workers' shaders w
|
||||
|
||||
void vk_usnap_drop(uint64_t addr, uint64_t n) {
|
||||
if (!U.mode || !n) return;
|
||||
plat_mutex_lock(&drop_lock);
|
||||
int lk = vk_exec_in_chunk();
|
||||
if (lk) plat_mutex_lock(&drop_lock);
|
||||
if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr);
|
||||
U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence};
|
||||
if (U.on)
|
||||
for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++)
|
||||
if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, __atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED);
|
||||
plat_mutex_unlock(&drop_lock);
|
||||
if (lk) plat_mutex_unlock(&drop_lock);
|
||||
}
|
||||
|
||||
uint64_t vk_usnap_gen(void) { return U.gen; }
|
||||
|
||||
Reference in New Issue
Block a user