Record the large draw groups of a submission on several replay workers at once.

Stage 2 of NX2NAVI_REPLAY_WORKERS (default 0, the replay as before). A run of draws is cut into groups: a group ends
where the render targets change after its first draw or clear, and before an inline uniform update, a mesh draw and a
samples-passed counter op, which the submitting thread replays itself in the order of the stream. A group of 64 draws
or more is first walked by a front pass on the submitting thread: it applies the state and binds, settles the CPU side
of the timestamp reports and decides the render conditions in the order of the stream, and copies the replay state at
the first op of each chunk. The workers and the submitting thread then take the chunks in order and record each into a
command buffer of its own, which vk_sync_append puts into the part in the order of the stream.

The frame's pass hooks (contact shadows, DLSS) run in a group's first chunk, which the others start after. What still
depends on the order between chunks waits until every earlier chunk of the group is done: a texture lookup that would
make or take back an entry returns nothing and the chunk replays the op again in turn, and so do the contact shadows'
pass, per-object motion and the HDR pass's source. The tables the chunks share take locks (the shader table, the
descriptor lists, mirror use and GPU write notes, uniform snapshot drops, target frames, the DLAA camera votes and the
previous-frame motion notes); each chunk keeps its own occlusion query open, its own feedback memo and no pass marks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
epicmeme546
2026-11-11 17:10:59 -05:00
committed by chrg6
parent 4fc32e8d84
commit 7534193b62
12 changed files with 474 additions and 140 deletions

View File

@@ -531,6 +531,15 @@ const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view);
void vk_texture_lock(void);
void vk_texture_unlock(void);
const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view);
// The replay workers (vk_exec.c "replay workers"): a chunk's lookup that would make or take back a texture entry
// before the chunk's turn returns nothing and says so here; the chunk replays the op again in turn
// (vk_texture_deferred_take clears it). vk_exec_turn_due: the calling thread records a chunk whose earlier chunks may not
// be done yet; vk_exec_turn waits until they are (no-op elsewhere).
int vk_texture_deferred(void);
int vk_texture_deferred_take(void);
int vk_exec_turn_due(void);
int vk_exec_in_chunk(void); // the calling thread records a chunk of a parallel group
void vk_exec_turn(void);
VkSampler vk_sampler_by_id_locked(int id);
// The slot's sampler with nearest filtering and no anisotropy: a depth texture read without a compare lands on one texel
// on the Switch although the game binds a linear sampler. VK_NULL_HANDLE

View File

@@ -112,6 +112,8 @@ static void mem_barrier(VkCommandBuffer cb) {
vkCmdPipelineBarrier2(cb, &d);
}
int vk_contact_pending(void) { return C.pending; }
void vk_contact_flush(VkExec *ex) {
if (!C.pending) return;
C.pending = 0;

View File

@@ -1540,8 +1540,16 @@ void vk_dlaa_jitter_check(VkExec *ex) {
static int scene_color(VkFormat f);
static void dlaa_draw_locked(VkExec *ex);
void vk_dlaa_draw(VkExec *ex) {
if (!dl_active() || D.ended) return;
static PlatMutex lock = PLAT_MUTEX_INIT; // the replay workers' scene draws count at once
plat_mutex_lock(&lock);
dlaa_draw_locked(ex);
plat_mutex_unlock(&lock);
}
static void dlaa_draw_locked(VkExec *ex) {
static int survey = -1; // NX2NAVI_DLAA_SURVEY=1 (test): what is drawn into the armed light target
if (survey < 0) survey = rt_knob("DLAA_SURVEY", "0");
static uint64_t dfn[16]; // scene draws writing depth, by depth test (reversed depth: GREATER/GEQUAL)

View File

@@ -47,6 +47,32 @@ static int x_ready;
// it records none: vk_exec_inline_begin hands out its command buffer.
static _Thread_local VkExec *x_cur;
// ---- parallel groups (NX2NAVI_REPLAY_WORKERS; "replay workers" below). A chunk: a part of a group's ops that one
// thread records into a command buffer of its own, from the replay state the front pass left at its first op.
#define GROUP_CHUNKS 17 // chunks a group is cut into at most: RW_MAX workers and the submitting thread
typedef struct {
VkExec ex; // the state at its first op, then the state it records with
const op_hdr *h; // its first op
uint32_t nops; // the ops it records (JUMPs not counted)
uint32_t cond0, ncond; // its render conditions' decisions in G.cond
int idx; // in its group: 0 runs the group's pass hooks
VkCommandBuffer cb;
int recorded;
volatile int32_t done;
} Chunk;
static struct {
Chunk c[GROUP_CHUNKS];
int n;
uint8_t *cond; // the decisions of the group's render conditions, in stream order
uint32_t ncond, ccond;
volatile int32_t next; // the next chunk to take (huge between groups)
volatile int32_t hooks; // chunk 0 began its first pass (or ended): the others may start
volatile uint32_t gen; // bumped when a group is handed out
} G = {.next = 1 << 30};
static _Thread_local Chunk *rw_chunk; // the chunk the calling thread records, NULL outside one
static _Thread_local int rw_ordered; // every earlier chunk of its group is done
static PlatMutex tfr_lock = PLAT_MUTEX_INIT; // target frames (vk_target_frames.h) noted by several chunks at once
// ---- NX2NAVI_TRACE=gfx frame counters
VkExecFrame vk_exec_frame;
@@ -752,8 +778,13 @@ void vk_exec_barrier_log(uint64_t frame) {
void vk_exec_pass_begin(VkExec *ex) {
if (ex->in_pass) return;
resolve_rt(ex);
vk_contact_pass_begin(ex);
vk_dlaa_pass(ex); // DLAA runs before the bloom chain's first pass (vk_dlaa.c)
if (vk_texture_deferred()) return; // a chunk's lookup waits for its turn (chunk_record): the op is replayed again
// A parallel group's targets do not change: its pass hooks, which follow the frame pass by pass, run in its first
// chunk, which the others start after (chunk_record).
if (!rw_chunk || !rw_chunk->idx) {
vk_contact_pass_begin(ex);
vk_dlaa_pass(ex); // DLAA runs before the bloom chain's first pass (vk_dlaa.c)
}
// Learned targets (VkTexImage.fb_learned) are attached in the feedback layout
VkImage want[9];
VkImageAspectFlags wasp[9];
@@ -792,7 +823,7 @@ void vk_exec_pass_begin(VkExec *ex) {
}
// The start mark goes between the barrier and vkCmdBeginRendering, its end right after vkCmdEndRendering
// (vk_exec_pass_end): outside the pass, and without breaking it.
VkPassMark *m = ex->mark = vk_pass_mark_begin(ex->cb, VK_MARK_PASS, ex->fence);
VkPassMark *m = ex->mark = rw_chunk ? NULL : vk_pass_mark_begin(ex->cb, VK_MARK_PASS, ex->fence); // one ring
if (m) {
m->step = ex->mark_passes++;
m->w = ex->pass_w, m->h = ex->pass_h, m->d = ex->pass_layers;
@@ -814,6 +845,7 @@ void vk_exec_pass_begin(VkExec *ex) {
vk_exec_frame.passes++;
ex->recorded++;
ex->in_pass = 1;
if (rw_chunk && !rw_chunk->idx) __atomic_store_n(&G.hooks, 1, __ATOMIC_RELEASE);
}
void vk_exec_pass_end(VkExec *ex) {
@@ -835,6 +867,7 @@ void vk_exec_pass_end(VkExec *ex) {
bc_copy_back(ex, &ex->color[i]);
if (vk_dump_armed) vk_dump_pass_end(ex);
uint64_t stamp = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1;
if (rw_chunk) plat_mutex_lock(&tfr_lock);
for (int i = 0; i < 8; i++)
if (ex->color[i].image) {
TfrTarget ft = tfr_of(&ex->color[i]);
@@ -844,6 +877,7 @@ void vk_exec_pass_end(VkExec *ex) {
TfrTarget ft = tfr_of(&ex->depth);
vk_texture_gpu_written(ex->depth.tex), tfr_note(&ft, stamp);
}
if (rw_chunk) plat_mutex_unlock(&tfr_lock);
}
VkCommandBuffer vk_exec_inline_begin(int *reopen) {
@@ -947,6 +981,7 @@ static void clear_color(VkExec *ex, const p_clear_color *c) {
if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : vk_cond_clears_run++;
if (ex->render_disabled) return;
resolve_rt(ex);
if (vk_texture_deferred()) return;
int i = c->index;
if (i < 0 || i >= (int)ex->pass_ncolor || !ex->color[i].image) {
warn_once(W_CLEAR_TARGET, "colour clear of target %d, which is not bound or not renderable: skipped", i);
@@ -980,6 +1015,7 @@ static void clear_ds(VkExec *ex, const p_clear_ds *c) {
if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : (vk_cond_clears_run++, vk_cond_depth_run++);
if (ex->render_disabled) return;
resolve_rt(ex);
if (vk_texture_deferred()) return;
const VkExecTarget *t = &ex->depth;
if (!t->image) return;
VkImageAspectFlags asp = 0;
@@ -1482,7 +1518,7 @@ void vk_exec_ahead(VkExec *ex) {
rt_log("vk: a draw that finds its pipeline missing has the rest of its submission read ahead for the stage "
"libraries the later draws need: %s (NX2NAVI_PIPE_AHEAD)", AH.on ? "on" : "off");
}
if (!AH.on || !AH.live || evo_in || AH.seq == vk_exec_submit_seq) return;
if (!AH.on || !AH.live || evo_in || rw_chunk || AH.seq == vk_exec_submit_seq) return; // not in a chunk
AH.seq = vk_exec_submit_seq;
uint64_t t0 = kern_now_ns();
uint32_t ops = 0;
@@ -1685,24 +1721,30 @@ static void op_one(VkExec *ex, const op_hdr *h) {
}
}
// ---- replay workers (NX2NAVI_REPLAY_WORKERS, default 0; stage 1 of recording a submission on several threads)
// A submission's draws come in runs: between two ops that have to stay in the replay's order on its own thread
// (dispatches, copies, texture and buffer clears, events, the game's barriers, calls and the end of a command buffer)
// there are only state, binds, draws and the ops a draw run carries (counter reports and resets, render conditions,
// inline uniform updates, render target changes and attachment clears). Hyrule Field, standing (NX2NAVI_REPLAY_RUNS,
// 2026-11-11): about 3,500 draws a frame in 82 runs, 96% of them in runs of 16 draws or more and 67% in runs of 256
// or more, with a counter report or reset beside nearly every draw and a render condition beside every second one.
// With NX2NAVI_REPLAY_WORKERS set, a run of RUN_MIN draws or more is recorded by a worker thread: it takes a copy of
// the replay state, records the run into a command buffer of its own pool (vk_sync_worker_begin) with a draw context
// of its own (vk_exec_draw.c DrawCtx), ends it behind a full barrier, and hands the state the run left back. The
// replay's command buffer up to the run ended behind a full barrier too, and vk_sync_append puts the worker's command
// buffer after it in the same part, so the GPU runs everything in the order of the op stream. Stage 1 has one worker
// and the submitting thread waits for each run: the order of every side effect is the replay's own, and what it shows
// is whether a run recorded apart, from a copy of the state, gives the same pictures. Later stages record the runs
// of a submission on several workers at once.
// ---- replay workers (NX2NAVI_REPLAY_WORKERS, default 0: the replay records every op on the submitting thread)
// A submission's draws come in runs between ops that have to stay in the replay's order on its own thread (dispatches,
// copies, texture and buffer clears, events, the game's barriers, calls). Hyrule Field, standing (NX2NAVI_REPLAY_RUNS,
// 2026-11-11): about 3,500 draws a frame in 82 runs, 96% of them in runs of 16 draws or more and 67% in runs of 256 or
// more, with a timestamp report beside nearly every draw and a render condition beside every second one.
// A run is cut into groups: a group ends where the render targets change after its first draw or clear, and before an
// inline uniform update, a mesh draw and a samples-passed counter op, which the submitting thread replays itself in
// the order of the stream like everything outside groups. A group of 2 * CHUNK_DRAWS draws or more is recorded by
// several threads at once:
// - the front pass, on the submitting thread, walks the group in the order of the stream before anything of it is
// recorded: it applies the state and binds, settles the CPU side of counter reports and decides the render conditions
// (vk_exec_xfer_front), and copies the replay state at the first op of each chunk;
// - the chunks are taken in order by the workers and the submitting thread, each recording into a command buffer of its
// own pool with a draw context of its own, from its copy of the state, behind the full barrier that ended the
// replay's command buffer before the group, and ending behind a full barrier of its own. vk_sync_append puts them
// after the replay's command buffer in the same part, in the order of the stream, and the replay goes on in a new one.
// The frame's pass hooks (vk_contact_pass_begin, vk_dlaa_pass) follow the frame pass by pass: they run in chunk 0,
// which reaches the group's pass first, and the others start after that pass began; groups follow one another.
// What still depends on the order between chunks waits for its turn (vk_exec_turn: every earlier chunk of the group
// done): a texture lookup that would make or take back an entry (vk_texture.c entry_of) returns nothing and the chunk
// replays the op again once in turn, and so do the contact shadows' pass, per-object motion and the HDR pass's source.
#define RW_MAX 16
#define RUN_MIN 16 // draws a run needs to go to a worker
#define RUN_PEEK 65536u // ops the submitting thread reads ahead to find a run's draws, at most
#define CHUNK_DRAWS 32 // draws a chunk takes at least
#define GROUP_PEEK 65536u // ops read ahead for a group, at most
// The ops a run may hold; JUMP is followed.
static int run_op(uint32_t op) {
@@ -1719,57 +1761,50 @@ static int run_op(uint32_t op) {
}
}
static int is_draw(uint32_t op) {
return op == OP_DRAW || op == OP_DRAW_IDX || op == OP_DRAW_IDX_IND || op == OP_DRAW_MESH || op == OP_DRAW_MESH_IND;
}
typedef struct {
PlatThread *t;
int idx;
int idx; // also its command pool (vk_sync_worker_begin); the submitting thread's is RW.n
void *dc; // its draw context (vk_exec_draw_ctx_new)
volatile int32_t go, done; // a run is handed over / recorded
volatile int32_t sleeping; // waits for wake rather than spinning
PlatEvent wake;
VkExec ex; // the replay state it records the run with, and leaves
const op_hdr *h; // the run's first op; then the op that ended it
int depth; // the call depth of the run (for vk_exec_ahead)
} RwWorker;
static struct {
int n; // workers; 0 off, -1 not read yet
RwWorker *w[RW_MAX];
const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op and found too few draws
uint64_t runs, run_draws, short_runs, wait_ns; // since the last perf line
const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small
uint64_t groups, group_draws, chunks, serial_draws, wait_ns; // since the last perf line
volatile uint64_t turns, turn_ns, redos;
} RW = {.n = -1};
static void rw_walk(RwWorker *w);
static void group_take(int pool);
static void rw_main(void *arg) {
RwWorker *w = arg;
plat_thread_priority(PLAT_PRIO_HIGHEST); // as the recording thread (rec_claim)
vk_exec_draw_ctx_use(w->dc);
uint32_t seen = 0;
for (;;) {
// spins a while after a run (the next one is usually a few microseconds away), then sleeps
for (unsigned i = 0; !__atomic_load_n(&w->go, __ATOMIC_ACQUIRE); i++) {
if (i < 200000) {
// spins about 100 us after a group (the next one of the frame is usually closer), then sleeps
uint64_t t0 = kern_now_ns();
for (unsigned i = 0; __atomic_load_n(&G.gen, __ATOMIC_ACQUIRE) == seen; i++) {
if ((i & 63) || kern_now_ns() - t0 < 100000) {
plat_pause();
continue;
}
__atomic_store_n(&w->sleeping, 1, __ATOMIC_SEQ_CST);
if (!__atomic_load_n(&w->go, __ATOMIC_SEQ_CST)) plat_event_wait_ns(&w->wake, 2000000);
if (__atomic_load_n(&G.gen, __ATOMIC_SEQ_CST) == seen) plat_event_wait_ns(&w->wake, 2000000);
__atomic_store_n(&w->sleeping, 0, __ATOMIC_RELAXED);
i = 0;
t0 = kern_now_ns();
}
__atomic_store_n(&w->go, 0, __ATOMIC_RELAXED);
rw_walk(w);
__atomic_store_n(&w->done, 1, __ATOMIC_RELEASE);
seen = __atomic_load_n(&G.gen, __ATOMIC_ACQUIRE);
group_take(w->idx);
}
}
static void rw_init(void) {
int n = cfg_int("NX2NAVI_REPLAY_WORKERS");
RW.n = n < 0 ? 0 : n > RW_MAX ? RW_MAX : n;
if (RW.n > 1) RW.n = 1; // stage 1: one worker
for (int k = 0; k < RW.n; k++) {
RwWorker *w = calloc(1, sizeof *w);
if (!w) rt_fatal("vk: out of memory (replay worker)");
@@ -1780,87 +1815,195 @@ static void rw_init(void) {
if (!(w->t = plat_thread_new(rw_main, w, name))) rt_fatal("vk: cannot start a replay worker");
RW.w[k] = w;
}
if (RW.n) rt_log("vk: runs of %d draws or more are recorded by %d replay worker%s (NX2NAVI_REPLAY_WORKERS); the "
"submitting thread waits for each run", RUN_MIN, RW.n, RW.n > 1 ? "s" : "");
if (RW.n) rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s and the submitting thread "
"(NX2NAVI_REPLAY_WORKERS)", 2 * CHUNK_DRAWS, RW.n, RW.n > 1 ? "s" : "");
}
// The draws from h up to the op that ends its run (*end), reading at most RUN_PEEK ops.
static uint32_t run_peek(const op_hdr *h, const op_hdr **end) {
int vk_exec_turn_due(void) { return rw_chunk && !rw_ordered; }
int vk_exec_in_chunk(void) { return rw_chunk != NULL; }
void vk_exec_turn(void) {
Chunk *c = rw_chunk;
if (!c || rw_ordered) return;
uint64_t t0 = kern_now_ns();
for (int i = 0; i < c->idx; i++)
while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause();
rw_ordered = 1;
__atomic_add_fetch(&RW.turns, 1, __ATOMIC_RELAXED);
__atomic_add_fetch(&RW.turn_ns, kern_now_ns() - t0, __ATOMIC_RELAXED);
}
// The image views a command buffer made for its targets go when its submission is done.
static void views_release(VkExec *ex, uint64_t fence) {
if (!ex->nviews) return;
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
if (!l) rt_fatal("vk: out of memory");
l->n = ex->nviews;
for (int i = 0; i < ex->nviews; i++) l->v[i] = ((ViewEnt *)ex->views)[i].view;
vk_completion_add(fence, free_views, l);
ex->nviews = 0;
}
// The draws of the group that starts at h (it ends at *end), reading at most GROUP_PEEK ops. *serial: it holds a
// mesh draw (those are replayed on the submitting thread).
static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end) {
uint32_t draws = 0;
for (uint32_t ops = 0; ops < RUN_PEEK; ops++) {
int begun = 0;
const p_set_rt *rt = &ex->rt;
for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the walk reports it
const void *a = nvn_op_payload(h);
if (h->op == OP_JUMP) {
h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target;
h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target;
continue;
}
if (!run_op(h->op)) break;
draws += (uint32_t)is_draw(h->op);
if (!run_op(h->op) || h->op == OP_UPDATE_UB || h->op == OP_DRAW_MESH || h->op == OP_DRAW_MESH_IND ||
vk_exec_xfer_serial(h->op, a))
break;
if (h->op == OP_SET_RT && memcmp(rt, a, sizeof *rt)) {
if (begun) break; // another pass: the next group
rt = a;
}
if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) draws++, begun = 1;
if (h->op == OP_CLEAR_COLOR || h->op == OP_CLEAR_DS) begun = 1;
h = nvn_op_next(h);
}
*end = h;
return draws;
}
// The worker's side: records the run from w->h into a command buffer of its own; w->h becomes the op that ended it.
static void rw_walk(RwWorker *w) {
VkExec *ex = &w->ex;
x_cur = ex;
ex->dc = w->dc;
ex->cb = vk_sync_worker_begin(w->idx);
// the replay's command buffer before the run ended behind a full barrier, with every image in GENERAL
ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL;
ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0;
vk_exec_draw_new_cb();
const op_hdr *h = w->h;
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the walk reports it
if (h->op == OP_JUMP) {
h = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(h))->target;
continue;
}
if (!run_op(h->op)) break;
AH.at[w->depth] = h, AH.depth = w->depth; // the submitting thread waits: where vk_exec_ahead reads on from
if (RR.on > 0) runs_op(h->op);
if (!op_linear(h->op)) vk_exec_brk++;
op_one(ex, h);
h = nvn_op_next(h);
}
vk_exec_barrier(ex); // the replay's next command buffer starts after the run, every image back in GENERAL
vk_sync_worker_end(ex->cb);
w->h = h;
x_cur = NULL;
static Chunk *chunk_open(const VkExec *ex, const op_hdr *h, int idx) {
Chunk *c = &G.c[idx];
memcpy(&c->ex, ex, sizeof *ex);
c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx;
return c;
}
// The submitting thread's side, at op h of the walk: when a run of RUN_MIN draws or more starts here, a worker records
// it and the walk goes on with the op that ended it, which is returned; otherwise h.
static const op_hdr *run_start(VkExec *ex, const op_hdr *h, int depth) {
// Records chunk c with the command pool `pool` (a worker's index; the submitting thread's is RW.n).
static void chunk_record(Chunk *c, int pool) {
if (c->idx)
while (!__atomic_load_n(&G.hooks, __ATOMIC_ACQUIRE)) plat_pause();
VkExec *ex = &c->ex, *was = x_cur;
if (pool < RW.n) ex->dc = RW.w[pool]->dc; // the submitting thread records with its own (the state's)
x_cur = ex, rw_chunk = c, rw_ordered = !c->idx;
vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond);
ex->cb = vk_sync_worker_begin(pool);
// the replay's command buffer before the group ended behind a full barrier, with every image in GENERAL
ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL;
ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0, ex->fb_fs = NULL;
ex->views = NULL, ex->nviews = ex->cap_views = 0;
vk_exec_draw_new_cb();
const op_hdr *o = c->h;
for (uint32_t k = 0; k < c->nops;) {
if (o->op == OP_JUMP) {
o = (const op_hdr *)(uintptr_t)((const p_jump *)nvn_op_payload(o))->target;
continue;
}
if (!op_linear(o->op)) vk_exec_brk++;
op_one(ex, o);
if (vk_texture_deferred_take()) { // a lookup would have made or taken back a texture entry: in turn, again
vk_exec_pass_end(ex);
ex->rt_resolved = 0, ex->dirty = VK_EXEC_DIRTY_ALL;
vk_exec_draw_new_cb();
vk_exec_turn();
__atomic_add_fetch(&RW.redos, 1, __ATOMIC_RELAXED);
op_one(ex, o);
}
k++, o = nvn_op_next(o);
}
if (!c->idx) __atomic_store_n(&G.hooks, 1, __ATOMIC_RELEASE); // a chunk 0 that began no pass
vk_exec_barrier(ex); // the next command buffer starts after it, every image back in GENERAL
vk_sync_worker_end(ex->cb);
views_release(ex, ex->fence);
free(ex->views);
ex->views = NULL;
c->cb = ex->cb, c->recorded = ex->recorded;
vk_exec_xfer_worker(0, NULL, 0);
x_cur = was, rw_chunk = NULL;
__atomic_store_n(&c->done, 1, __ATOMIC_RELEASE);
}
static void group_take(int pool) {
for (;;) {
int i = __atomic_fetch_add(&G.next, 1, __ATOMIC_ACQ_REL);
if (i >= __atomic_load_n(&G.n, __ATOMIC_ACQUIRE)) return;
chunk_record(&G.c[i], pool);
}
}
// The submitting thread, at op h of the walk: when a group large enough to split starts here, it is recorded by the
// workers and this thread, and the walk goes on with the op that ended it, which is returned; otherwise h (the walk
// replays the group itself, without reading ahead again before its end).
static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) {
const op_hdr *end;
uint32_t draws = run_peek(h, &end);
if (draws < RUN_MIN || !vk_sync_append_room(1)) {
RW.skip_to[depth] = end, RW.short_runs += draws != 0;
uint32_t draws = group_peek(ex, h, &end);
if (end == h) return h; // an op the submitting thread replays itself
int n = (int)(draws / CHUNK_DRAWS);
if (n > RW.n + 1) n = RW.n + 1;
if (n > GROUP_CHUNKS) n = GROUP_CHUNKS;
if (n < 2 || vk_dump_armed || !vk_sync_append_room(n)) {
RW.skip_to[depth] = end, RW.serial_draws += draws;
return h;
}
uint64_t t0 = kern_now_ns();
vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL
RwWorker *w = RW.w[0];
memcpy(&w->ex, ex, sizeof *ex);
w->h = h, w->depth = depth;
__atomic_store_n(&w->done, 0, __ATOMIC_RELAXED);
__atomic_store_n(&w->go, 1, __ATOMIC_SEQ_CST);
if (__atomic_load_n(&w->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&w->wake);
while (!__atomic_load_n(&w->done, __ATOMIC_ACQUIRE)) plat_pause();
// the state the run left goes on in a new command buffer of the replay's, after the worker's in the part
int recorded = ex->recorded + w->ex.recorded;
VkCommandBuffer wcb = w->ex.cb;
struct DrawCtx *own = ex->dc;
memcpy(ex, &w->ex, sizeof *ex);
ex->dc = own;
ex->cb = vk_sync_append(&wcb, 1);
ex->recorded = ex->barrier_at = recorded; // the worker's command buffer ended with a full barrier
ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL, ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0;
vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL
vk_exec_draw_group_begin(ex);
// the front pass
uint32_t per = (draws + (uint32_t)n - 1) / (uint32_t)n, cd = 0;
int nc = 1, cut = 0;
G.ncond = 0;
Chunk *c = chunk_open(ex, h, 0);
for (const op_hdr *o = h; o != end;) {
const void *a = nvn_op_payload(o);
if (o->op == OP_JUMP) {
o = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target;
continue;
}
if (cut) c = chunk_open(ex, o, nc++), cd = 0, cut = 0;
switch (o->op) {
case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND:
if (++cd == per && nc < n) cut = 1;
break;
case OP_CLEAR_COLOR: case OP_CLEAR_DS: break; // recorded by the chunk
case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: {
int on = vk_exec_xfer_front(ex, o->op, a);
if (o->op == OP_RENDER_COND) {
if (G.ncond == G.ccond) {
G.ccond = G.ccond ? G.ccond * 2 : 1024;
if (!(G.cond = realloc(G.cond, G.ccond))) rt_fatal("vk: out of memory (render conditions)");
}
G.cond[G.ncond++] = (uint8_t)on;
c->ncond++;
}
break;
}
default: op_one(ex, o); break; // state, binds, render targets
}
c->nops++;
o = nvn_op_next(o);
}
// handed out: the workers and this thread take the chunks in order
for (int i = 0; i < nc; i++) G.c[i].done = 0;
G.hooks = 0;
__atomic_store_n(&G.n, nc, __ATOMIC_RELEASE);
__atomic_store_n(&G.next, 0, __ATOMIC_RELEASE);
__atomic_add_fetch(&G.gen, 1, __ATOMIC_SEQ_CST);
for (int k = 0; k < RW.n; k++)
if (__atomic_load_n(&RW.w[k]->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&RW.w[k]->wake);
group_take(RW.n);
for (int i = 0; i < nc; i++)
while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause();
__atomic_store_n(&G.next, 1 << 30, __ATOMIC_RELEASE);
// the state the front pass left goes on in a new command buffer of the replay's, after the chunks' in the part
VkCommandBuffer cbs[GROUP_CHUNKS];
int rec = 0;
for (int i = 0; i < nc; i++) cbs[i] = G.c[i].cb, rec += G.c[i].recorded;
ex->cb = vk_sync_append(cbs, nc);
ex->recorded += rec, ex->barrier_at = ex->recorded; // the last chunk ended with a full barrier
ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL, ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0, ex->fb_fs = NULL;
vk_exec_draw_new_cb();
RW.runs++, RW.run_draws += draws, RW.wait_ns += kern_now_ns() - t0;
return w->h;
RW.groups++, RW.group_draws += draws, RW.chunks += (uint64_t)nc, RW.wait_ns += kern_now_ns() - t0;
return end;
}
static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) {
@@ -1868,9 +2011,9 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) {
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_)
rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size);
if (RW.skip_to[depth] == h) RW.skip_to[depth] = NULL; // the short run read ahead is behind
if (RW.skip_to[depth] == h) RW.skip_to[depth] = NULL; // the small group read ahead is behind
if (RW.n > 0 && !RW.skip_to[depth] && !evo_in && h->op != OP_JUMP && run_op(h->op)) {
const op_hdr *stop = run_start(ex, h, depth);
const op_hdr *stop = group_start(ex, h, depth);
if (stop != h) {
if (split_draws && split_wanted()) maybe_split(ex);
h = stop;
@@ -2034,14 +2177,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
vk_exec_draw_new_cb();
}
if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory
if (ex->nviews) {
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
if (!l) rt_fatal("vk: out of memory");
l->n = ex->nviews;
for (int i = 0; i < ex->nviews; i++) l->v[i] = ((ViewEnt *)ex->views)[i].view;
vk_completion_add(fence, free_views, l);
ex->nviews = 0;
}
views_release(ex, fence);
ex->cb = VK_NULL_HANDLE;
if (n) n_subs++;
if ((rt_trace & RT_TRACE_PERF) && n_subs >= 240) {
@@ -2065,12 +2201,18 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
char cbs[1600];
vk_sync_cb_stats(cbs, sizeof cbs);
rt_log("perf: completion thread since the last line, by callback (distance from vk_completion_add):%s", cbs);
if (RW.n > 0)
rt_log("perf: replay workers: %.1f runs per submission of %.0f draws each, %.2f ms each with the hand-over; "
"%.1f shorter runs per submission stayed on the submitting thread", (double)RW.runs / (double)n_subs,
RW.runs ? (double)RW.run_draws / (double)RW.runs : 0.0,
RW.runs ? (double)RW.wait_ns / 1e6 / (double)RW.runs : 0.0, (double)RW.short_runs / (double)n_subs);
RW.runs = RW.run_draws = RW.short_runs = RW.wait_ns = 0;
if (RW.n > 0) {
uint64_t turns = __atomic_exchange_n(&RW.turns, 0, __ATOMIC_RELAXED);
uint64_t turn_ns = __atomic_exchange_n(&RW.turn_ns, 0, __ATOMIC_RELAXED);
uint64_t redos = __atomic_exchange_n(&RW.redos, 0, __ATOMIC_RELAXED);
rt_log("perf: replay workers: per submission %.1f groups of %.0f draws in %.1f chunks, %.2f ms each; %.0f draws "
"in smaller groups; waits for an earlier chunk %.1f (%.3f ms), ops replayed again in turn %.2f",
(double)RW.groups / (double)n_subs, RW.groups ? (double)RW.group_draws / (double)RW.groups : 0.0,
RW.groups ? (double)RW.chunks / (double)RW.groups : 0.0,
RW.groups ? (double)RW.wait_ns / 1e6 / (double)RW.groups : 0.0, (double)RW.serial_draws / (double)n_subs,
(double)turns / (double)n_subs, (double)turn_ns / 1e6 / (double)n_subs, (double)redos / (double)n_subs);
}
RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0;
evo_held = evo_resumed = evo_forced = evo_switches = 0;
n_subs = n_parts = n_idle_parts = n_pass_splits = 0;
}

View File

@@ -154,6 +154,16 @@ void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence);
// record: vk_exec_draw.c DrawCtx), and the one the calling thread records with from now on (NULL: the replay's).
void *vk_exec_draw_ctx_new(void);
void vk_exec_draw_ctx_use(void *ctx);
// vk_exec_draw.c, before the chunks of a parallel group are recorded (submitting thread): the stand-ins are made and the
// texture pool set this submission took is brought up to date, which the chunks then only bind.
void vk_exec_draw_group_begin(VkExec *ex);
// vk_exec_xfer.c, parallel groups: vk_exec_xfer_serial says an op stays on the submitting thread (a samples-passed reset
// or report); vk_exec_xfer_front is the front pass's CPU side of a counter report, render enable or render condition
// (returns the condition's decision, 1 renders); vk_exec_xfer_worker turns the worker side on for the calling thread with
// its chunk's decisions, or off.
int vk_exec_xfer_serial(uint32_t op, const void *payload);
int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload);
void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond);
// vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex
// buffers): the replay continues in a new one (vk_sync_split).
void vk_exec_draw_new_cb(void);
@@ -302,6 +312,7 @@ void vk_aa_composed(int ok);
// vk_contact.c: a pass about to begin (its targets resolved, no pass open), and a dispatch or copy about to be recorded
void vk_contact_pass_begin(VkExec *ex);
void vk_contact_flush(VkExec *ex);
int vk_contact_pending(void); // a contact shadows' pass waits for the next transfer outside a pass
// vk_exec_xfer.c, samples-passed counting: a draw about to be recorded in the open pass (begins an occlusion query
// when counting and none is active), and the open pass about to end (ends the active query).
void vk_exec_oq_draw(VkExec *ex);

View File

@@ -275,6 +275,7 @@ typedef struct DrawCtx {
VkDescriptorBufferInfo mo_ovr[2][32]; // [vertex, fragment][slot]; buffer 0: not replaced
struct { int valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM; // the shader memo (shader_memo_on)
struct { int valid; VkPipeState s; const VkPipeInfo *info; } PM; // the pipeline memo (pipe_memo_on)
void *fm; // feedback_scan's memo per texture handle (FmEnt[FM_N]), made at first use
VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views
VkDescSlot bind_d[MAX_DESC];
VkBufferView bind_bv[MAX_DESC];
@@ -291,6 +292,10 @@ void *vk_exec_draw_ctx_new(void) {
void vk_exec_draw_ctx_use(void *c) { dc = c ? c : &dc_main; }
static PlatMutex shader_lock = PLAT_MUTEX_INIT; // shader_tab: the replay workers look shaders up at once
static PlatMutex rs_lock = PLAT_MUTEX_INIT; // rs_lists
static PlatMutex pool_mt = PLAT_MUTEX_INIT; // pool_sync from a chunk of a parallel group
static void cb_done(void *arg) { // completion thread
CbRes *r = arg;
VkCtx *c = vk_ctx();
@@ -1066,6 +1071,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
if (serial) {
uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded
set = shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)];
plat_mutex_lock(&shader_lock);
for (int w = 0; w < SHADER_WAYS && set[w].used; w++) {
ShaderEnt *e = &set[w];
if (e->serial != serial || e->stage != st || memcmp(&e->a, &a, sizeof a)) continue;
@@ -1074,8 +1080,11 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
memmove(&set[1], &set[0], w * sizeof *set);
set[0] = hit;
}
return set[0].s;
const GfxShader *found = set[0].s;
plat_mutex_unlock(&shader_lock);
return found;
}
plat_mutex_unlock(&shader_lock);
}
int gs = st == NVN_STAGE_VS ? GFX_STAGE_VS : st == NVN_STAGE_FS ? GFX_STAGE_FS : st == NVN_STAGE_TASK ? GFX_STAGE_TS
: st == NVN_STAGE_MESH ? GFX_STAGE_MS : GFX_STAGE_CS;
@@ -1089,8 +1098,10 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st,
(unsigned long long)serial);
else if (set) { // in front, the least recently used way drops out
plat_mutex_lock(&shader_lock);
memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set);
set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s};
plat_mutex_unlock(&shader_lock);
}
return s;
}
@@ -1607,7 +1618,7 @@ static int td_mode(void) {
// texture bindings change the whole-scan memo below, but the handles they bind were mostly looked up before, in this
// pass or an earlier one. Direct-mapped.
#define FM_N 8192
static struct { uint64_t h, tok; uint32_t gen; SubRange r; } FM[FM_N];
typedef struct { uint64_t h, tok; uint32_t gen; SubRange r; } FmEnt; // DrawCtx.fm: one table per recording thread
static SubRange sampled_sub(uint64_t h) { // texture lock held
view_t v;
@@ -1624,6 +1635,8 @@ static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) {
int n = gfx_shader_resources(fs, GFX_RES_TEX, &res);
uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup
int memo = td_mode(), locked = 0;
FmEnt *fm = ex->dc->fm;
if (!fm && !(fm = ex->dc->fm = calloc(FM_N, sizeof *fm))) rt_fatal("vk: out of memory (feedback memo)");
for (int r = 0; r < n; r++) {
// Only bound slots (0..127) are looked at. The texture and sampler pool arrays (res.pool: 32768 and 4096
// descriptors, bindless handles) have slot -1: walking them ran up to about 100,000 empty iterations per scan,
@@ -1635,7 +1648,7 @@ static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) {
int slot = res[r].slot + k;
uint64_t h = ex->tex[NVN_STAGE_FS][slot];
if (!h) continue;
typeof(FM[0]) *f = &FM[(h * 0x9E3779B97F4A7C15ull) >> 51];
FmEnt *f = &fm[(h * 0x9E3779B97F4A7C15ull) >> 51];
if (memo && f->h == h && slot_still(&f->gen, f->tok, h, gen)) {
if (memo == 2) {
if (!locked) vk_texture_lock(), locked = 1;
@@ -2439,6 +2452,26 @@ static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) {
static void pool_sync_entries(VkExec *ex);
static void pool_sync(VkExec *ex);
// A chunk of a parallel group (vk_exec.c "replay workers") binds the set as vk_exec_draw_group_begin left it, up to date
// when the group began; the chunk that takes this submission's first set takes it in turn, under pool_mt. Elsewhere the
// replay syncs the pools at every draw that binds them, as before.
static void pool_sync_draw(VkExec *ex) {
if (!vk_exec_in_chunk()) return pool_sync(ex);
if (__atomic_load_n(&PL.cur_fence, __ATOMIC_ACQUIRE) == ex->fence) return;
vk_exec_turn();
plat_mutex_lock(&pool_mt);
pool_sync(ex);
plat_mutex_unlock(&pool_mt);
}
void vk_exec_draw_group_begin(VkExec *ex) {
dummies_init();
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) pool_sync(ex); // the set this submission took
}
static void pool_sync(VkExec *ex) {
if (!PL.layout) vk_pool_set_layout();
uint64_t from = 0, pre = PL.logn;
@@ -3066,6 +3099,7 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si
static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) {
DrawCtx *const cx = ex->dc;
if (info == cx->rs_last) return cx->rs_last_l;
plat_mutex_lock(&rs_lock);
if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info);
RsList *l = nvn_hm_get(&rs_lists, &info);
if (!l) {
@@ -3115,6 +3149,7 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) {
memcpy(l->r, tmp, sizeof *tmp * n);
nvn_hm_put(&rs_lists, &info, l);
}
plat_mutex_unlock(&rs_lock);
cx->rs_last = info, cx->rs_last_l = l;
return l;
}
@@ -3501,7 +3536,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
}
}
if (info->pool) { // set 2: the texture and sampler pools
pool_sync(ex);
pool_sync_draw(ex);
vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 2, 1, &PL.set, 0, NULL);
cx->RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again
}
@@ -4193,11 +4228,13 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
vk_exec_frame.bind_ns += (uint64_t)(bt_now() - bt);
vk_exec_frame.bind_calls++;
}
if (vk_texture_deferred()) return 0; // a chunk's lookup waits for its turn: the op is replayed again
if (fbl & ex->fb_dirty) feedback_barrier(ex);
feedback_written(ex, &s, fbl);
if (!mesh) bind_vertex_buffers(ex);
if (vk_dump_armed) vk_dump_draw(ex, &s, p, info, mesh && !fb);
if (vk_hdr_track) vk_hdr_draw(ex, info);
if (vk_texture_deferred()) return 0;
vk_exec_oq_draw(ex);
DRS.issued[mesh]++;
draw_stats_tick();
@@ -4742,6 +4779,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi
int nu = gfx_shader_resources(cx->B.st.vs, GFX_RES_UBO, &ur), reads = 0;
for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4;
if (!reads) return;
vk_exec_turn(); // a replay worker's chunk: the motion list follows the stream
MO.cand++;
uint64_t tp = t0 ? kern_now_ns() : 0;
if (t0) MO.ns_part[0] += tp - t0, MO.n_part[0]++;

View File

@@ -222,6 +222,18 @@ static uint64_t oq_count; // completion thread: samples since the
typedef struct { uint64_t addr, seq; int reset, lost; uint32_t n; uint32_t q[]; } OqSegment;
// The replay workers (vk_exec.c "parallel groups") record draws of one group at once. A samples-passed reset or report
// never falls inside such a group (vk_exec_xfer_serial), so counting does not change while the workers record: each
// keeps the query open in its own pass (XW.open), takes ring slots atomically and adds the slots it closes to the
// segment the next report takes, under oq_seg_lock. XW.cond: the decisions of its chunk's render conditions, made by the
// front pass in the order of the stream (vk_exec_xfer_front).
static _Thread_local struct { int on, open; const uint8_t *cond; uint32_t ncond, i; } XW = {.open = -1};
static PlatMutex oq_seg_lock = PLAT_MUTEX_INIT;
void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond) {
XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0;
}
static int oq_ready(void) {
if (oq_state) return oq_state > 0;
VkCtx *v = vk_ctx();
@@ -248,31 +260,36 @@ static int oq_ready(void) {
}
static void oq_close(VkExec *ex) {
if (oq_open < 0) return;
vkCmdEndQuery(ex->cb, oq_pool, (uint32_t)oq_open);
int *open = XW.on ? &XW.open : &oq_open;
if (*open < 0) return;
vkCmdEndQuery(ex->cb, oq_pool, (uint32_t)*open);
ex->recorded++;
if (XW.on) plat_mutex_lock(&oq_seg_lock);
if (oq_nseg == oq_capseg) {
oq_capseg = oq_capseg ? oq_capseg * 2 : 64;
oq_seg = realloc(oq_seg, sizeof *oq_seg * oq_capseg);
if (!oq_seg) rt_fatal("vk: out of memory");
}
oq_seg[oq_nseg++] = (uint32_t)oq_open;
oq_open = -1;
oq_seg[oq_nseg++] = (uint32_t)*open;
if (XW.on) plat_mutex_unlock(&oq_seg_lock);
*open = -1;
}
void vk_exec_oq_draw(VkExec *ex) {
if (!oq_counting || oq_open >= 0) return;
uint32_t q = oq_next % OQ_N;
int *open = XW.on ? &XW.open : &oq_open;
if (!oq_counting || *open >= 0) return;
// a worker takes its slot at once (a busy one is lost, not tried again): the others take theirs meanwhile
uint32_t q = (XW.on ? __atomic_fetch_add(&oq_next, 1, __ATOMIC_RELAXED) : oq_next) % OQ_N;
if (__atomic_load_n(&oq_busy[q], __ATOMIC_ACQUIRE)) {
warn_once(W_QUERY, "occlusion query ring full: draws left uncounted");
oq_lost = 1;
__atomic_store_n(&oq_lost, 1, __ATOMIC_RELAXED);
return;
}
oq_busy[q] = 1;
oq_next++;
__atomic_store_n(&oq_busy[q], 1, __ATOMIC_RELAXED);
if (!XW.on) oq_next++;
vkCmdBeginQuery(ex->cb, oq_pool, q, oq_flags);
ex->recorded++;
oq_open = (int)q;
*open = (int)q;
}
void vk_exec_oq_pass_end(VkExec *ex) { oq_close(ex); }
@@ -986,7 +1003,55 @@ static void clear_tex(VkExec *ex, const p_clear_tex *p) {
vk_texture_gpu_written(t);
}
// ---- parallel groups (vk_exec.c): the CPU side in the front pass, the rest in the workers
// 1 when the op has to be replayed on the submitting thread in the order of the stream: a samples-passed reset or report
// ends a segment of occlusion counts, which the workers' queries are added to.
int vk_exec_xfer_serial(uint32_t op, const void *payload) {
int type;
if (op == OP_REPORT_COUNTER) type = (int)((const p_report_counter *)payload)->type;
else if (op == OP_RESET_COUNTER) type = (int)((const p_reset_counter *)payload)->type;
else return 0;
return type == NVN_COUNTER_TYPE_SAMPLES_PASSED && oq_ready();
}
// The front pass, in the order of the stream, before the workers record: a counter report other than samples passed
// (its completion callback and pending entry, which later conditions read), a render enable and a render condition
// (decided here, on the reports replayed so far). Returns the condition's decision: 1 renders.
int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload) {
switch (op) {
case OP_REPORT_COUNTER: report_counter(fx, payload); return 1; // not samples passed (vk_exec_xfer_serial)
case OP_RESET_COUNTER: return 1; // other counters have nothing to reset
case OP_RENDER_ENABLE: fx->render_disabled = !((const p_flag *)payload)->value; return 1;
case OP_RENDER_COND:
fx->render_cond = *(const p_render_cond *)payload;
fx->has_render_cond = 1;
fx->render_disabled = !cond_enabled(payload);
return !fx->render_disabled;
default: rt_fatal("vk: op %s has no front pass", nvn_op_name(op));
}
return 1;
}
// A worker: what the front pass left, in its chunk. The contact shadows' pass (vk_contact_flush) is recorded where
// the replay would, once every earlier chunk of the group is done.
static void xfer_worker(VkExec *ex, uint32_t op, const void *payload) {
if (op != OP_RENDER_ENABLE && vk_contact_pending()) vk_exec_turn(), vk_contact_flush(ex);
switch (op) {
case OP_REPORT_COUNTER: case OP_RESET_COUNTER: break;
case OP_RENDER_ENABLE: ex->render_disabled = !((const p_flag *)payload)->value; break;
case OP_RENDER_COND:
ex->render_cond = *(const p_render_cond *)payload;
ex->has_render_cond = 1;
if (XW.i >= XW.ncond) rt_fatal("vk: a replay worker met more render conditions than its front pass decided");
ex->render_disabled = !XW.cond[XW.i++];
break;
default: rt_fatal("vk: op %s in a parallel group", nvn_op_name(op));
}
}
static void op_xfer_impl(VkExec *ex, uint32_t op, const void *payload) {
if (XW.on) return xfer_worker(ex, op, payload);
if (op != OP_RENDER_ENABLE && op != OP_SIGNAL_EVENT) vk_contact_flush(ex);
switch (op) {
case OP_COPY_B2B: copy_b2b(ex, payload); break;

View File

@@ -773,6 +773,7 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) {
for (int i = 0; i < 4; i++) // unlocked first look: most draws go elsewhere (a stale slot only costs the lock below)
known |= __atomic_load_n(&recs[i].win, __ATOMIC_RELAXED) == t->image;
if (!known) return;
vk_exec_turn(); // a replay worker's chunk: the last such draw of the stream decides, as in the replay
VkImage src = VK_NULL_HANDLE;
VkFormat fmt = VK_FORMAT_UNDEFINED;
int ntex = 0, slot = -1;

View File

@@ -2560,7 +2560,24 @@ void vk_mem_shadow_sync(uint64_t fence) {
plat_mutex_unlock(&shadow_lock);
}
// The replay workers (vk_exec.c) bind and write from several threads at once: a mirror's first use in a submission
// (creation, sync) and the notes of GPU writes go under use_lock. A use the submission already synced takes no lock.
static PlatMutex use_lock = PLAT_MUTEX_INIT;
static void mem_use_locked(VkMemSpan *sp);
void vk_mem_use(VkMemSpan *sp) {
PoolRec *r = sp->pool;
if (r->mstate == MS_ON && r->acc_sub == msub) {
r->vd_use_sub = msub;
sp->buffer = r->mbuf;
return;
}
plat_mutex_lock(&use_lock);
mem_use_locked(sp);
plat_mutex_unlock(&use_lock);
}
static void mem_use_locked(VkMemSpan *sp) {
PoolRec *r = sp->pool;
if (r->mstate == MS_REFUSED && r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0 && evict_enabled()) {
plat_dec64(&m_refused_n[r->refused_why]); // counted again if refused again
@@ -2578,7 +2595,11 @@ void vk_mem_use(VkMemSpan *sp) {
void vk_mem_pick(VkMemSpan *sp) {
PoolRec *r = sp->pool;
if (!r || r->mstate != MS_ON) return;
if (r->acc_sub != msub) mirror_sync(r);
if (r->acc_sub != msub) {
plat_mutex_lock(&use_lock);
if (r->acc_sub != msub) mirror_sync(r);
plat_mutex_unlock(&use_lock);
}
sp->buffer = r->mbuf;
}
@@ -3195,8 +3216,16 @@ static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) {
r->st_wb += size - o;
range_add(&wbl, &nwbl, &cwbl, r, sp->offset + o, size - o); // what found no snapshot room
}
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 0); }
void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { gpu_wrote(sp, size, 1); }
void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) {
plat_mutex_lock(&use_lock);
gpu_wrote(sp, size, 0);
plat_mutex_unlock(&use_lock);
}
void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) {
plat_mutex_lock(&use_lock);
gpu_wrote(sp, size, 1);
plat_mutex_unlock(&use_lock);
}
// Adds [a, b) to l, n sorted ranges that do not touch; returns the new count. Above cap ranges (l holds cap + 1) the two
// nearest merge, so a full list copies the fewest bytes that were not flushed; cap 1 keeps one span over all flushes.

View File

@@ -25,6 +25,7 @@
#include "cfg.h"
#include "host.h"
#include "platform/platform.h"
#include "vk.h"
#include "vk_exec.h"
#include "vk_dlaa.h"
@@ -131,9 +132,18 @@ void vk_mvprev_draw(VkExec *ex) {
P.ns_draw += kern_now_ns() - t0;
}
static void mvprev_draw_locked(VkExec *ex);
static PlatMutex draw_lock = PLAT_MUTEX_INIT; // the replay workers' scene draws note blocks at once
static void mvprev_draw_(VkExec *ex) {
const VkExecTarget *z = &ex->depth;
if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return;
plat_mutex_lock(&draw_lock);
mvprev_draw_locked(ex);
plat_mutex_unlock(&draw_lock);
}
static void mvprev_draw_locked(VkExec *ex) {
int check = P.bytes[0] != NULL; // NX2NAVI_MVPREV_CHECK=1: the summary's checks (3.6 ms a frame)
for (int index = 0; index < BANKS; index++) {
if (!(P.mask >> index & 1)) continue;

View File

@@ -3618,10 +3618,25 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) {
start_line(e, NULL, gone_desc, gone_rel, e->og_start == START_COPY_GONE ? 2 : filled ? 3 : copied);
}
// A replay worker's chunk before its turn (vk_exec.c "replay workers"): a lookup that would make or take back an entry
// returns nothing and sets this; the chunk replays the op again once every earlier chunk is done, so entries are made,
// initialised and taken back in the order of the stream, as the replay would.
static _Thread_local int tex_deferred;
int vk_texture_deferred(void) { return tex_deferred; }
int vk_texture_deferred_take(void) {
int d = tex_deferred;
tex_deferred = 0;
return d;
}
static Tex *entry_of(const tex_t *t) {
TexKey k;
make_key(t, &k);
Tex *e = nvn_hm_get(&by_key, &k);
if ((!e || e->retiring || e->released) && vk_exec_turn_due()) {
tex_deferred = 1;
return NULL;
}
if (e && e->retiring) { // retired by disuse: the pool refresh leaves it out, any other lookup takes it back
if (pool_ctx) return NULL;
retire_cancel(e);

View File

@@ -417,7 +417,7 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI
if (!U.on || !n) return 0;
// Consecutive bindings mostly name the region the last one did: that one is tried before the binary search. The
// regions are sorted and disjoint, so the region that holds addr is the one first_above finds.
static int hint;
static _Thread_local int hint; // per thread: the replay workers look up at once
int i = hint;
if (i >= U.nreg || addr < U.reg[i].lo || addr >= U.reg[i].hi) i = first_above(addr);
if (i >= U.nreg) return 0;
@@ -431,13 +431,17 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI
return 1;
}
static PlatMutex drop_lock = PLAT_MUTEX_INIT; // the replay workers' shaders write storage at once
void vk_usnap_drop(uint64_t addr, uint64_t n) {
if (!U.mode || !n) return;
plat_mutex_lock(&drop_lock);
if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr);
U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence};
if (!U.on) return;
for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++)
if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, U.gen++;
if (U.on)
for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++)
if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, __atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED);
plat_mutex_unlock(&drop_lock);
}
uint64_t vk_usnap_gen(void) { return U.gen; }