From ffa599ef33526256f62c0678a20b2bf5b3cef4cb Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:06:33 -0500 Subject: [PATCH] Let the submitting thread go on past the replay workers' groups, with up to four groups in flight. With NX2NAVI_REPLAY_PIPE=1 (off by default) the submitting thread hands a group's chunks to the workers, takes chunks of it itself until none is left to take, and goes on with the ops after the group in a new command buffer after the chunks' places in the part (vk_sync_append_later), also into the next group's front pass. Up to four groups are handed out and not in their part yet; their chunks wait in one queue in the order of the stream. The submitting thread waits for them only where the order needs it: for room, before a part goes to the GPU, before copies and texture or buffer clears, where DLAA's run or the contact shadows' hook acts, and where a lookup would make a texture entry or take back a retired one (the op goes again afterwards, as a chunk's does in turn). What the chunks read stays as the stream had it while the submitting thread runs ahead: uniform snapshot drops count from their place in the stream, a chunk reads only the inline uniform updates before its group, the HDR present's source goes by stream position, the texture pool set syncs under the chunks' lock, each group keeps its own render conditions, occlusion segments and closed query slots, and the queries after a group's last report go into the next segment made after it. A group's first chunk runs its pass hooks after the group before began its pass; turns wait for every earlier chunk, of earlier groups too. Locks the chunks take, the submitting thread takes while it is behind. Draws keep the sampled-texture descriptors they resolved per draw context and reuse them without the texture lock while their slot is unchanged; the workers waited for that lock at nearly every draw. Texture lookups that change nothing no longer wait for their turn. The perf line says where the submitting thread waited, what the chunks took, and which texture lookups waited; NX2NAVI_GPU_TIMELINE also shows the chunks and the waits. Off by default because it does not pay yet: in Hyrule Field with 3 workers it shortens the replay (11.5 against 12.1 ms a frame) but not the frame (49.2 against 49.6 fps in the same batch). The GPU gets a group's commands only once its last chunk is recorded, so the main thread's wait for the GPU after the frame's first submission grew by 0.8 ms. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 6 +- runtime/vk.h | 28 ++- runtime/vk_contact.c | 11 +- runtime/vk_dlaa.c | 9 +- runtime/vk_exec.c | 427 ++++++++++++++++++++++++++++++++++------- runtime/vk_exec.h | 26 +-- runtime/vk_exec_draw.c | 97 ++++++++-- runtime/vk_exec_xfer.c | 74 ++++--- runtime/vk_hdr.c | 10 +- runtime/vk_memory.c | 8 +- runtime/vk_mvprev.c | 2 +- runtime/vk_sync.c | 39 ++-- runtime/vk_texture.c | 21 +- runtime/vk_usnap.c | 30 ++- 14 files changed, 623 insertions(+), 165 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index bfae613c..6fe813dc 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -62,7 +62,7 @@ static const char *const known[] = { "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", - "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", + "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", @@ -121,7 +121,7 @@ static const struct { const char *key, *section; } sections[] = { {"BIND_RIGHT", "input"}, {"BIND_DOWN", "input"}, {"BIND_LSTICK_LEFT", "input"}, {"BIND_LSTICK_UP", "input"}, {"BIND_LSTICK_RIGHT", "input"}, {"BIND_LSTICK_DOWN", "input"}, {"BIND_RSTICK_LEFT", "input"}, {"BIND_RSTICK_UP", "input"}, {"BIND_RSTICK_RIGHT", "input"}, {"BIND_RSTICK_DOWN", "input"}, {"BIND_SHAKE", "input"}, - {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, + {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, {"REPLAY_PIPE", "performance"}, {"FREE_CORES", "performance"}, {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, @@ -249,7 +249,7 @@ static const CfgMeta metas[] = { {"COOP_ACCEPT", "ask", NULL, LIVE}, // Advanced: scheduling {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, - {"REPLAY_WORKERS", "0", "0..16", RESTART}, + {"REPLAY_WORKERS", "0", "0..16", RESTART}, {"REPLAY_PIPE", "0", "0|1", RESTART}, {"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART}, // dynamic resolution (dynres.c): the scene follows the GPU time toward DYN_RES_TARGET milliseconds {"DYN_RES", "0", "0|1", LIVE}, {"DYN_RES_TARGET", "16.7", "1..100", LIVE}, {"DYN_RES_STEP", "45", "8..600", LIVE}, diff --git a/runtime/vk.h b/runtime/vk.h index 2daa385f..ca6724f8 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -542,8 +542,22 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view); // be done yet; vk_exec_turn waits until they are (no-op elsewhere). int vk_texture_deferred(void); int vk_texture_deferred_take(void); +void vk_texture_deferred_stats(uint64_t out[3]); // since the last call, by what the lookup found: none, retired, released int vk_exec_turn_due(void); int vk_exec_in_chunk(void); // the calling thread records a chunk of a parallel group +// vk_exec_shared: other threads may record now (a chunk, or the submitting thread past a group, NX2NAVI_REPLAY_PIPE), +// so the shared tables lock; vk_exec_behind: the calling thread is that submitting thread; vk_exec_hooks_wait: then +// waits until the group's first chunk ran the pass hooks. +int vk_exec_shared(void); +int vk_exec_behind(void); +void vk_exec_hooks_wait(void); +// A pass hook (vk_contact_pass_begin, vk_dlaa_pass) about to change what draws read or to record: the submitting +// thread past a group waits for the group first; a group's hooks run in its first chunk, in turn already. +void vk_exec_hook_turn(void); +// Where the calling thread records in the order of the stream as the replay workers cut it (later records larger). +uint64_t vk_exec_stream_pos(void); +// The epoch of a uniform snapshot drop the calling thread makes (write) or of the drops it sees (vk_usnap.c). +uint64_t vk_exec_epoch(int write); void vk_exec_turn(void); VkSampler vk_sampler_by_id_locked(int id); // The slot's sampler with nearest filtering and no anisotropy: a depth texture read without a compare lands on one texel @@ -719,13 +733,15 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t // batch) without a signal, and returns a new begun one for the rest of the submission. The submission's timeline value // is signalled by its last part, whose signal covers every earlier part in submission order. VkCommandBuffer vk_sync_split(void); -// Inside vk_exec_record, for the replay workers (vk_exec.c): vk_sync_append ends the command buffer recorded so far, -// puts it and then cbs (recorded by the workers) into the part, in that order, and returns a new begun one for the rest -// of the part; vk_sync_append_room says whether the part has room for n more. A worker claims and begins a command -// buffer of its own pool for the submission being recorded with vk_sync_worker_begin(w) and ends it with -// vk_sync_worker_end, on its own thread, while the submitting thread waits for it. +// Inside vk_exec_record, for the replay workers (vk_exec.c): vk_sync_append_later ends the command buffer recorded so +// far, puts it into the part followed by n places for command buffers the workers record (*at the first), and returns +// a new begun one for the rest of the part; vk_sync_append_fill puts the workers' command buffers into their places, +// before the part ends; vk_sync_append_room says whether the part has room for n more. A worker claims and begins a +// command buffer of its own pool for the submission being recorded with vk_sync_worker_begin(w) and ends it with +// vk_sync_worker_end, on its own thread. int vk_sync_append_room(int n); -VkCommandBuffer vk_sync_append(const VkCommandBuffer *cbs, int n); +VkCommandBuffer vk_sync_append_later(int n, int *at); +void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n); VkCommandBuffer vk_sync_worker_begin(int w); void vk_sync_worker_end(VkCommandBuffer cb); extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) diff --git a/runtime/vk_contact.c b/runtime/vk_contact.c index 35bf6fd4..c7ac4e73 100644 --- a/runtime/vk_contact.c +++ b/runtime/vk_contact.c @@ -115,6 +115,13 @@ static void mem_barrier(VkCommandBuffer cb) { int vk_contact_pending(void) { return C.pending; } void vk_contact_flush(VkExec *ex) { + // The submitting thread past a group (vk_exec.c NX2NAVI_REPLAY_PIPE): the group's first chunk may have made the + // pass due and a later one records it where the replay would; only one still due after the group is this one's. + if (vk_exec_behind()) { + vk_exec_hooks_wait(); + if (!C.pending) return; + vk_exec_turn(); + } if (!C.pending) return; C.pending = 0; if (ex->in_pass) return; @@ -166,13 +173,15 @@ void vk_contact_pass_begin(VkExec *ex) { // The camera block: c4 of the fragment stage, 0x940 bytes. The composite binds it only after its pass begins, so // the newest one seen at any pass start serves. const p_bind_buf *b = &ex->ubo[1][1]; - if (b->addr && b->size == 0x940) C.c4 = b->addr; + if (b->addr && b->size == 0x940 && C.c4 != b->addr) vk_exec_hook_turn(), C.c4 = b->addr; if (is_gbuffer(ex, &scene, &depth)) { + vk_exec_hook_turn(); C.pending = 0; C.scene = scene, C.depth = depth, C.w = ex->pass_w, C.h = ex->pass_h; return; } if (is_composite(ex)) { + vk_exec_hook_turn(); C.pending = C.c4 != 0; return; } diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index eabecf39..3854faab 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -1544,7 +1544,7 @@ static void dlaa_draw_locked(VkExec *ex); void vk_dlaa_draw(VkExec *ex) { if (!dl_active() || D.ended) return; static PlatMutex lock = PLAT_MUTEX_INIT; // the replay workers' scene draws count at once (vk_exec.c chunks) - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&lock); dlaa_draw_locked(ex); if (lk) plat_mutex_unlock(&lock); @@ -2110,6 +2110,9 @@ void vk_dlaa_pass(VkExec *ex) { int big = D.share_off || native || c->width >= 0.95 * D.color.width / share; int same = c->width == D.color.width && c->height == D.color.height; if (one && big && c->width > D.color.width && c->height > D.color.height) { + // DLSS runs: the replay workers' draws read the jitter and its end, and list the per-object motion it + // takes (vk_exec.c "replay workers"); the G-buffer and the arming above change nothing they read + vk_exec_hook_turn(); D.ended = D.done = D.up = 1; evaluate(ex, c); } else if (one && same && native && scene_color(c->format)) { @@ -2123,9 +2126,11 @@ void vk_dlaa_pass(VkExec *ex) { if (!told++) rt_log("dlaa: the pass the HUD is drawn into is at the scene's size (%ux%u, format %d): DLSS writes there", c->width, c->height, (int)c->format); + vk_exec_hook_turn(); D.ended = D.done = D.up = 1; evaluate(ex, c); } else if (one && same && output_target(c->format)) { + vk_exec_hook_turn(); D.ended = D.done = 1; evaluate(ex, NULL); } @@ -2133,6 +2138,7 @@ void vk_dlaa_pass(VkExec *ex) { } if (!D.ended) { if (!ex->pass_w || ex->pass_w * 4 > D.color.width) return; + vk_exec_hook_turn(); D.ended = 1; if (D.color.width >= o[0]) D.done = 1, evaluate(ex, NULL); // DLAA in place; else wait for the scaling pass return; @@ -2140,6 +2146,7 @@ void vk_dlaa_pass(VkExec *ex) { // the game's scaling pass: one colour target of the output size, no depth if (ex->pass_ncolor == 1 && c->image && !z->image && scene_color(c->format) && c->width > D.color.width && c->width == o[0] && c->height == o[1] && c->samples <= 1 && !c->is3d) { + vk_exec_hook_turn(); D.done = D.up = 1; evaluate(ex, c); } diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index a75bdca0..883d6ab0 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -50,29 +50,54 @@ static _Thread_local VkExec *x_cur; // ---- parallel groups (NX2NAVI_REPLAY_WORKERS; "replay workers" below). A chunk: a part of a group's ops that one // thread records into a command buffer of its own, from the replay state the front pass left at its first op. #define GROUP_CHUNKS 17 // chunks a group is cut into at most: RW_MAX workers and the submitting thread +#define LIVE_GROUPS 4 // groups handed out whose command buffers are not in their part yet, at most +typedef struct Group Group; typedef struct { VkExec ex; // the state at its first op, then the state it records with const op_hdr *h; // its first op uint32_t nops; // the ops it records (JUMPs not counted) - uint32_t cond0, ncond; // its render conditions' decisions in G.cond + uint32_t cond0, ncond; // its render conditions' decisions in its group's cond uint64_t oq; // the occlusion counting at its first op (vk_exec_xfer_chunk_state) uint32_t slot0, nslots; // its block of occlusion query slots (vk_exec_xfer_slots) uint32_t brk; // its ops that are not state or a draw (vk_exec_brk_now) int idx; // in its group: 0 runs the group's pass hooks + Group *g; // its group VkCommandBuffer cb; int recorded; volatile int32_t done; } Chunk; -static struct { +struct Group { Chunk c[GROUP_CHUNKS]; int n; uint8_t *cond; // the decisions of the group's render conditions, in stream order uint32_t ncond, ccond; - volatile int32_t next; // the next chunk to take (huge between groups) volatile int32_t hooks; // chunk 0 began its first pass (or ended): the others may start - volatile uint32_t gen; // bumped when a group is handed out -} G = {.next = 1 << 30}; + volatile int32_t ndone; // its chunks done + volatile uint64_t seq; // its number: Q.out when it was handed out (vk_exec_stream_pos) + uint32_t upd; // the submission's inline uniform updates before it (vk_exec_upd_top) + int at; // its chunks' places in the part's list (vk_sync_append_later) + uint64_t out_ns; // when it was handed out +}; +// Group number s is GQ[s % LIVE_GROUPS]. Its chunks wait in Q in the order of the stream once it is handed out; the +// workers (and the submitting thread, while it waits) take them in that order. +static Group GQ[LIVE_GROUPS]; +#define QN (LIVE_GROUPS * GROUP_CHUNKS) +static struct { + Chunk *q[QN]; // chunk number i of those handed out is q[i % QN] + volatile uint64_t put, take; // chunks handed out, taken + volatile uint64_t done; // every group up to this number has all its chunks done + uint64_t out, drained; // the submitting thread: groups handed out, and put into their parts +} Q; +static PlatMutex done_lock = PLAT_MUTEX_INIT; // Q.done moves on (groups_done_advance) static _Thread_local Chunk *rw_chunk; // the chunk the calling thread records, NULL outside one +// The submitting thread records on past groups whose chunks may not all be done yet (NX2NAVI_REPLAY_PIPE). +static _Thread_local int rw_behind; +// Why it waited for them (RW.pw_*): room for the next group, an op chunks never record, a pass's hooks, an effect that +// waits for its turn (vk_exec_turn), a texture entry, a part split, the submission's end; and the waits that found +// every group done already. +enum { PW_NEXT, PW_OP, PW_HOOKS, PW_TURN, PW_TEX, PW_SPLIT, PW_END, PW_DONE, PW_N }; +static void group_wait(int why, int help); +static void group_drain(VkExec *ex, int why); static _Thread_local const op_hdr *rw_op; // in a chunk: the op it records (vk_exec_mesh_run reads on from it) static _Thread_local int rw_ordered; // every earlier chunk of its group is done static PlatMutex tfr_lock = PLAT_MUTEX_INIT; // target frames (vk_target_frames.h) noted by several chunks at once @@ -786,6 +811,9 @@ void vk_exec_pass_begin(VkExec *ex) { // A parallel group's targets do not change: its pass hooks, which follow the frame pass by pass, run in its first // chunk, which the others start after (chunk_record). if (!rw_chunk || !rw_chunk->idx) { + // the submitting thread past a group: the hooks read what its first chunk's left, and wait for the whole + // group where they change what its draws read (vk_exec_hook_turn) + if (!rw_chunk) vk_exec_hooks_wait(); vk_contact_pass_begin(ex); vk_dlaa_pass(ex); // DLAA runs before the bloom chain's first pass (vk_dlaa.c) } @@ -849,7 +877,7 @@ void vk_exec_pass_begin(VkExec *ex) { vk_exec_frame.passes++; ex->recorded++; ex->in_pass = 1; - if (rw_chunk && !rw_chunk->idx) __atomic_store_n(&G.hooks, 1, __ATOMIC_RELEASE); + if (rw_chunk && !rw_chunk->idx) __atomic_store_n(&rw_chunk->g->hooks, 1, __ATOMIC_RELEASE); } void vk_exec_pass_end(VkExec *ex) { @@ -871,7 +899,8 @@ void vk_exec_pass_end(VkExec *ex) { bc_copy_back(ex, &ex->color[i]); if (vk_dump_armed) vk_dump_pass_end(ex); uint64_t stamp = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1; - if (rw_chunk) plat_mutex_lock(&tfr_lock); + int lk = vk_exec_shared(); + if (lk) plat_mutex_lock(&tfr_lock); for (int i = 0; i < 8; i++) if (ex->color[i].image) { TfrTarget ft = tfr_of(&ex->color[i]); @@ -881,7 +910,7 @@ void vk_exec_pass_end(VkExec *ex) { TfrTarget ft = tfr_of(&ex->depth); vk_texture_gpu_written(ex->depth.tex), tfr_note(&ft, stamp); } - if (rw_chunk) plat_mutex_unlock(&tfr_lock); + if (lk) plat_mutex_unlock(&tfr_lock); } VkCommandBuffer vk_exec_inline_begin(int *reopen) { @@ -1103,6 +1132,10 @@ static void maybe_split(VkExec *ex) { split_at = split_n ? split_n : INT_MAX; } if (!split_n) return; + if (Q.drained < Q.out) { // groups the submitting thread went on past go into the part first, once done + if (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < Q.out) return; // split_draws stays: a later op asks again + group_drain(ex, PW_SPLIT); + } // the first part while the GPU is idle: at a pass boundary only, where a split costs no pass restart int idle = !ex->in_pass && split_idle && split_draws && !part_start && vk_sync_fence_done() >= vk_sync_issued(); if (!idle) { @@ -1750,6 +1783,34 @@ static void op_one(VkExec *ex, const op_hdr *h) { // replays the op again once in turn, and so do the contact shadows' pass and the HDR pass's source. Per-object motion // lists its draws under a lock (vk_exec_draw.c mvobj_draw), and the occlusion queries' slots and segments follow the // stream (vk_exec_xfer.c vk_exec_xfer_slots, vk_exec_xfer_group_merge). +// NX2NAVI_REPLAY_PIPE=1 (default 0): the submitting thread hands the group to the workers, takes chunks of it as they do +// until none is left to take, and goes on with the ops after it, in a new command buffer after the chunks' places in +// the part (vk_sync_append_later), while the last chunks are recorded; also into the front pass of the next group: up +// to LIVE_GROUPS groups are handed out and not in their part yet, their chunks taken from one queue in the order of +// the stream (Q). Hyrule Field, 3 workers (2026-11-12): the frame's groups took 6.8 ms of the submitting thread and the +// ops between them 4.6. A group's chunk 0 runs its pass hooks after the group before began its pass, and waits for +// every earlier group where a hook acts; a turn (vk_exec_turn) waits for every earlier chunk, of earlier groups too. +// The submitting thread waits for the groups (group_wait), and puts their command buffers and query slots in place +// (group_drain, oldest first: groups_put_done), only where the order needs it: +// - for room when LIVE_GROUPS are out, before a part is split off or submitted (a part due before a group goes to the +// GPU first: its work would otherwise wait for the group's last chunk too), and before an op that chunks never +// record (pipe_op: copies, clears of textures and buffers); +// - where a pass's hook acts: DLAA's run (vk_dlaa.c: the jitter and its end, and the per-object motion lists it takes) +// and the contact shadows' targets (vk_contact.c) change what earlier draws read (vk_exec_hook_turn); +// - when a lookup would make a texture entry or take back a retired one: the op returns early and goes again once the +// groups are done (op_again), as a chunk's does in turn (vk_texture.c lookup_quiet: what changes nothing, or takes +// back a released render target that kept its image, needs no turn); +// - uniform snapshot drops count from where they are in the stream (vk_exec_epoch: a group's chunks do not see the +// drops of ops after the group), the texture pool set syncs under the lock the chunks take, a chunk reads only the +// inline uniform updates before its group (vk_exec_upd_top), the HDR present's source goes by stream position +// (vk_exec_stream_pos), and the queries after a group's last report go into the next segment made after its front +// pass (vk_exec_xfer.c GS). +// Locks a chunk takes, the submitting thread takes too while it is behind (vk_exec_shared). The chunks it records, +// at the hand-out or while it waits between two ops, use a draw context of its own (RW.help_dc). +// Off by default: Hyrule Field, 3 workers, standing (2026-11-12), it shortened the replay (11.5 against 12.1 ms a +// frame) but not the frame (49.2 against 49.6 fps in the same batch). The main thread's wait for the GPU after the +// frame's first submission grew by 0.8 ms: the GPU gets a group's commands only once its last chunk is recorded, and +// a part goes only when every group in it is done, so fewer parts reach it, later. #define RW_MAX 16 #define CHUNK_DRAWS 32 // draws a chunk takes at least #define GROUP_PEEK 65536u // ops read ahead for a group, at most @@ -1777,8 +1838,13 @@ typedef struct { PlatEvent wake; } RwWorker; +static const char *const pw_name[PW_N] = {"next group", "op", "pass hooks", "turn", "texture entry", "split", + "submission end", "done before"}; + static struct { int n; // workers; 0 off, -1 not read yet + int pipe; // NX2NAVI_REPLAY_PIPE + void *help_dc; // the submitting thread's draw context for chunks it takes while it waits RwWorker *w[RW_MAX]; const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small uint64_t groups, group_draws, chunks, serial_draws, wait_ns; // since the last perf line @@ -1788,7 +1854,10 @@ static struct { // the submitting thread, since the last perf line: ops outside groups by kind (SER_*), and in groups the front // pass, its own chunks and the wait for the others' uint64_t ser_n[4], ser_ns[4], front_ns, own_ns, spin_ns; - volatile uint64_t turns, turn_ns, redos; + // NX2NAVI_REPLAY_PIPE, since the last perf line: waits by why (PW_*), the time from a group's hand-out to the end + // of the wait for it, and the ops that waited (PW_OP) + uint64_t pw_n[PW_N], pw_ns[PW_N], behind_ns, pd_op[OP_COUNT_]; + volatile uint64_t turns, turn_ns, redos, chunk_ns; // chunk_ns: the chunks' recording, every thread's } RW = {.n = -1}; static void group_take(int pool); @@ -1803,26 +1872,33 @@ static int ser_kind(uint32_t op) { } } +static Chunk *chunk_take(void); +static void chunk_record(Chunk *c, int pool); + static void rw_main(void *arg) { RwWorker *w = arg; plat_thread_priority(PLAT_PRIO_HIGHEST); // as the recording thread (rec_claim) + perf_lock_timing_here(); // NX2NAVI_LOCK_TIME, as on the recording thread vk_exec_draw_ctx_use(w->dc); - uint32_t seen = 0; for (;;) { - // spins about 100 us after a group (the next one of the frame is usually closer), then sleeps + Chunk *c = chunk_take(); + if (c) { + chunk_record(c, w->idx); + continue; + } + // spins about 100 us after a chunk (the next group of the frame is usually closer), then sleeps uint64_t t0 = kern_now_ns(); - for (unsigned i = 0; __atomic_load_n(&G.gen, __ATOMIC_ACQUIRE) == seen; i++) { + for (unsigned i = 0; __atomic_load_n(&Q.take, __ATOMIC_SEQ_CST) >= __atomic_load_n(&Q.put, __ATOMIC_SEQ_CST); i++) { if ((i & 63) || kern_now_ns() - t0 < 100000) { plat_pause(); continue; } __atomic_store_n(&w->sleeping, 1, __ATOMIC_SEQ_CST); - if (__atomic_load_n(&G.gen, __ATOMIC_SEQ_CST) == seen) plat_event_wait_ns(&w->wake, 2000000); + if (__atomic_load_n(&Q.take, __ATOMIC_SEQ_CST) >= __atomic_load_n(&Q.put, __ATOMIC_SEQ_CST)) + plat_event_wait_ns(&w->wake, 2000000); __atomic_store_n(&w->sleeping, 0, __ATOMIC_RELAXED); t0 = kern_now_ns(); } - seen = __atomic_load_n(&G.gen, __ATOMIC_ACQUIRE); - group_take(w->idx); } } @@ -1839,19 +1915,76 @@ static void rw_init(void) { if (!(w->t = plat_thread_new(rw_main, w, name))) rt_fatal("vk: cannot start a replay worker"); RW.w[k] = w; } - if (RW.n) rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s and the submitting thread " - "(NX2NAVI_REPLAY_WORKERS)", 2 * CHUNK_DRAWS, RW.n, RW.n > 1 ? "s" : ""); + RW.pipe = RW.n > 0 && cfg_bool("NX2NAVI_REPLAY_PIPE"); + if (RW.pipe) RW.help_dc = vk_exec_draw_ctx_new(); + if (RW.pipe) + rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s while the submitting thread goes on " + "past them (NX2NAVI_REPLAY_WORKERS, NX2NAVI_REPLAY_PIPE)", 2 * CHUNK_DRAWS, RW.n, RW.n > 1 ? "s" : ""); + else if (RW.n) + rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s and the submitting thread " + "(NX2NAVI_REPLAY_WORKERS)", 2 * CHUNK_DRAWS, RW.n, RW.n > 1 ? "s" : ""); } -int vk_exec_turn_due(void) { return rw_chunk && !rw_ordered; } +// Every group before g has all its chunks done. +static int groups_before_done(const Group *g) { return __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) + 1 >= g->seq; } + +// Every chunk before c in the stream is done: those of earlier groups, and its group's earlier ones. +static int chunk_in_turn(const Chunk *c) { + if (!groups_before_done(c->g)) return 0; + for (int i = 0; i < c->idx; i++) + if (!__atomic_load_n(&c->g->c[i].done, __ATOMIC_ACQUIRE)) return 0; + return 1; +} + +int vk_exec_turn_due(void) { + if (!rw_chunk) { + if (rw_behind && __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) >= Q.out) rw_behind = 0; + return rw_behind; + } + if (!rw_ordered && chunk_in_turn(rw_chunk)) rw_ordered = 1; + return !rw_ordered; +} int vk_exec_in_chunk(void) { return rw_chunk != NULL; } +int vk_exec_shared(void) { return rw_chunk || rw_behind; } +int vk_exec_behind(void) { return !rw_chunk && rw_behind; } + +// A chunk of a group comes after the ops before the group and after its group's earlier chunks; the submitting +// thread after every chunk of the groups it handed out. +uint64_t vk_exec_stream_pos(void) { return rw_chunk ? rw_chunk->g->seq << 5 | (uint64_t)rw_chunk->idx : Q.out << 5 | 31; } + +// Uniform snapshot drops (vk_usnap.c): a chunk's are its group's; the submitting thread's come after every group it +// handed out. A chunk sees the drops of epochs up to its group's, the submitting thread every one. +uint64_t vk_exec_epoch(int write) { return rw_chunk ? rw_chunk->g->seq : write ? Q.out + 1 : UINT64_MAX; } + +// The inline uniform updates of the submission a draw may read (vk_exec_draw.c upd_read): a chunk only those before +// its group. +uint32_t vk_exec_upd_top(uint32_t all) { return rw_chunk ? rw_chunk->g->upd : all; } + +void vk_exec_hooks_wait(void) { + if (!vk_exec_behind()) return; + const Group *g = &GQ[Q.out % LIVE_GROUPS]; // the last handed out, whose first chunk ran after the others' + while (!__atomic_load_n(&g->hooks, __ATOMIC_ACQUIRE)) plat_pause(); +} + +void vk_exec_hook_turn(void) { + if (!rw_chunk) { + group_wait(PW_HOOKS, 0); + return; + } + // a group's hooks run in its first chunk, after its group's earlier chunks; the earlier groups' draws read what + // the hook changes + while (!groups_before_done(rw_chunk->g)) plat_pause(); +} void vk_exec_turn(void) { Chunk *c = rw_chunk; - if (!c || rw_ordered) return; + if (!c) { // the submitting thread: after every chunk of the groups it went on past + group_wait(PW_TURN, 0); + return; + } + if (rw_ordered) return; uint64_t t0 = kern_now_ns(); - for (int i = 0; i < c->idx; i++) - while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause(); + while (!chunk_in_turn(c)) plat_pause(); rw_ordered = 1; __atomic_add_fetch(&RW.turns, 1, __ATOMIC_RELAXED); __atomic_add_fetch(&RW.turn_ns, kern_now_ns() - t0, __ATOMIC_RELAXED); @@ -1908,22 +2041,52 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end return draws; } -static Chunk *chunk_open(const VkExec *ex, const op_hdr *h, int idx) { - Chunk *c = &G.c[idx]; +static Chunk *chunk_open(Group *g, const VkExec *ex, const op_hdr *h, int idx) { + Chunk *c = &g->c[idx]; memcpy(&c->ex, ex, sizeof *ex); - c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx, c->oq = vk_exec_xfer_chunk_state(); + c->h = h, c->nops = 0, c->cond0 = g->ncond, c->ncond = 0, c->idx = idx, c->g = g, c->oq = vk_exec_xfer_chunk_state(); c->nslots = 0, c->brk = 0; return c; } +// The groups whose every chunk is done, in order (Q.done), once a group's last chunk is. +static void groups_done_advance(void) { + plat_mutex_lock(&done_lock); + for (;;) { + uint64_t d = Q.done; + const Group *g = &GQ[(d + 1) % LIVE_GROUPS]; + if (__atomic_load_n(&g->seq, __ATOMIC_ACQUIRE) != d + 1 || __atomic_load_n(&g->ndone, __ATOMIC_ACQUIRE) != g->n) + break; + __atomic_store_n(&Q.done, d + 1, __ATOMIC_RELEASE); + } + plat_mutex_unlock(&done_lock); +} + +static Chunk *chunk_take(void) { + uint64_t t = __atomic_load_n(&Q.take, __ATOMIC_ACQUIRE); + while (t < __atomic_load_n(&Q.put, __ATOMIC_ACQUIRE)) + if (__atomic_compare_exchange_n(&Q.take, &t, t + 1, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) return Q.q[t % QN]; + return NULL; +} + // Records chunk c with the command pool `pool` (a worker's index; the submitting thread's is RW.n). static void chunk_record(Chunk *c, int pool) { + Group *g = c->g; if (c->idx) - while (!__atomic_load_n(&G.hooks, __ATOMIC_ACQUIRE)) plat_pause(); + while (!__atomic_load_n(&g->hooks, __ATOMIC_ACQUIRE)) plat_pause(); + else if (!groups_before_done(g)) { // the pass hooks follow the frame pass by pass: after the group before's + const Group *p = &GQ[(g->seq - 1) % LIVE_GROUPS]; // not done, so not put in its part yet: still its slot + while (!__atomic_load_n(&p->hooks, __ATOMIC_ACQUIRE)) plat_pause(); + } + uint64_t t0 = kern_now_ns(); + gfx_tl_note('c', t0, (uint64_t)c->idx, (uint64_t)c->nops); VkExec *ex = &c->ex, *was = x_cur; - if (pool < RW.n) ex->dc = RW.w[pool]->dc; // the submitting thread records with its own (the state's) - x_cur = ex, rw_chunk = c, rw_ordered = !c->idx; - vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond, c->oq, c->slot0, c->nslots, c->idx); + // the submitting thread records with its own (the state's), or one of its own past the group (RW.pipe) + int help = pool == RW.n && RW.pipe; + if (pool < RW.n) ex->dc = RW.w[pool]->dc; + else if (help) ex->dc = RW.help_dc, vk_exec_draw_ctx_use(RW.help_dc); + x_cur = ex, rw_chunk = c, rw_ordered = !c->idx && groups_before_done(g); + vk_exec_xfer_worker(1, g->cond + c->cond0, c->ncond, c->oq, c->slot0, c->nslots, c->idx, (int)(g - GQ)); ex->cb = vk_sync_worker_begin(pool); // the replay's command buffer before the group ended behind a full barrier, with every image in GENERAL ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL; @@ -1949,24 +2112,104 @@ static void chunk_record(Chunk *c, int pool) { } k++, o = nvn_op_next(o); } - if (!c->idx) __atomic_store_n(&G.hooks, 1, __ATOMIC_RELEASE); // a chunk 0 that began no pass + if (!c->idx) __atomic_store_n(&g->hooks, 1, __ATOMIC_RELEASE); // a chunk 0 that began no pass vk_exec_barrier(ex); // the next command buffer starts after it, every image back in GENERAL vk_sync_worker_end(ex->cb); views_release(ex, ex->fence); free(ex->views); ex->views = NULL; c->cb = ex->cb, c->recorded = ex->recorded; - vk_exec_xfer_worker(0, NULL, 0, 0, 0, 0, 0); + vk_exec_xfer_worker(0, NULL, 0, 0, 0, 0, 0, 0); x_cur = was, rw_chunk = NULL; + if (help) vk_exec_draw_ctx_use(NULL); + __atomic_add_fetch(&RW.chunk_ns, kern_now_ns() - t0, __ATOMIC_RELAXED); + gfx_tl_note('d', 0, (uint64_t)c->idx, 0); __atomic_store_n(&c->done, 1, __ATOMIC_RELEASE); + if (__atomic_add_fetch(&g->ndone, 1, __ATOMIC_ACQ_REL) == g->n) groups_done_advance(); } static void group_take(int pool) { - for (;;) { - int i = __atomic_fetch_add(&G.next, 1, __ATOMIC_ACQ_REL); - if (i >= __atomic_load_n(&G.n, __ATOMIC_ACQUIRE)) return; - chunk_record(&G.c[i], pool); + for (Chunk *c; (c = chunk_take());) chunk_record(c, pool); +} + +// The submitting thread past groups it handed out (RW.pipe): until every chunk of them is done. With `help` (only +// between two ops) it records the chunks no worker took yet itself. Then nothing of them runs beside it any more. +static void group_wait(int why, int help) { + if (!rw_behind) return; + uint64_t t0 = kern_now_ns(); + int done = __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) >= Q.out; + if (!done) { + gfx_tl_note('q', t0, (uint64_t)(uintptr_t)pw_name[why], 0); + if (help) group_take(RW.n); + while (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < Q.out) plat_pause(); + gfx_tl_note('u', 0, 0, 0); } + rw_behind = 0; + uint64_t t1 = kern_now_ns(); + if (done) RW.pw_n[PW_DONE]++; + else RW.pw_n[why]++, RW.pw_ns[why] += t1 - t0; +} + +// A done group's command buffers into their places in the part, its closed query slots into their segments. +static void group_put(VkExec *ex, Group *g) { + vk_exec_xfer_group_merge((int)(g - GQ), g->n); + VkCommandBuffer cbs[GROUP_CHUNKS]; + int rec = 0; + for (int i = 0; i < g->n; i++) cbs[i] = g->c[i].cb, rec += g->c[i].recorded; + vk_sync_append_fill(g->at, cbs, g->n); + if (ex->barrier_at == ex->recorded) ex->barrier_at += rec; // nothing since the chunks' closing barrier + ex->recorded += rec; + RW.behind_ns += kern_now_ns() - g->out_ns; + gfx_tl_note('h', 0, 0, 0); +} + +// The done groups not in their parts yet, oldest first; when every one handed out is, the submitting thread is no +// longer behind. +static void groups_put_done(VkExec *ex) { + uint64_t d = __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE); + while (Q.drained < d) Q.drained++, group_put(ex, &GQ[Q.drained % LIVE_GROUPS]); + if (Q.drained == Q.out) rw_behind = 0; +} + +static void group_drain(VkExec *ex, int why) { + if (Q.drained == Q.out) return; + group_wait(why, 1); + groups_put_done(ex); +} + +// Before the front pass of a new group: its slot holds a group put in its part (the oldest goes in first). +static void group_room(VkExec *ex) { + if (Q.out - Q.drained < LIVE_GROUPS) return; + uint64_t want = Q.drained + 1; + if (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < want) { + uint64_t t0 = kern_now_ns(); + gfx_tl_note('q', t0, (uint64_t)(uintptr_t)pw_name[PW_NEXT], 0); + group_take(RW.n); + while (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < want) plat_pause(); + gfx_tl_note('u', 0, 0, 0); + RW.pw_n[PW_NEXT]++, RW.pw_ns[PW_NEXT] += kern_now_ns() - t0; + } + groups_put_done(ex); +} + +// The ops the submitting thread replays past a group it handed out; before any other it waits for the group. The +// ops a chunk records, the game's barriers, dispatches, and events (a wait reads memory, a signal's value goes there +// from the completion thread): their lookups go again once the group is done (op_again), and they leave what the +// chunks read as it was (vk_exec_shared, vk_exec_epoch, vk_exec_upd_top). +static int pipe_op(uint32_t op) { + return run_op(op) || op == OP_BARRIER || op == OP_DISPATCH || op == OP_DISPATCH_IND || op == OP_WAIT_EVENT || + op == OP_SIGNAL_EVENT; +} + +// A lookup of op h would have made or taken back a texture entry before the group it went on past was done (the +// submitting thread's vk_texture_deferred): once it is, h goes again, as a chunk's op does in turn. +static void op_again(VkExec *ex, const op_hdr *h) { + vk_exec_pass_end(ex); + ex->rt_resolved = 0, ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + group_wait(PW_TEX, 1); + __atomic_add_fetch(&RW.redos, 1, __ATOMIC_RELAXED); + op_one(ex, h); } // The submitting thread, at op h of the walk: when a group large enough to split starts here, it is recorded by the @@ -1987,23 +2230,30 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { RW.skip_to[depth] = end, RW.serial_draws += draws, RW.small_end[why] += draws; return h; } + if (RW.pipe) { + // a part due goes to the GPU before the group, with the groups still recording in it: its GPU work would + // otherwise wait for this group's chunks too + if (split_draws && split_wanted()) group_drain(ex, PW_SPLIT), maybe_split(ex); + group_room(ex); + } + Group *g = &GQ[(Q.out + 1) % LIVE_GROUPS]; uint64_t t0 = kern_now_ns(); gfx_tl_note('g', 0, draws, (uint64_t)n); vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL - vk_exec_draw_group_begin(ex); + g->upd = vk_exec_draw_group_begin(ex); // the front pass uint32_t per = (draws + (uint32_t)n - 1) / (uint32_t)n, cd = 0; int nc = 1, cut = 0; - G.ncond = 0; - vk_exec_xfer_group(1); - Chunk *c = chunk_open(ex, h, 0); + g->ncond = 0; + vk_exec_xfer_group((int)(g - GQ)); + Chunk *c = chunk_open(g, ex, h, 0); for (const op_hdr *o = h; o != end;) { const void *a = nvn_op_payload(o); if (o->op == OP_JUMP) { o = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; } - if (cut) c = chunk_open(ex, o, nc++), cd = 0, cut = 0; + if (cut) c = chunk_open(g, ex, o, nc++), cd = 0, cut = 0; switch (o->op) { case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: case OP_DRAW_MESH: case OP_DRAW_MESH_IND: if (vk_exec_xfer_counting() && !ex->render_disabled) c->nslots++; // at most one query each @@ -2013,11 +2263,11 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: { int on = vk_exec_xfer_front(ex, o->op, a); if (o->op == OP_RENDER_COND) { - if (G.ncond == G.ccond) { - G.ccond = G.ccond ? G.ccond * 2 : 1024; - if (!(G.cond = realloc(G.cond, G.ccond))) rt_fatal("vk: out of memory (render conditions)"); + if (g->ncond == g->ccond) { + g->ccond = g->ccond ? g->ccond * 2 : 1024; + if (!(g->cond = realloc(g->cond, g->ccond))) rt_fatal("vk: out of memory (render conditions)"); } - G.cond[G.ncond++] = (uint8_t)on; + g->cond[g->ncond++] = (uint8_t)on; c->ncond++; } break; @@ -2027,35 +2277,42 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { c->nops++; o = nvn_op_next(o); } - vk_exec_xfer_group(0); // the segments stay listed for the workers until the next group - for (int i = 0; i < nc; i++) G.c[i].slot0 = vk_exec_xfer_slots(G.c[i].nslots); // in the order of the stream - // handed out: the workers and this thread take the chunks in order - for (int i = 0; i < nc; i++) G.c[i].done = 0; - G.hooks = 0; - __atomic_store_n(&G.n, nc, __ATOMIC_RELEASE); - __atomic_store_n(&G.next, 0, __ATOMIC_RELEASE); + vk_exec_xfer_group(-1); // the segments stay listed for the workers until the group is put in its part + for (int i = 0; i < nc; i++) g->c[i].slot0 = vk_exec_xfer_slots(g->c[i].nslots); // in the order of the stream + // handed out: the workers (and without RW.pipe this thread) take the chunks in order + for (int i = 0; i < nc; i++) g->c[i].done = 0; + g->hooks = 0, g->ndone = 0, g->n = nc; + __atomic_store_n(&g->seq, ++Q.out, __ATOMIC_RELEASE); + // the replay's command buffer so far goes into the part, then the chunks' places, and the replay goes on in a new + // one at once + ex->cb = vk_sync_append_later(nc, &g->at); + for (int i = 0; i < nc; i++) Q.q[(Q.put + (uint64_t)i) % QN] = &g->c[i]; uint64_t tf = kern_now_ns(); - __atomic_add_fetch(&G.gen, 1, __ATOMIC_SEQ_CST); + g->out_ns = tf; + __atomic_store_n(&Q.put, Q.put + (uint64_t)nc, __ATOMIC_SEQ_CST); for (int k = 0; k < RW.n; k++) if (__atomic_load_n(&RW.w[k]->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&RW.w[k]->wake); - group_take(RW.n); - uint64_t to = kern_now_ns(); - for (int i = 0; i < nc; i++) - while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause(); - uint64_t tw = kern_now_ns(); - RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.spin_ns += tw - to; - vk_exec_xfer_group_merge(nc); // the chunks' closed query slots into their segments - __atomic_store_n(&G.next, 1 << 30, __ATOMIC_RELEASE); - // the state the front pass left goes on in a new command buffer of the replay's, after the chunks' in the part - VkCommandBuffer cbs[GROUP_CHUNKS]; - int rec = 0; - for (int i = 0; i < nc; i++) cbs[i] = G.c[i].cb, rec += G.c[i].recorded; - ex->cb = vk_sync_append(cbs, nc); - gfx_tl_note('h', 0, 0, 0); - ex->recorded += rec, ex->barrier_at = ex->recorded; // the last chunk ended with a full barrier + ex->barrier_at = ex->recorded; // the chunks end with a full barrier ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL, ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0, ex->fb_fs = NULL; vk_exec_draw_new_cb(); - RW.groups++, RW.group_draws += draws, RW.chunks += (uint64_t)nc, RW.wait_ns += kern_now_ns() - t0; + RW.groups++, RW.group_draws += draws, RW.chunks += (uint64_t)nc; + if (RW.pipe) { + // the submitting thread takes chunks too, as the workers wake up, and goes on once none is left to take: the + // GPU waits for the group's part until its last chunk is done + rw_behind = 1; + group_take(RW.n); + uint64_t to = kern_now_ns(); + RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.wait_ns += to - t0; + return end; + } + group_take(RW.n); + uint64_t to = kern_now_ns(); + while (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < g->seq) plat_pause(); + uint64_t tw = kern_now_ns(); + RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.spin_ns += tw - to; + groups_put_done(ex); + vk_exec_draw_new_cb(); // the submitting thread's own chunks used its draw context + RW.wait_ns += kern_now_ns() - t0; return end; } @@ -2065,6 +2322,7 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); if (RW.skip_to[depth] == h) RW.skip_to[depth] = NULL; // the small group read ahead is behind + if (Q.drained < Q.out && Q.drained < __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE)) groups_put_done(ex); if (RW.n > 0 && !RW.skip_to[depth] && !evo_in && h->op != OP_JUMP && run_op(h->op)) { const op_hdr *stop = group_start(ex, h, depth); if (stop != h) { @@ -2095,14 +2353,16 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { } default: if (RW.n > 0) { // the perf line's split of the replay outside groups + if (Q.drained < Q.out && !pipe_op(h->op)) RW.pd_op[h->op]++, group_drain(ex, PW_OP); int k = ser_kind(h->op); uint64_t t = kern_now_ns(); op_one(ex, h); + if (vk_texture_deferred_take()) op_again(ex, h); RW.ser_n[k]++, RW.ser_ns[k] += kern_now_ns() - t; } else op_one(ex, h); break; } - if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); + if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); // their ops go through this walk's checks if (split_draws && split_wanted()) maybe_split(ex); h = nvn_op_next(h); } @@ -2161,6 +2421,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t VkExec *ex = &X; if (!x_ready) state_reset(ex), vk_dump_init(), x_ready = 1; if (RW.n < 0) rw_init(); + if (Q.drained != Q.out) rt_fatal("vk: a parallel group outlived its submission"); const tex_t *keep[9]; // the render targets carry over to this submission (set_rt compares them by pointer) memcpy(keep, ex->rt.color, sizeof ex->rt.color); keep[8] = ex->rt.depth; @@ -2224,6 +2485,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t AH.live = 0; evo_queue = 0; } + group_drain(ex, PW_END); vk_exec_pass_end(ex); if (RR.on > 0) runs_break(RB_SUBMIT); x_cur = NULL; @@ -2306,6 +2568,35 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t (double)RW.front_ns / ns, (double)RW.own_ns / ns, (double)RW.spin_ns / ns); memset(RW.ser_n, 0, sizeof RW.ser_n), memset(RW.ser_ns, 0, sizeof RW.ser_ns); RW.front_ns = RW.own_ns = RW.spin_ns = 0; + rt_log("perf: replay workers: the chunks took %.3f ms per submission, every thread's", + (double)__atomic_exchange_n(&RW.chunk_ns, 0, __ATOMIC_RELAXED) / ns); + if (RW.pipe) { + o = 0; + for (int k = 0; k < PW_N; k++) + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.2f (%.3f ms)", k ? "," : "", pw_name[k], + (double)RW.pw_n[k] / (double)n_subs, (double)RW.pw_ns[k] / ns); + rt_log("perf: replay workers: past a group the submitting thread went on %.3f ms per submission until " + "it was done; waits for it per submission:%s", (double)RW.behind_ns / ns, e); + o = 0; + for (int k = 0; k < 5; k++) { // the ops that waited the most often + int best = 0; + for (int op = 1; op < OP_COUNT_; op++) + if (RW.pd_op[op] > RW.pd_op[best]) best = op; + if (!RW.pd_op[best]) break; + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.2f", k ? "," : "", nvn_op_name((uint32_t)best), + (double)RW.pd_op[best] / (double)n_subs); + RW.pd_op[best] = 0; + } + if (o) rt_log("perf: replay workers: ops that waited for a group, per submission:%s", e); + uint64_t td[3]; + vk_texture_deferred_stats(td); + rt_log("perf: replay workers: texture lookups put off until their turn, per submission: no entry " + "%.2f, a retired one %.2f, a released one %.2f", (double)td[0] / (double)n_subs, + (double)td[1] / (double)n_subs, (double)td[2] / (double)n_subs); + memset(RW.pw_n, 0, sizeof RW.pw_n), memset(RW.pw_ns, 0, sizeof RW.pw_ns); + memset(RW.pd_op, 0, sizeof RW.pd_op); + RW.behind_ns = 0; + } } RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index de4dd75d..47fa8696 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -155,20 +155,24 @@ void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence); void *vk_exec_draw_ctx_new(void); void vk_exec_draw_ctx_use(void *ctx); // vk_exec_draw.c, before the chunks of a parallel group are recorded (submitting thread): the stand-ins are made and the -// texture pool set this submission took is brought up to date, which the chunks then only bind. -void vk_exec_draw_group_begin(VkExec *ex); -// vk_exec_xfer.c, parallel groups: vk_exec_xfer_group turns the front pass on (it lists the group's occlusion segments) -// or off; vk_exec_xfer_front replays a counter report or reset, render enable or render condition in the front pass -// (returns the condition's decision, 1 renders); vk_exec_xfer_chunk_state is the occlusion counting at a chunk's first -// op, which vk_exec_xfer_worker hands to the worker side with the chunk's decisions (on), or turns it off. -void vk_exec_xfer_group(int on); +// texture pool set this submission took is brought up to date, which the chunks then only bind. Returns the inline +// uniform updates of the submission so far (vk_exec_upd_top). +uint32_t vk_exec_draw_group_begin(VkExec *ex); +// vk_exec.c: of the submission's inline uniform updates (all), those a draw on the calling thread may read. +uint32_t vk_exec_upd_top(uint32_t all); +// vk_exec_xfer.c, parallel groups: vk_exec_xfer_group turns the front pass of the group in slot `group` on (it lists +// the group's occlusion segments) or off (-1); vk_exec_xfer_front replays a counter report or reset, render enable or +// render condition in the front pass (returns the condition's decision, 1 renders); vk_exec_xfer_chunk_state is the +// occlusion counting at a chunk's first op, which vk_exec_xfer_worker hands to the worker side with the chunk's +// decisions and its group's slot (on), or turns it off. +void vk_exec_xfer_group(int group); int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload); uint64_t vk_exec_xfer_chunk_state(void); void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots, - int chunk); -// After every chunk of the group is done (submitting thread): the slots the chunks closed go into their segments, in -// the order of the chunks. -void vk_exec_xfer_group_merge(int nchunks); + int chunk, int group); +// After every chunk of the group in slot `group` is done (submitting thread, groups oldest first): the slots the +// chunks closed go into their segments, in the order of the chunks. +void vk_exec_xfer_group_merge(int group, int nchunks); // The front pass: whether a draw here counts samples (a query slot), and n slots of the ring in the order of the stream // for one chunk (the first is returned; vk_exec_xfer_worker hands them to the chunk's worker). int vk_exec_xfer_counting(void); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 43f8136b..01d28755 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -242,6 +242,21 @@ typedef struct { } MemoKey; // ---- the recording context +// bind_resources: the sampled-texture descriptors this context resolved, each with the generations and slot token it +// was resolved under: while its slot, the submission and the samplers are as they were (slot_still, as the texture +// descriptor memo TD checks, without the lock) it holds what TD would give, and a hit takes no texture lock. The +// replay workers' chunks waited for that lock at nearly every draw: 6.1 s of waits a minute in bind_resources alone +// (Hyrule Field, 3 workers, NX2NAVI_LOCK_TIME, 2026-11-12). +#define TM_N 512 +typedef struct { + uint64_t h, tok; + uint32_t gen, ep, sg; + uint16_t type; + uint8_t separate, valid; + VkImage image; + VkDescriptorImageInfo out; +} TexMemo; + // What the draws, binds and dispatches keep from one call to the next while they record a command buffer: the state // the command buffer holds (B), the signatures that let a draw skip its descriptor build or push (DS, RS, PB), the // previous draw's shaders and pipeline (SM, PM), the shader lookup's caches (AC, stagem, slotm) and the submission's @@ -280,6 +295,7 @@ typedef struct DrawCtx { struct { const void *d; uint64_t ind; VkDeviceSize at; } ma[64]; // mesh_ring_entry: entries filled ahead int ma_n, ma_i; // ... how many, and the next one a draw takes uint32_t ma_brk; // ... vk_exec_brk_now when they were filled + TexMemo TM[TM_N]; // bind_resources: sampled-texture descriptors (tm_take) VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views VkDescSlot bind_d[MAX_DESC]; VkBufferView bind_bv[MAX_DESC]; @@ -1083,7 +1099,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded set = shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)]; - int lk = vk_exec_in_chunk(); // only the chunks of a parallel group look up at once + int lk = vk_exec_shared(); // only while a parallel group records do several threads look up at once if (lk) plat_mutex_lock(&shader_lock); for (int w = 0; w < SHADER_WAYS && set[w].used; w++) { ShaderEnt *e = &set[w]; @@ -1111,7 +1127,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st, (unsigned long long)serial); else if (set) { // in front, the least recently used way drops out - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&shader_lock); memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set); set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s}; @@ -1377,7 +1393,8 @@ static void upd_note(const VkExec *ex, const p_update_ub *u) { // 1 when an update of this submission supplied [a, a+n). static int upd_read(const VkExec *ex, uint64_t a, void *out, size_t n) { if (UPD.fence != ex->fence) return 0; - for (uint32_t i = UPD.n, end = UPD.n > UPD_N ? UPD.n - UPD_N : 0; i-- > end;) { + uint32_t top = vk_exec_upd_top(UPD.n); // a replay worker's chunk: the updates before its group + for (uint32_t i = top, end = top > UPD_N ? top - UPD_N : 0; i-- > end;) { const typeof(UPD.u[0]) *e = &UPD.u[i % UPD_N]; if (a >= e->hi || a + n <= e->lo) continue; if (a < e->lo || a + n > e->hi) return 0; // partly covered: memory @@ -1787,10 +1804,14 @@ static struct { } TD[TD_N]; static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, - VkDescriptorImageInfo *out) { + VkDescriptorImageInfo *out, VkImage *img_out) { int td_on = td_mode(); - VkImage image; - if (!h || storage || !td_on) return (void)tex_info_full(ex, r, h, storage, separate, out, &image); + VkImage image = VK_NULL_HANDLE; + if (!h || storage || !td_on) { + tex_info_full(ex, r, h, storage, separate, out, &image); + if (img_out) *img_out = image; + return; + } uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 51]; if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->sgen == sg && @@ -1802,6 +1823,7 @@ static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int s *out = m->out; out->imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT : VK_IMAGE_LAYOUT_GENERAL; + if (img_out) *img_out = m->image; if (td_on == 2) { VkDescriptorImageInfo full; tex_info_full(ex, r, h, storage, separate, &full, &image); @@ -1817,6 +1839,7 @@ static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int s // entry never matches, and the next binding fills it *m = (typeof(*m)){.h = h, .tok = tok, .gen = gen, .epoch = ep, .sgen = sg, .type = r->type, .separate = (uint8_t)separate, .valid = 1, .reduced = (uint8_t)reduced, .image = image, .out = *out}; + if (img_out) *img_out = image; } // Returns whether the image bound was made without its top mip levels (texture detail, VkTexImage.skip). @@ -2470,9 +2493,16 @@ static void pool_sync(VkExec *ex); // A chunk of a parallel group (vk_exec.c "replay workers") binds the set as vk_exec_draw_group_begin left it, up to date // when the group began; the chunk that takes this submission's first set takes it in turn, under pool_mt. Elsewhere the -// replay syncs the pools at every draw that binds them, as before. +// replay syncs the pools at every draw that binds them, as before, under pool_mt past a group whose chunks still record +// (NX2NAVI_REPLAY_PIPE): the set is the submission's, read by the GPU when the part runs. static void pool_sync_draw(VkExec *ex) { - if (!vk_exec_in_chunk()) return pool_sync(ex); + if (!vk_exec_shared()) return pool_sync(ex); + if (!vk_exec_in_chunk()) { + plat_mutex_lock(&pool_mt); + pool_sync(ex); + plat_mutex_unlock(&pool_mt); + return; + } if (__atomic_load_n(&PL.cur_fence, __ATOMIC_ACQUIRE) == ex->fence) return; vk_exec_turn(); plat_mutex_lock(&pool_mt); @@ -2482,11 +2512,17 @@ static void pool_sync_draw(VkExec *ex) { static int mesh_ring_make(void); -void vk_exec_draw_group_begin(VkExec *ex) { +uint32_t vk_exec_draw_group_begin(VkExec *ex) { dummies_init(); if (vk_ctx()->has_mesh_shader && !vk_ctx()->mesh_fallback) mesh_ring_make(); // the chunks' indirect mesh draws if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); - if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) pool_sync(ex); // the set this submission took + if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) { // the set this submission took + int lk = vk_exec_shared(); // earlier groups may still record (NX2NAVI_REPLAY_PIPE) + if (lk) plat_mutex_lock(&pool_mt); + pool_sync(ex); + if (lk) plat_mutex_unlock(&pool_mt); + } + return UPD.fence == ex->fence ? UPD.n : 0; } static void pool_sync(VkExec *ex) { @@ -2749,7 +2785,8 @@ static void pool_sync_entries(VkExec *ex) { typedef struct { VkImage src, img; VmaAllocation mem; VkImageView view[2]; uint32_t w, h, layers; VkFormat fmt; } Twin; static Twin twins[TWINS]; static unsigned twin_next; -static int twin_dispatch; // op_dispatch*: the resources being bound are a dispatch's +// op_dispatch*: the resources being bound are a dispatch's, on this thread (the replay workers record draws meanwhile) +static _Thread_local int twin_dispatch; static VkBuffer twin_buf; static VmaAllocation twin_buf_mem; static VkDeviceSize twin_buf_cap; @@ -2899,8 +2936,9 @@ static Twin *twin_update(VkExec *ex, const VkTexImage *img) { // The image descriptor of slot h; in a dispatch, a compare-sampled colour image the driver cannot compare is swapped // for its depth twin; in a pass larger than it, the scene's light target, once DLSS scaled it this frame, for DLSS's // output (vk_dlaa.c). -static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) { - tex_info_lookup(ex, r, h, storage, separate, out); +static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out, + VkImage *img_out) { + tex_info_lookup(ex, r, h, storage, separate, out, img_out); if (h && !storage && vk_dlaa_swap_img && ex->in_pass && ex->pass_w > vk_dlaa_swap_img->width + 1 && vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL) == vk_dlaa_swap_img) out->imageView = vk_dlaa_swap_view, out->imageLayout = VK_IMAGE_LAYOUT_GENERAL; @@ -3116,7 +3154,7 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { DrawCtx *const cx = ex->dc; if (info == cx->rs_last) return cx->rs_last_l; - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&rs_lock); if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info); RsList *l = nvn_hm_get(&rs_lists, &info); @@ -3458,15 +3496,36 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } else { kind = 1; int locked = 0; // tex_info looks up with the texture lock held; kept descriptors (pb_take) need none + // the context's memo (TexMemo) for sampled textures while other threads record (the lock is free + // otherwise), outside a dispatch's depth twins and DLSS's stand-in image, with the generations read + // before any lookup below + int tm = !img && !r->exact && td_mode() == 1 && !vk_dlaa_swap_img && !twin_dispatch && vk_exec_shared(); + uint32_t tg = tm ? vk_texture_slots_gen() : 0, te = tm ? vk_texture_epoch() : 0; + uint32_t ts = tm ? vk_sampler_gen() : 0; for (uint32_t k = 0; k < cnt; k++) { if (pb_take(ex, b, nd + k, k, &o[k])) continue; - if (!locked) vk_texture_lock(), locked = 1; int slot = r->slot + (int)k; uint64_t h = 0; if (img) h = slot >= 0 && slot < 32 ? ex->img[st][slot] : 0; else h = slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0; uint64_t tt = bt_now(); - tex_info(ex, r, h, img, r->separate, &o[k].image); + TexMemo *m = tm && h ? &cx->TM[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 9 ^ + (uint64_t)r->separate) >> 55 & (TM_N - 1)] : NULL; + if (m && m->valid && m->h == h && m->type == r->type && m->separate == (uint8_t)r->separate && + m->ep == te && m->sg == ts && slot_still(&m->gen, m->tok, h, tg)) { + o[k].image = m->out; + o[k].image.imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) + ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT + : VK_IMAGE_LAYOUT_GENERAL; + } else { + if (!locked) vk_texture_lock(), locked = 1; + VkImage ti = VK_NULL_HANDLE; + uint64_t tok = m ? token_of(h) : 0; // before the lookup + tex_info(ex, r, h, img, r->separate, &o[k].image, &ti); + if (m) *m = (TexMemo){.h = h, .tok = tok, .gen = tg, .ep = te, .sg = ts, .type = r->type, + .separate = (uint8_t)r->separate, .valid = 1, .image = ti, + .out = o[k].image}; + } vk_exec_frame.bind_tex_ns += (uint64_t)(bt_now() - tt); o[k].w[2] = (uint32_t)o[k].image.imageLayout; // zero the tail padding pb_keep(cx, nd + k, &o[k]); @@ -4526,7 +4585,7 @@ static int mo_desc(VkExec *ex, const VkPipeInfo *info, MoDraw *q) { vk_texture_lock(); for (uint32_t k = 0; k < cnt; k++) { int slot = r->slot + (int)k; - tex_info(ex, r, slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0, 0, r->separate, &o[k].image); + tex_info(ex, r, slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0, 0, r->separate, &o[k].image, NULL); } vk_texture_unlock(); } else { @@ -4667,7 +4726,7 @@ static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSiz const GfxShader *vs; uint32_t banks; int listed = mvobj_pick(ex, &vs, &banks, t0); - int lk = listed && vk_exec_in_chunk(); + int lk = listed && vk_exec_shared(); if (lk) plat_mutex_lock(&mo_lock); if (listed) mvobj_draw_(ex, d, ib, ib_off, t0, vs, banks); if (t0) __atomic_add_fetch(&MO.ns_check, kern_now_ns() - t0, __ATOMIC_RELAXED); @@ -6240,6 +6299,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { vk_exec_frame.bind_calls++; } twin_dispatch = 0; + if (vk_texture_deferred()) return; // past a parallel group: the op goes again once it is done (vk_exec.c op_again) VkPassMark *m = dispatch_mark(ex, cs, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z); vkCmdDispatch(ex->cb, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z); vk_pass_mark_end(ex->cb, m); @@ -6279,6 +6339,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vk_exec_frame.bind_calls++; } twin_dispatch = 0; + if (vk_texture_deferred()) return; // as in op_dispatch if (cx->B.c0_valid[2]) { // the shader binds c0 // NVN2 programs read the group counts from c0 byte 0 (c0_info); at 0x200 their bank holds pushed constants // or the driver's tables, which the copy would overwrite diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index e3dde4fd..5e4c83f8 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -241,36 +241,43 @@ typedef struct { uint64_t addr, seq; int reset; volatile int32_t lost; uint32_t // times the result calls and a reset per slot, 3.5 ms more of the wait after a frame's first submission (RX 7900 XT, // Hyrule Field, 3 workers). A chunk that needs more than its block takes them from the ring. static _Thread_local struct { - int on, open, counting, reported, keep, chunk; + int on, open, counting, reported, keep, chunk, g; uint32_t seg, qnext, qend; const uint8_t *cond; uint32_t ncond, i; } XW = {.open = -1}; -static struct { int on; OqSegment **s; uint32_t n, cap; } GS; // the front pass's segments of the group +// GS: per group slot (vk_exec.c LIVE_GROUPS), the front pass's segments of the group. tail: with NX2NAVI_REPLAY_PIPE +// the submitting thread replays on past the group before its chunks are done: the first segment made after the +// group's front pass (tail_wait), by the next group's front pass or the replay, takes the queries the chunks close +// after the group's last report or reset, which in the stream come before that one's own. +#define GS_GROUPS 4 +static struct { int tail_wait; OqSegment **s, *tail; uint32_t n, cap; } GS[GS_GROUPS]; +static int gs_on = -1; // the group slot whose front pass runs #define CP_CHUNKS 32 -static struct { uint32_t *seg, *q, n, cap; } CP[CP_CHUNKS]; // per chunk index: closed slots and their segment +static struct { uint32_t *seg, *q, n, cap; } CP[GS_GROUPS][CP_CHUNKS]; // per chunk: closed slots and their segment -void vk_exec_xfer_group(int on) { // on: a new group's front pass begins; off: it ended, the list stays - if (on) GS.n = 0; - GS.on = on; +void vk_exec_xfer_group(int group) { // a group's front pass begins (its slot), or it ended (-1): the list stays + if (group >= GS_GROUPS) rt_fatal("vk: parallel group slot %d of %d", group, GS_GROUPS); + if (group >= 0) GS[group].n = 0, GS[group].tail_wait = 0, GS[group].tail = NULL; + else if (gs_on >= 0) GS[gs_on].tail_wait = 1; // until vk_exec_xfer_group_merge + gs_on = group; } int vk_exec_xfer_counting(void) { return oq_counting; } -uint32_t vk_exec_xfer_slots(uint32_t n) { - uint32_t first = oq_next; - oq_next += n; - return first; +uint32_t vk_exec_xfer_slots(uint32_t n) { // the workers of a group the submitting thread went on past take some too + return __atomic_fetch_add(&oq_next, n, __ATOMIC_RELAXED); } uint64_t vk_exec_xfer_chunk_state(void) { - return (uint64_t)GS.n << 8 | (uint64_t)(oq_counting & 1) | (uint64_t)(oq_reported & 1) << 1 | (uint64_t)(oq_keep & 1) << 2; + return (uint64_t)GS[gs_on].n << 8 | (uint64_t)(oq_counting & 1) | (uint64_t)(oq_reported & 1) << 1 | (uint64_t)(oq_keep & 1) << 2; } void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots, - int chunk) { + int chunk, int group) { if (chunk < 0 || chunk >= CP_CHUNKS) rt_fatal("vk: replay worker chunk %d of at most %d", chunk, CP_CHUNKS); - XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0, XW.chunk = chunk; + if (group < 0 || group >= GS_GROUPS) rt_fatal("vk: parallel group slot %d of %d", group, GS_GROUPS); + XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0, XW.chunk = chunk, XW.g = group; XW.qnext = slot0, XW.qend = slot0 + nslots; XW.counting = (int)(state & 1), XW.reported = (int)(state >> 1 & 1), XW.keep = (int)(state >> 2 & 1); XW.seg = (uint32_t)(state >> 8); @@ -278,7 +285,7 @@ void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t s // Adds closed slot q to the segment a worker's queries belong to now, or the pending list. static void seg_add(uint32_t q) { - __typeof__(CP[0]) *l = &CP[XW.chunk]; + __typeof__(CP[0][0]) *l = &CP[XW.g][XW.chunk]; if (l->n == l->cap) { l->cap = l->cap ? l->cap * 2 : 256; if (!(l->seg = realloc(l->seg, sizeof *l->seg * l->cap)) || !(l->q = realloc(l->q, sizeof *l->q * l->cap))) @@ -287,13 +294,21 @@ static void seg_add(uint32_t q) { l->seg[l->n] = XW.seg, l->q[l->n++] = q; } -void vk_exec_xfer_group_merge(int nchunks) { +void vk_exec_xfer_group_merge(int group, int nchunks) { + __typeof__(GS[0]) *gs = &GS[group]; for (int c = 0; c < nchunks && c < CP_CHUNKS; c++) { - __typeof__(CP[0]) *l = &CP[c]; + __typeof__(CP[0][0]) *l = &CP[group][c]; for (uint32_t i = 0; i < l->n; i++) { uint32_t q = l->q[i]; - if (l->seg[i] < GS.n) { - OqSegment *s = GS.s[l->seg[i]]; + if (l->seg[i] < gs->n) { + OqSegment *s = gs->s[l->seg[i]]; + if (s->n == s->cap) { + s->cap = s->cap ? s->cap * 2 : 16; + if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); + } + s->q[s->n++] = q; + } else if (gs->tail) { + OqSegment *s = gs->tail; if (s->n == s->cap) { s->cap = s->cap ? s->cap * 2 : 16; if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); @@ -309,6 +324,7 @@ void vk_exec_xfer_group_merge(int nchunks) { } l->n = 0; } + gs->tail_wait = 0, gs->tail = NULL; } static int oq_ready(void) { @@ -356,17 +372,16 @@ static void oq_close(VkExec *ex) { void vk_exec_oq_draw(VkExec *ex) { int *open = XW.on ? &XW.open : &oq_open; if (!(XW.on ? XW.counting : oq_counting) || *open >= 0) return; - // a worker takes the next slot of its chunk's block, past it one from the ring at once; a busy one is lost, not - // tried again - uint32_t q = (!XW.on ? oq_next : XW.qnext != XW.qend ? XW.qnext++ : __atomic_fetch_add(&oq_next, 1, __ATOMIC_RELAXED)) % OQ_N; + // a worker takes the next slot of its chunk's block, past it one from the ring at once (as the replay does, while + // a group's workers may still take some); a busy one is lost, not tried again + uint32_t q = (XW.on && XW.qnext != XW.qend ? XW.qnext++ : __atomic_fetch_add(&oq_next, 1, __ATOMIC_RELAXED)) % OQ_N; if (__atomic_load_n(&oq_busy[q], __ATOMIC_ACQUIRE)) { warn_once(W_QUERY, "occlusion query ring full: draws left uncounted"); - if (XW.on && XW.seg < GS.n) __atomic_store_n(&GS.s[XW.seg]->lost, 1, __ATOMIC_RELAXED); + if (XW.on && XW.seg < GS[XW.g].n) __atomic_store_n(&GS[XW.g].s[XW.seg]->lost, 1, __ATOMIC_RELAXED); else __atomic_store_n(&oq_lost, 1, __ATOMIC_RELAXED); return; } __atomic_store_n(&oq_busy[q], 1, __ATOMIC_RELAXED); - if (!XW.on) oq_next++; vkCmdBeginQuery(ex->cb, oq_pool, q, oq_flags); ex->recorded++; *open = (int)q; @@ -500,13 +515,16 @@ static void oq_segment(VkExec *ex, uint64_t addr, int reset) { s->cap = oq_nseg; s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg; if (oq_nseg) memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg); - if (GS.on) { // the front pass: the workers add the queries of the group's ops before it - if (GS.n == GS.cap) { - GS.cap = GS.cap ? GS.cap * 2 : 64; - if (!(GS.s = realloc(GS.s, sizeof *GS.s * GS.cap))) rt_fatal("vk: out of memory"); + if (gs_on >= 0) { // the front pass: the workers add the queries of the group's ops before it + __typeof__(GS[0]) *gs = &GS[gs_on]; + if (gs->n == gs->cap) { + gs->cap = gs->cap ? gs->cap * 2 : 64; + if (!(gs->s = realloc(gs->s, sizeof *gs->s * gs->cap))) rt_fatal("vk: out of memory"); } - GS.s[GS.n++] = s; + gs->s[gs->n++] = s; } + for (int g = 0; g < GS_GROUPS; g++) // the first made after a group's front pass + if (GS[g].tail_wait && !GS[g].tail) GS[g].tail = s; oq_nseg = 0, oq_lost = 0; vk_completion_add_result(ex->fence, oq_cb, s); } diff --git a/runtime/vk_hdr.c b/runtime/vk_hdr.c index 108d552b..43ef0152 100644 --- a/runtime/vk_hdr.c +++ b/runtime/vk_hdr.c @@ -755,7 +755,8 @@ void vk_hdr_shader_swap(uint32_t **spirv, uint64_t *words) { int vk_hdr_track; // vk_exec_draw.c calls vk_hdr_draw: set at the first present with NX2NAVI_HDR=1 static volatile int32_t g_active; // the swapchain is HDR10 (gfx_present.c vk_hdr_set_active) static PlatMutex rec_lock = PLAT_MUTEX_INIT; -static struct { VkImage win, src; VkFormat fmt; } recs[4]; // per presented window image: what the last draw copied +// per presented window image: what the last draw copied, and where in the stream (vk_exec_stream_pos) +static struct { VkImage win, src; VkFormat fmt; uint64_t pos; } recs[4]; static int nrecs; void vk_hdr_set_active(int on) { plat_xchg32(&g_active, on); } @@ -773,7 +774,8 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) { for (int i = 0; i < 4; i++) // unlocked first look: most draws go elsewhere (a stale slot only costs the lock below) known |= __atomic_load_n(&recs[i].win, __ATOMIC_RELAXED) == t->image; if (!known) return; - vk_exec_turn(); // a replay worker's chunk: the last such draw of the stream decides, as in the replay + // the last such draw of the stream decides, as in the replay, also when replay workers record draws at once + uint64_t pos = vk_exec_stream_pos(); VkImage src = VK_NULL_HANDLE; VkFormat fmt = VK_FORMAT_UNDEFINED; int ntex = 0, slot = -1; @@ -790,7 +792,7 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) { } plat_mutex_lock(&rec_lock); for (int i = 0; i < nrecs; i++) - if (recs[i].win == t->image) recs[i].src = src, recs[i].fmt = fmt; + if (recs[i].win == t->image && pos >= recs[i].pos) recs[i].src = src, recs[i].fmt = fmt, recs[i].pos = pos; plat_mutex_unlock(&rec_lock); } @@ -801,7 +803,7 @@ void vk_hdr_present_source(const VkTexImage *win, VkImage *image, VkFormat *form int r = -1; for (int i = 0; i < nrecs; i++) if (recs[i].win == win->image) r = i; - if (r < 0 && nrecs < 4) r = nrecs++, recs[r].win = win->image, recs[r].src = VK_NULL_HANDLE; + if (r < 0 && nrecs < 4) r = nrecs++, recs[r].win = win->image, recs[r].src = VK_NULL_HANDLE, recs[r].pos = 0; // kept until the next draw into the window texture: a present again without a new copy shows the same picture if (r >= 0 && recs[r].src && vk_hdr_active()) *image = recs[r].src, *format = recs[r].fmt; plat_mutex_unlock(&rec_lock); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 7fc26090..1571b8b8 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2609,7 +2609,7 @@ void vk_mem_use(VkMemSpan *sp) { } // a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors) if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) return; - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); mem_use_locked(sp); if (lk) plat_mutex_unlock(&use_lock); @@ -2634,7 +2634,7 @@ void vk_mem_pick(VkMemSpan *sp) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON) return; if (r->acc_sub != msub) { - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); if (r->acc_sub != msub) mirror_sync(r); if (lk) plat_mutex_unlock(&use_lock); @@ -3256,13 +3256,13 @@ static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) { range_add(&wbl, &nwbl, &cwbl, r, sp->offset + o, size - o); // what found no snapshot room } void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); gpu_wrote(sp, size, 0); if (lk) plat_mutex_unlock(&use_lock); } void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); gpu_wrote(sp, size, 1); if (lk) plat_mutex_unlock(&use_lock); diff --git a/runtime/vk_mvprev.c b/runtime/vk_mvprev.c index 682fd6d9..31c506cc 100644 --- a/runtime/vk_mvprev.c +++ b/runtime/vk_mvprev.c @@ -138,7 +138,7 @@ static PlatMutex draw_lock = PLAT_MUTEX_INIT; // the replay workers' scene dra static void mvprev_draw_(VkExec *ex) { const VkExecTarget *z = &ex->depth; if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return; - int lk = vk_exec_in_chunk(); // only the chunks of a parallel group (vk_exec.c) note at once + int lk = vk_exec_shared(); // only while a parallel group (vk_exec.c) records do several threads note at once if (lk) plat_mutex_lock(&draw_lock); mvprev_draw_locked(ex); if (lk) plat_mutex_unlock(&draw_lock); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 37a5591d..11b3db00 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -456,8 +456,9 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order // NX2NAVI_GPU_TIMELINE=: from that presented frame on, TL_FRAMES frames of what the CPU and the GPU did go to the // log once they are complete, in milliseconds from the first of those presents: each replay's begin and end and its // thread, each timed part's submit, GPU start and end (calibrated timestamps, so with NX2NAVI_TRACE=perf), the waits for -// the GPU after a frame's first submission, the replay workers' groups and the presents. It tells whether the GPU waits -// for the replay, for a wait, or for the CPU behind one. Off (and one load per note) without the setting. +// the GPU after a frame's first submission, the replay workers' groups, chunks and the submitting thread's waits for +// them, and the presents. It tells whether the GPU waits for the replay, for a wait, or for the CPU behind one. Off +// (and one load per note) without the setting. #define TL_CAP 16384 #define TL_FRAMES 4 typedef struct { uint64_t t, a, b; uint32_t tid; int kind; } TlEv; @@ -502,7 +503,12 @@ static void tl_dump(void) { case 'w': rt_log("vk: tl %8.3f t%-5u wait ends", at, e->tid); break; case 'g': rt_log("vk: tl %8.3f t%-5u group of %llu draws in %llu chunks", at, e->tid, (unsigned long long)e->a, (unsigned long long)e->b); break; - case 'h': rt_log("vk: tl %8.3f t%-5u group recorded", at, e->tid); break; + case 'h': rt_log("vk: tl %8.3f t%-5u group recorded and in the part", at, e->tid); break; + case 'q': rt_log("vk: tl %8.3f t%-5u waits for its group: %s", at, e->tid, (const char *)(uintptr_t)e->a); break; + case 'u': rt_log("vk: tl %8.3f t%-5u group done, goes on", at, e->tid); break; + case 'c': rt_log("vk: tl %8.3f t%-5u records chunk %llu (%llu ops)", at, e->tid, (unsigned long long)e->a, + (unsigned long long)e->b); break; + case 'd': rt_log("vk: tl %8.3f t%-5u chunk %llu done", at, e->tid, (unsigned long long)e->a); break; case 'S': rt_log("vk: tl %8.3f t%-5u part submitted, value %llu", at, e->tid, (unsigned long long)e->a); break; case 'G': rt_log("vk: tl %8.3f gpu part runs %.3f ms (submitted %.3f ms before)", at, (double)(e->a - e->t) / 1e6, (double)(int64_t)(e->t - e->b) / 1e6); break; @@ -783,25 +789,30 @@ static uint32_t part_list(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, V uint32_t n = 0; VkCommandBufferSubmitInfo one = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}; if (pro) one.commandBuffer = pro, cbi[n++] = one; - for (int i = 0; i < npart_cbs; i++) one.commandBuffer = part_cbs[i], cbi[n++] = one; + for (int i = 0; i < npart_cbs; i++) { + if (!part_cbs[i]) rt_fatal("vk: a replay worker's command buffer is missing from a part"); + one.commandBuffer = part_cbs[i], cbi[n++] = one; + } if (with_last) one.commandBuffer = last, cbi[n++] = one; npart_cbs = 0; return n; } // The replay workers (vk_exec.c "replay workers"): the replay's command buffer being recorded ends and joins the -// part's list, then the n command buffers the workers recorded, in that order, and a new command buffer of the same -// part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers' commands run -// between the replay's commands before and after. vk_sync_append_room: whether the list has room for n more. +// part's list, then n places for the command buffers the workers record, in that order, and a new command buffer of +// the same part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers' +// commands run between the replay's commands before and after. The places are filled (vk_sync_append_fill) before the +// part is submitted, which part_list checks. vk_sync_append_room: whether the list has room for n more. int vk_sync_append_room(int n) { return npart_cbs + n + 1 <= PART_CBS; } -VkCommandBuffer vk_sync_append(const VkCommandBuffer *cbs, int n) { +VkCommandBuffer vk_sync_append_later(int n, int *at) { if (!vk_sync_append_room(n)) rt_fatal("vk: a submission part of more than %d command buffers", PART_CBS); VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of it is in cb, ended by its recorder part_cbs[npart_cbs++] = cb; - for (int i = 0; i < n; i++) part_cbs[npart_cbs++] = cbs[i]; + *at = npart_cbs; + for (int i = 0; i < n; i++) part_cbs[npart_cbs++] = VK_NULL_HANDLE; CmdSlot *s = take_slot(c, cur_fence); // the ended one keeps its slot until the value is reached VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; @@ -810,10 +821,16 @@ VkCommandBuffer vk_sync_append(const VkCommandBuffer *cbs, int n) { return s->cb; } +void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n) { + if (at < 0 || at + n > npart_cbs) rt_fatal("vk: replay worker command buffers past the part's list"); + for (int i = 0; i < n; i++) part_cbs[at + i] = cbs[i]; +} + // The replay workers' command buffers: each worker has a command pool of its own (a pool belongs to one thread at a // time) and claims command buffers from it for the submission being recorded, as the replay claims its slots. Called -// on the worker while the submitting thread holds sub_lock and waits for it, so cur_fence stays put. -#define SYNC_WORKERS 16 +// on the worker while the submitting thread holds sub_lock and records on (NX2NAVI_REPLAY_PIPE) or waits for it: the +// part with the worker's command buffer is not submitted before it is done, so cur_fence stays put. +#define SYNC_WORKERS 17 // the replay workers (at most 16) and the submitting thread static struct { VkCommandPool pool; CmdSlot *slots; int n; } WP[SYNC_WORKERS]; VkCommandBuffer vk_sync_worker_begin(int w) { diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 151072ec..6c36441c 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -3720,6 +3720,10 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { // returns nothing and sets this; the chunk replays the op again once every earlier chunk is done, so entries are made, // initialised and taken back in the order of the stream, as the replay would. static _Thread_local int tex_deferred; +static volatile uint64_t n_deferred[3]; // by what the lookup found: no entry, a retired one, a released one +void vk_texture_deferred_stats(uint64_t out[3]) { + for (int i = 0; i < 3; i++) out[i] = __atomic_exchange_n(&n_deferred[i], 0, __ATOMIC_RELAXED); +} int vk_texture_deferred(void) { return tex_deferred; } int vk_texture_deferred_take(void) { int d = tex_deferred; @@ -3727,11 +3731,26 @@ int vk_texture_deferred_take(void) { return d; } +// A lookup of k that finds e (NULL: no entry) and needs no turn (vk_exec_turn_due): it returns nothing without changing +// anything (the pool refresh asking for a held key or a retired entry, a creation that failed lately: the texture pool +// refresh looks its unresolved slots up at every sync), or it is the submitting thread's past a group (vk_exec_behind) +// and takes back a released entry whose image stayed its own. That keeps its content and is ordered after the +// group's chunks anyway: one of them that looks the entry up later finds it in use, as it would have after taking it +// back itself. A transient render target the frame releases and takes back at every pass that draws it did that after +// nearly every group (Hyrule Field, 2026-11-12). +static int lookup_quiet(const Tex *e, const TexKey *k) { + if (e && e->retiring) return pool_ctx; + if (e) return !e->superseded && vk_exec_behind(); + if (holds.n && nvn_hm_get(&holds, k)) return pool_ctx; + return create_held(k); +} + static Tex *entry_of(const tex_t *t) { TexKey k; make_key(t, &k); Tex *e = nvn_hm_get(&by_key, &k); - if ((!e || e->retiring || e->released) && vk_exec_turn_due()) { + if ((!e || e->retiring || e->released) && vk_exec_turn_due() && !lookup_quiet(e, &k)) { + __atomic_add_fetch(&n_deferred[!e ? 0 : e->retiring ? 1 : 2], 1, __ATOMIC_RELAXED); tex_deferred = 1; return NULL; } diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index e9e8b4ab..c9a6ff9c 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -44,8 +44,9 @@ typedef struct Arena { } Arena; typedef struct { uint64_t lo, hi; } Rng; -typedef struct { uint64_t lo, hi, off; int dropped; } Reg; -typedef struct { uint64_t lo, hi, fence; } Wrote; +// dropped: 0, or the epoch of its first drop in the stream (vk_exec_epoch); epoch: of the write +typedef struct { uint64_t lo, hi, off, dropped; } Reg; +typedef struct { uint64_t lo, hi, fence, epoch; } Wrote; static struct { int mode; // rt_knob: 0 off, 1 on, 2 ab @@ -422,7 +423,7 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI if (i >= U.nreg || addr < U.reg[i].lo || addr >= U.reg[i].hi) i = first_above(addr); if (i >= U.nreg) return 0; const Reg *r = &U.reg[i]; - if (r->dropped || addr < r->lo || addr + n > r->hi) return 0; + if ((r->dropped && r->dropped <= vk_exec_epoch(0)) || addr < r->lo || addr + n > r->hi) return 0; uint64_t off = r->off + (addr - r->lo); if (off % align) return 0; *out = (VkDescriptorBufferInfo){U.arena->buf, off, n}; @@ -433,15 +434,22 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI static PlatMutex drop_lock = PLAT_MUTEX_INIT; // the replay workers' shaders write storage at once +// The replay workers (vk_exec.c NX2NAVI_REPLAY_PIPE) record a group's draws while the submitting thread goes on past +// it: a drop holds from its place in the stream on (vk_exec_epoch), so a chunk does not see the drops of ops after its +// group, and the submitting thread sees every one. void vk_usnap_drop(uint64_t addr, uint64_t n) { if (!U.mode || !n) return; - int lk = vk_exec_in_chunk(); + uint64_t ep = vk_exec_epoch(1); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&drop_lock); if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr); - U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence}; + U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence, ep}; if (U.on) for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++) - if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, __atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED); + if (!U.reg[i].dropped || ep < U.reg[i].dropped) { + if (!U.reg[i].dropped) U.drops++; + U.reg[i].dropped = ep, __atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED); + } if (lk) plat_mutex_unlock(&drop_lock); } @@ -450,8 +458,9 @@ uint64_t vk_usnap_gen(void) { return U.gen; } // A GPU write of a submission not known to be complete touches [lo, hi). A linear scan: vk_usnap_drop appends after // wr_prune sorted the list, so wr_hits's binary search does not hold here; only called on rare paths. static int wr_overlaps(uint64_t lo, uint64_t hi) { + uint64_t ep = vk_exec_epoch(0); for (int i = 0; i < U.nwr; i++) - if (U.wr[i].lo < hi && lo < U.wr[i].hi) return 1; + if (U.wr[i].lo < hi && lo < U.wr[i].hi && U.wr[i].epoch <= ep) return 1; return 0; } @@ -464,7 +473,12 @@ static int cpu_written(uint64_t addr, uint64_t n) { // the same test vk_usnap_ int vk_usnap_read(uint64_t addr, uint64_t n, void *dst) { const uint8_t *s = vk_usnap_bytes(addr, n); if (s) return memcpy(dst, s, n), 1; - if (!n || !cpu_written(addr, n) || wr_overlaps(addr, addr + n)) return 0; + if (!n || !cpu_written(addr, n)) return 0; + int lk = vk_exec_shared(); // a chunk's drop may grow the list + if (lk) plat_mutex_lock(&drop_lock); + int w = wr_overlaps(addr, addr + n); + if (lk) plat_mutex_unlock(&drop_lock); + if (w) return 0; memcpy(dst, (const void *)(uintptr_t)addr, n); return 2; }