diff --git a/runtime/cfg.c b/runtime/cfg.c index bfae613c..6fe813dc 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -62,7 +62,7 @@ static const char *const known[] = { "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", - "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", + "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", @@ -121,7 +121,7 @@ static const struct { const char *key, *section; } sections[] = { {"BIND_RIGHT", "input"}, {"BIND_DOWN", "input"}, {"BIND_LSTICK_LEFT", "input"}, {"BIND_LSTICK_UP", "input"}, {"BIND_LSTICK_RIGHT", "input"}, {"BIND_LSTICK_DOWN", "input"}, {"BIND_RSTICK_LEFT", "input"}, {"BIND_RSTICK_UP", "input"}, {"BIND_RSTICK_RIGHT", "input"}, {"BIND_RSTICK_DOWN", "input"}, {"BIND_SHAKE", "input"}, - {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, + {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, {"REPLAY_PIPE", "performance"}, {"FREE_CORES", "performance"}, {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, @@ -249,7 +249,7 @@ static const CfgMeta metas[] = { {"COOP_ACCEPT", "ask", NULL, LIVE}, // Advanced: scheduling {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, - {"REPLAY_WORKERS", "0", "0..16", RESTART}, + {"REPLAY_WORKERS", "0", "0..16", RESTART}, {"REPLAY_PIPE", "0", "0|1", RESTART}, {"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART}, // dynamic resolution (dynres.c): the scene follows the GPU time toward DYN_RES_TARGET milliseconds {"DYN_RES", "0", "0|1", LIVE}, {"DYN_RES_TARGET", "16.7", "1..100", LIVE}, {"DYN_RES_STEP", "45", "8..600", LIVE}, diff --git a/runtime/vk.h b/runtime/vk.h index 2daa385f..ca6724f8 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -542,8 +542,22 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view); // be done yet; vk_exec_turn waits until they are (no-op elsewhere). int vk_texture_deferred(void); int vk_texture_deferred_take(void); +void vk_texture_deferred_stats(uint64_t out[3]); // since the last call, by what the lookup found: none, retired, released int vk_exec_turn_due(void); int vk_exec_in_chunk(void); // the calling thread records a chunk of a parallel group +// vk_exec_shared: other threads may record now (a chunk, or the submitting thread past a group, NX2NAVI_REPLAY_PIPE), +// so the shared tables lock; vk_exec_behind: the calling thread is that submitting thread; vk_exec_hooks_wait: then +// waits until the group's first chunk ran the pass hooks. +int vk_exec_shared(void); +int vk_exec_behind(void); +void vk_exec_hooks_wait(void); +// A pass hook (vk_contact_pass_begin, vk_dlaa_pass) about to change what draws read or to record: the submitting +// thread past a group waits for the group first; a group's hooks run in its first chunk, in turn already. +void vk_exec_hook_turn(void); +// Where the calling thread records in the order of the stream as the replay workers cut it (later records larger). +uint64_t vk_exec_stream_pos(void); +// The epoch of a uniform snapshot drop the calling thread makes (write) or of the drops it sees (vk_usnap.c). +uint64_t vk_exec_epoch(int write); void vk_exec_turn(void); VkSampler vk_sampler_by_id_locked(int id); // The slot's sampler with nearest filtering and no anisotropy: a depth texture read without a compare lands on one texel @@ -719,13 +733,15 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t // batch) without a signal, and returns a new begun one for the rest of the submission. The submission's timeline value // is signalled by its last part, whose signal covers every earlier part in submission order. VkCommandBuffer vk_sync_split(void); -// Inside vk_exec_record, for the replay workers (vk_exec.c): vk_sync_append ends the command buffer recorded so far, -// puts it and then cbs (recorded by the workers) into the part, in that order, and returns a new begun one for the rest -// of the part; vk_sync_append_room says whether the part has room for n more. A worker claims and begins a command -// buffer of its own pool for the submission being recorded with vk_sync_worker_begin(w) and ends it with -// vk_sync_worker_end, on its own thread, while the submitting thread waits for it. +// Inside vk_exec_record, for the replay workers (vk_exec.c): vk_sync_append_later ends the command buffer recorded so +// far, puts it into the part followed by n places for command buffers the workers record (*at the first), and returns +// a new begun one for the rest of the part; vk_sync_append_fill puts the workers' command buffers into their places, +// before the part ends; vk_sync_append_room says whether the part has room for n more. A worker claims and begins a +// command buffer of its own pool for the submission being recorded with vk_sync_worker_begin(w) and ends it with +// vk_sync_worker_end, on its own thread. int vk_sync_append_room(int n); -VkCommandBuffer vk_sync_append(const VkCommandBuffer *cbs, int n); +VkCommandBuffer vk_sync_append_later(int n, int *at); +void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n); VkCommandBuffer vk_sync_worker_begin(int w); void vk_sync_worker_end(VkCommandBuffer cb); extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) diff --git a/runtime/vk_contact.c b/runtime/vk_contact.c index 35bf6fd4..c7ac4e73 100644 --- a/runtime/vk_contact.c +++ b/runtime/vk_contact.c @@ -115,6 +115,13 @@ static void mem_barrier(VkCommandBuffer cb) { int vk_contact_pending(void) { return C.pending; } void vk_contact_flush(VkExec *ex) { + // The submitting thread past a group (vk_exec.c NX2NAVI_REPLAY_PIPE): the group's first chunk may have made the + // pass due and a later one records it where the replay would; only one still due after the group is this one's. + if (vk_exec_behind()) { + vk_exec_hooks_wait(); + if (!C.pending) return; + vk_exec_turn(); + } if (!C.pending) return; C.pending = 0; if (ex->in_pass) return; @@ -166,13 +173,15 @@ void vk_contact_pass_begin(VkExec *ex) { // The camera block: c4 of the fragment stage, 0x940 bytes. The composite binds it only after its pass begins, so // the newest one seen at any pass start serves. const p_bind_buf *b = &ex->ubo[1][1]; - if (b->addr && b->size == 0x940) C.c4 = b->addr; + if (b->addr && b->size == 0x940 && C.c4 != b->addr) vk_exec_hook_turn(), C.c4 = b->addr; if (is_gbuffer(ex, &scene, &depth)) { + vk_exec_hook_turn(); C.pending = 0; C.scene = scene, C.depth = depth, C.w = ex->pass_w, C.h = ex->pass_h; return; } if (is_composite(ex)) { + vk_exec_hook_turn(); C.pending = C.c4 != 0; return; } diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index eabecf39..3854faab 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -1544,7 +1544,7 @@ static void dlaa_draw_locked(VkExec *ex); void vk_dlaa_draw(VkExec *ex) { if (!dl_active() || D.ended) return; static PlatMutex lock = PLAT_MUTEX_INIT; // the replay workers' scene draws count at once (vk_exec.c chunks) - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&lock); dlaa_draw_locked(ex); if (lk) plat_mutex_unlock(&lock); @@ -2110,6 +2110,9 @@ void vk_dlaa_pass(VkExec *ex) { int big = D.share_off || native || c->width >= 0.95 * D.color.width / share; int same = c->width == D.color.width && c->height == D.color.height; if (one && big && c->width > D.color.width && c->height > D.color.height) { + // DLSS runs: the replay workers' draws read the jitter and its end, and list the per-object motion it + // takes (vk_exec.c "replay workers"); the G-buffer and the arming above change nothing they read + vk_exec_hook_turn(); D.ended = D.done = D.up = 1; evaluate(ex, c); } else if (one && same && native && scene_color(c->format)) { @@ -2123,9 +2126,11 @@ void vk_dlaa_pass(VkExec *ex) { if (!told++) rt_log("dlaa: the pass the HUD is drawn into is at the scene's size (%ux%u, format %d): DLSS writes there", c->width, c->height, (int)c->format); + vk_exec_hook_turn(); D.ended = D.done = D.up = 1; evaluate(ex, c); } else if (one && same && output_target(c->format)) { + vk_exec_hook_turn(); D.ended = D.done = 1; evaluate(ex, NULL); } @@ -2133,6 +2138,7 @@ void vk_dlaa_pass(VkExec *ex) { } if (!D.ended) { if (!ex->pass_w || ex->pass_w * 4 > D.color.width) return; + vk_exec_hook_turn(); D.ended = 1; if (D.color.width >= o[0]) D.done = 1, evaluate(ex, NULL); // DLAA in place; else wait for the scaling pass return; @@ -2140,6 +2146,7 @@ void vk_dlaa_pass(VkExec *ex) { // the game's scaling pass: one colour target of the output size, no depth if (ex->pass_ncolor == 1 && c->image && !z->image && scene_color(c->format) && c->width > D.color.width && c->width == o[0] && c->height == o[1] && c->samples <= 1 && !c->is3d) { + vk_exec_hook_turn(); D.done = D.up = 1; evaluate(ex, c); } diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index a75bdca0..883d6ab0 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -50,29 +50,54 @@ static _Thread_local VkExec *x_cur; // ---- parallel groups (NX2NAVI_REPLAY_WORKERS; "replay workers" below). A chunk: a part of a group's ops that one // thread records into a command buffer of its own, from the replay state the front pass left at its first op. #define GROUP_CHUNKS 17 // chunks a group is cut into at most: RW_MAX workers and the submitting thread +#define LIVE_GROUPS 4 // groups handed out whose command buffers are not in their part yet, at most +typedef struct Group Group; typedef struct { VkExec ex; // the state at its first op, then the state it records with const op_hdr *h; // its first op uint32_t nops; // the ops it records (JUMPs not counted) - uint32_t cond0, ncond; // its render conditions' decisions in G.cond + uint32_t cond0, ncond; // its render conditions' decisions in its group's cond uint64_t oq; // the occlusion counting at its first op (vk_exec_xfer_chunk_state) uint32_t slot0, nslots; // its block of occlusion query slots (vk_exec_xfer_slots) uint32_t brk; // its ops that are not state or a draw (vk_exec_brk_now) int idx; // in its group: 0 runs the group's pass hooks + Group *g; // its group VkCommandBuffer cb; int recorded; volatile int32_t done; } Chunk; -static struct { +struct Group { Chunk c[GROUP_CHUNKS]; int n; uint8_t *cond; // the decisions of the group's render conditions, in stream order uint32_t ncond, ccond; - volatile int32_t next; // the next chunk to take (huge between groups) volatile int32_t hooks; // chunk 0 began its first pass (or ended): the others may start - volatile uint32_t gen; // bumped when a group is handed out -} G = {.next = 1 << 30}; + volatile int32_t ndone; // its chunks done + volatile uint64_t seq; // its number: Q.out when it was handed out (vk_exec_stream_pos) + uint32_t upd; // the submission's inline uniform updates before it (vk_exec_upd_top) + int at; // its chunks' places in the part's list (vk_sync_append_later) + uint64_t out_ns; // when it was handed out +}; +// Group number s is GQ[s % LIVE_GROUPS]. Its chunks wait in Q in the order of the stream once it is handed out; the +// workers (and the submitting thread, while it waits) take them in that order. +static Group GQ[LIVE_GROUPS]; +#define QN (LIVE_GROUPS * GROUP_CHUNKS) +static struct { + Chunk *q[QN]; // chunk number i of those handed out is q[i % QN] + volatile uint64_t put, take; // chunks handed out, taken + volatile uint64_t done; // every group up to this number has all its chunks done + uint64_t out, drained; // the submitting thread: groups handed out, and put into their parts +} Q; +static PlatMutex done_lock = PLAT_MUTEX_INIT; // Q.done moves on (groups_done_advance) static _Thread_local Chunk *rw_chunk; // the chunk the calling thread records, NULL outside one +// The submitting thread records on past groups whose chunks may not all be done yet (NX2NAVI_REPLAY_PIPE). +static _Thread_local int rw_behind; +// Why it waited for them (RW.pw_*): room for the next group, an op chunks never record, a pass's hooks, an effect that +// waits for its turn (vk_exec_turn), a texture entry, a part split, the submission's end; and the waits that found +// every group done already. +enum { PW_NEXT, PW_OP, PW_HOOKS, PW_TURN, PW_TEX, PW_SPLIT, PW_END, PW_DONE, PW_N }; +static void group_wait(int why, int help); +static void group_drain(VkExec *ex, int why); static _Thread_local const op_hdr *rw_op; // in a chunk: the op it records (vk_exec_mesh_run reads on from it) static _Thread_local int rw_ordered; // every earlier chunk of its group is done static PlatMutex tfr_lock = PLAT_MUTEX_INIT; // target frames (vk_target_frames.h) noted by several chunks at once @@ -786,6 +811,9 @@ void vk_exec_pass_begin(VkExec *ex) { // A parallel group's targets do not change: its pass hooks, which follow the frame pass by pass, run in its first // chunk, which the others start after (chunk_record). if (!rw_chunk || !rw_chunk->idx) { + // the submitting thread past a group: the hooks read what its first chunk's left, and wait for the whole + // group where they change what its draws read (vk_exec_hook_turn) + if (!rw_chunk) vk_exec_hooks_wait(); vk_contact_pass_begin(ex); vk_dlaa_pass(ex); // DLAA runs before the bloom chain's first pass (vk_dlaa.c) } @@ -849,7 +877,7 @@ void vk_exec_pass_begin(VkExec *ex) { vk_exec_frame.passes++; ex->recorded++; ex->in_pass = 1; - if (rw_chunk && !rw_chunk->idx) __atomic_store_n(&G.hooks, 1, __ATOMIC_RELEASE); + if (rw_chunk && !rw_chunk->idx) __atomic_store_n(&rw_chunk->g->hooks, 1, __ATOMIC_RELEASE); } void vk_exec_pass_end(VkExec *ex) { @@ -871,7 +899,8 @@ void vk_exec_pass_end(VkExec *ex) { bc_copy_back(ex, &ex->color[i]); if (vk_dump_armed) vk_dump_pass_end(ex); uint64_t stamp = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1; - if (rw_chunk) plat_mutex_lock(&tfr_lock); + int lk = vk_exec_shared(); + if (lk) plat_mutex_lock(&tfr_lock); for (int i = 0; i < 8; i++) if (ex->color[i].image) { TfrTarget ft = tfr_of(&ex->color[i]); @@ -881,7 +910,7 @@ void vk_exec_pass_end(VkExec *ex) { TfrTarget ft = tfr_of(&ex->depth); vk_texture_gpu_written(ex->depth.tex), tfr_note(&ft, stamp); } - if (rw_chunk) plat_mutex_unlock(&tfr_lock); + if (lk) plat_mutex_unlock(&tfr_lock); } VkCommandBuffer vk_exec_inline_begin(int *reopen) { @@ -1103,6 +1132,10 @@ static void maybe_split(VkExec *ex) { split_at = split_n ? split_n : INT_MAX; } if (!split_n) return; + if (Q.drained < Q.out) { // groups the submitting thread went on past go into the part first, once done + if (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < Q.out) return; // split_draws stays: a later op asks again + group_drain(ex, PW_SPLIT); + } // the first part while the GPU is idle: at a pass boundary only, where a split costs no pass restart int idle = !ex->in_pass && split_idle && split_draws && !part_start && vk_sync_fence_done() >= vk_sync_issued(); if (!idle) { @@ -1750,6 +1783,34 @@ static void op_one(VkExec *ex, const op_hdr *h) { // replays the op again once in turn, and so do the contact shadows' pass and the HDR pass's source. Per-object motion // lists its draws under a lock (vk_exec_draw.c mvobj_draw), and the occlusion queries' slots and segments follow the // stream (vk_exec_xfer.c vk_exec_xfer_slots, vk_exec_xfer_group_merge). +// NX2NAVI_REPLAY_PIPE=1 (default 0): the submitting thread hands the group to the workers, takes chunks of it as they do +// until none is left to take, and goes on with the ops after it, in a new command buffer after the chunks' places in +// the part (vk_sync_append_later), while the last chunks are recorded; also into the front pass of the next group: up +// to LIVE_GROUPS groups are handed out and not in their part yet, their chunks taken from one queue in the order of +// the stream (Q). Hyrule Field, 3 workers (2026-11-12): the frame's groups took 6.8 ms of the submitting thread and the +// ops between them 4.6. A group's chunk 0 runs its pass hooks after the group before began its pass, and waits for +// every earlier group where a hook acts; a turn (vk_exec_turn) waits for every earlier chunk, of earlier groups too. +// The submitting thread waits for the groups (group_wait), and puts their command buffers and query slots in place +// (group_drain, oldest first: groups_put_done), only where the order needs it: +// - for room when LIVE_GROUPS are out, before a part is split off or submitted (a part due before a group goes to the +// GPU first: its work would otherwise wait for the group's last chunk too), and before an op that chunks never +// record (pipe_op: copies, clears of textures and buffers); +// - where a pass's hook acts: DLAA's run (vk_dlaa.c: the jitter and its end, and the per-object motion lists it takes) +// and the contact shadows' targets (vk_contact.c) change what earlier draws read (vk_exec_hook_turn); +// - when a lookup would make a texture entry or take back a retired one: the op returns early and goes again once the +// groups are done (op_again), as a chunk's does in turn (vk_texture.c lookup_quiet: what changes nothing, or takes +// back a released render target that kept its image, needs no turn); +// - uniform snapshot drops count from where they are in the stream (vk_exec_epoch: a group's chunks do not see the +// drops of ops after the group), the texture pool set syncs under the lock the chunks take, a chunk reads only the +// inline uniform updates before its group (vk_exec_upd_top), the HDR present's source goes by stream position +// (vk_exec_stream_pos), and the queries after a group's last report go into the next segment made after its front +// pass (vk_exec_xfer.c GS). +// Locks a chunk takes, the submitting thread takes too while it is behind (vk_exec_shared). The chunks it records, +// at the hand-out or while it waits between two ops, use a draw context of its own (RW.help_dc). +// Off by default: Hyrule Field, 3 workers, standing (2026-11-12), it shortened the replay (11.5 against 12.1 ms a +// frame) but not the frame (49.2 against 49.6 fps in the same batch). The main thread's wait for the GPU after the +// frame's first submission grew by 0.8 ms: the GPU gets a group's commands only once its last chunk is recorded, and +// a part goes only when every group in it is done, so fewer parts reach it, later. #define RW_MAX 16 #define CHUNK_DRAWS 32 // draws a chunk takes at least #define GROUP_PEEK 65536u // ops read ahead for a group, at most @@ -1777,8 +1838,13 @@ typedef struct { PlatEvent wake; } RwWorker; +static const char *const pw_name[PW_N] = {"next group", "op", "pass hooks", "turn", "texture entry", "split", + "submission end", "done before"}; + static struct { int n; // workers; 0 off, -1 not read yet + int pipe; // NX2NAVI_REPLAY_PIPE + void *help_dc; // the submitting thread's draw context for chunks it takes while it waits RwWorker *w[RW_MAX]; const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small uint64_t groups, group_draws, chunks, serial_draws, wait_ns; // since the last perf line @@ -1788,7 +1854,10 @@ static struct { // the submitting thread, since the last perf line: ops outside groups by kind (SER_*), and in groups the front // pass, its own chunks and the wait for the others' uint64_t ser_n[4], ser_ns[4], front_ns, own_ns, spin_ns; - volatile uint64_t turns, turn_ns, redos; + // NX2NAVI_REPLAY_PIPE, since the last perf line: waits by why (PW_*), the time from a group's hand-out to the end + // of the wait for it, and the ops that waited (PW_OP) + uint64_t pw_n[PW_N], pw_ns[PW_N], behind_ns, pd_op[OP_COUNT_]; + volatile uint64_t turns, turn_ns, redos, chunk_ns; // chunk_ns: the chunks' recording, every thread's } RW = {.n = -1}; static void group_take(int pool); @@ -1803,26 +1872,33 @@ static int ser_kind(uint32_t op) { } } +static Chunk *chunk_take(void); +static void chunk_record(Chunk *c, int pool); + static void rw_main(void *arg) { RwWorker *w = arg; plat_thread_priority(PLAT_PRIO_HIGHEST); // as the recording thread (rec_claim) + perf_lock_timing_here(); // NX2NAVI_LOCK_TIME, as on the recording thread vk_exec_draw_ctx_use(w->dc); - uint32_t seen = 0; for (;;) { - // spins about 100 us after a group (the next one of the frame is usually closer), then sleeps + Chunk *c = chunk_take(); + if (c) { + chunk_record(c, w->idx); + continue; + } + // spins about 100 us after a chunk (the next group of the frame is usually closer), then sleeps uint64_t t0 = kern_now_ns(); - for (unsigned i = 0; __atomic_load_n(&G.gen, __ATOMIC_ACQUIRE) == seen; i++) { + for (unsigned i = 0; __atomic_load_n(&Q.take, __ATOMIC_SEQ_CST) >= __atomic_load_n(&Q.put, __ATOMIC_SEQ_CST); i++) { if ((i & 63) || kern_now_ns() - t0 < 100000) { plat_pause(); continue; } __atomic_store_n(&w->sleeping, 1, __ATOMIC_SEQ_CST); - if (__atomic_load_n(&G.gen, __ATOMIC_SEQ_CST) == seen) plat_event_wait_ns(&w->wake, 2000000); + if (__atomic_load_n(&Q.take, __ATOMIC_SEQ_CST) >= __atomic_load_n(&Q.put, __ATOMIC_SEQ_CST)) + plat_event_wait_ns(&w->wake, 2000000); __atomic_store_n(&w->sleeping, 0, __ATOMIC_RELAXED); t0 = kern_now_ns(); } - seen = __atomic_load_n(&G.gen, __ATOMIC_ACQUIRE); - group_take(w->idx); } } @@ -1839,19 +1915,76 @@ static void rw_init(void) { if (!(w->t = plat_thread_new(rw_main, w, name))) rt_fatal("vk: cannot start a replay worker"); RW.w[k] = w; } - if (RW.n) rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s and the submitting thread " - "(NX2NAVI_REPLAY_WORKERS)", 2 * CHUNK_DRAWS, RW.n, RW.n > 1 ? "s" : ""); + RW.pipe = RW.n > 0 && cfg_bool("NX2NAVI_REPLAY_PIPE"); + if (RW.pipe) RW.help_dc = vk_exec_draw_ctx_new(); + if (RW.pipe) + rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s while the submitting thread goes on " + "past them (NX2NAVI_REPLAY_WORKERS, NX2NAVI_REPLAY_PIPE)", 2 * CHUNK_DRAWS, RW.n, RW.n > 1 ? "s" : ""); + else if (RW.n) + rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s and the submitting thread " + "(NX2NAVI_REPLAY_WORKERS)", 2 * CHUNK_DRAWS, RW.n, RW.n > 1 ? "s" : ""); } -int vk_exec_turn_due(void) { return rw_chunk && !rw_ordered; } +// Every group before g has all its chunks done. +static int groups_before_done(const Group *g) { return __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) + 1 >= g->seq; } + +// Every chunk before c in the stream is done: those of earlier groups, and its group's earlier ones. +static int chunk_in_turn(const Chunk *c) { + if (!groups_before_done(c->g)) return 0; + for (int i = 0; i < c->idx; i++) + if (!__atomic_load_n(&c->g->c[i].done, __ATOMIC_ACQUIRE)) return 0; + return 1; +} + +int vk_exec_turn_due(void) { + if (!rw_chunk) { + if (rw_behind && __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) >= Q.out) rw_behind = 0; + return rw_behind; + } + if (!rw_ordered && chunk_in_turn(rw_chunk)) rw_ordered = 1; + return !rw_ordered; +} int vk_exec_in_chunk(void) { return rw_chunk != NULL; } +int vk_exec_shared(void) { return rw_chunk || rw_behind; } +int vk_exec_behind(void) { return !rw_chunk && rw_behind; } + +// A chunk of a group comes after the ops before the group and after its group's earlier chunks; the submitting +// thread after every chunk of the groups it handed out. +uint64_t vk_exec_stream_pos(void) { return rw_chunk ? rw_chunk->g->seq << 5 | (uint64_t)rw_chunk->idx : Q.out << 5 | 31; } + +// Uniform snapshot drops (vk_usnap.c): a chunk's are its group's; the submitting thread's come after every group it +// handed out. A chunk sees the drops of epochs up to its group's, the submitting thread every one. +uint64_t vk_exec_epoch(int write) { return rw_chunk ? rw_chunk->g->seq : write ? Q.out + 1 : UINT64_MAX; } + +// The inline uniform updates of the submission a draw may read (vk_exec_draw.c upd_read): a chunk only those before +// its group. +uint32_t vk_exec_upd_top(uint32_t all) { return rw_chunk ? rw_chunk->g->upd : all; } + +void vk_exec_hooks_wait(void) { + if (!vk_exec_behind()) return; + const Group *g = &GQ[Q.out % LIVE_GROUPS]; // the last handed out, whose first chunk ran after the others' + while (!__atomic_load_n(&g->hooks, __ATOMIC_ACQUIRE)) plat_pause(); +} + +void vk_exec_hook_turn(void) { + if (!rw_chunk) { + group_wait(PW_HOOKS, 0); + return; + } + // a group's hooks run in its first chunk, after its group's earlier chunks; the earlier groups' draws read what + // the hook changes + while (!groups_before_done(rw_chunk->g)) plat_pause(); +} void vk_exec_turn(void) { Chunk *c = rw_chunk; - if (!c || rw_ordered) return; + if (!c) { // the submitting thread: after every chunk of the groups it went on past + group_wait(PW_TURN, 0); + return; + } + if (rw_ordered) return; uint64_t t0 = kern_now_ns(); - for (int i = 0; i < c->idx; i++) - while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause(); + while (!chunk_in_turn(c)) plat_pause(); rw_ordered = 1; __atomic_add_fetch(&RW.turns, 1, __ATOMIC_RELAXED); __atomic_add_fetch(&RW.turn_ns, kern_now_ns() - t0, __ATOMIC_RELAXED); @@ -1908,22 +2041,52 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end return draws; } -static Chunk *chunk_open(const VkExec *ex, const op_hdr *h, int idx) { - Chunk *c = &G.c[idx]; +static Chunk *chunk_open(Group *g, const VkExec *ex, const op_hdr *h, int idx) { + Chunk *c = &g->c[idx]; memcpy(&c->ex, ex, sizeof *ex); - c->h = h, c->nops = 0, c->cond0 = G.ncond, c->ncond = 0, c->idx = idx, c->oq = vk_exec_xfer_chunk_state(); + c->h = h, c->nops = 0, c->cond0 = g->ncond, c->ncond = 0, c->idx = idx, c->g = g, c->oq = vk_exec_xfer_chunk_state(); c->nslots = 0, c->brk = 0; return c; } +// The groups whose every chunk is done, in order (Q.done), once a group's last chunk is. +static void groups_done_advance(void) { + plat_mutex_lock(&done_lock); + for (;;) { + uint64_t d = Q.done; + const Group *g = &GQ[(d + 1) % LIVE_GROUPS]; + if (__atomic_load_n(&g->seq, __ATOMIC_ACQUIRE) != d + 1 || __atomic_load_n(&g->ndone, __ATOMIC_ACQUIRE) != g->n) + break; + __atomic_store_n(&Q.done, d + 1, __ATOMIC_RELEASE); + } + plat_mutex_unlock(&done_lock); +} + +static Chunk *chunk_take(void) { + uint64_t t = __atomic_load_n(&Q.take, __ATOMIC_ACQUIRE); + while (t < __atomic_load_n(&Q.put, __ATOMIC_ACQUIRE)) + if (__atomic_compare_exchange_n(&Q.take, &t, t + 1, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) return Q.q[t % QN]; + return NULL; +} + // Records chunk c with the command pool `pool` (a worker's index; the submitting thread's is RW.n). static void chunk_record(Chunk *c, int pool) { + Group *g = c->g; if (c->idx) - while (!__atomic_load_n(&G.hooks, __ATOMIC_ACQUIRE)) plat_pause(); + while (!__atomic_load_n(&g->hooks, __ATOMIC_ACQUIRE)) plat_pause(); + else if (!groups_before_done(g)) { // the pass hooks follow the frame pass by pass: after the group before's + const Group *p = &GQ[(g->seq - 1) % LIVE_GROUPS]; // not done, so not put in its part yet: still its slot + while (!__atomic_load_n(&p->hooks, __ATOMIC_ACQUIRE)) plat_pause(); + } + uint64_t t0 = kern_now_ns(); + gfx_tl_note('c', t0, (uint64_t)c->idx, (uint64_t)c->nops); VkExec *ex = &c->ex, *was = x_cur; - if (pool < RW.n) ex->dc = RW.w[pool]->dc; // the submitting thread records with its own (the state's) - x_cur = ex, rw_chunk = c, rw_ordered = !c->idx; - vk_exec_xfer_worker(1, G.cond + c->cond0, c->ncond, c->oq, c->slot0, c->nslots, c->idx); + // the submitting thread records with its own (the state's), or one of its own past the group (RW.pipe) + int help = pool == RW.n && RW.pipe; + if (pool < RW.n) ex->dc = RW.w[pool]->dc; + else if (help) ex->dc = RW.help_dc, vk_exec_draw_ctx_use(RW.help_dc); + x_cur = ex, rw_chunk = c, rw_ordered = !c->idx && groups_before_done(g); + vk_exec_xfer_worker(1, g->cond + c->cond0, c->ncond, c->oq, c->slot0, c->nslots, c->idx, (int)(g - GQ)); ex->cb = vk_sync_worker_begin(pool); // the replay's command buffer before the group ended behind a full barrier, with every image in GENERAL ex->recorded = 0, ex->barrier_at = 0, ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL; @@ -1949,24 +2112,104 @@ static void chunk_record(Chunk *c, int pool) { } k++, o = nvn_op_next(o); } - if (!c->idx) __atomic_store_n(&G.hooks, 1, __ATOMIC_RELEASE); // a chunk 0 that began no pass + if (!c->idx) __atomic_store_n(&g->hooks, 1, __ATOMIC_RELEASE); // a chunk 0 that began no pass vk_exec_barrier(ex); // the next command buffer starts after it, every image back in GENERAL vk_sync_worker_end(ex->cb); views_release(ex, ex->fence); free(ex->views); ex->views = NULL; c->cb = ex->cb, c->recorded = ex->recorded; - vk_exec_xfer_worker(0, NULL, 0, 0, 0, 0, 0); + vk_exec_xfer_worker(0, NULL, 0, 0, 0, 0, 0, 0); x_cur = was, rw_chunk = NULL; + if (help) vk_exec_draw_ctx_use(NULL); + __atomic_add_fetch(&RW.chunk_ns, kern_now_ns() - t0, __ATOMIC_RELAXED); + gfx_tl_note('d', 0, (uint64_t)c->idx, 0); __atomic_store_n(&c->done, 1, __ATOMIC_RELEASE); + if (__atomic_add_fetch(&g->ndone, 1, __ATOMIC_ACQ_REL) == g->n) groups_done_advance(); } static void group_take(int pool) { - for (;;) { - int i = __atomic_fetch_add(&G.next, 1, __ATOMIC_ACQ_REL); - if (i >= __atomic_load_n(&G.n, __ATOMIC_ACQUIRE)) return; - chunk_record(&G.c[i], pool); + for (Chunk *c; (c = chunk_take());) chunk_record(c, pool); +} + +// The submitting thread past groups it handed out (RW.pipe): until every chunk of them is done. With `help` (only +// between two ops) it records the chunks no worker took yet itself. Then nothing of them runs beside it any more. +static void group_wait(int why, int help) { + if (!rw_behind) return; + uint64_t t0 = kern_now_ns(); + int done = __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) >= Q.out; + if (!done) { + gfx_tl_note('q', t0, (uint64_t)(uintptr_t)pw_name[why], 0); + if (help) group_take(RW.n); + while (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < Q.out) plat_pause(); + gfx_tl_note('u', 0, 0, 0); } + rw_behind = 0; + uint64_t t1 = kern_now_ns(); + if (done) RW.pw_n[PW_DONE]++; + else RW.pw_n[why]++, RW.pw_ns[why] += t1 - t0; +} + +// A done group's command buffers into their places in the part, its closed query slots into their segments. +static void group_put(VkExec *ex, Group *g) { + vk_exec_xfer_group_merge((int)(g - GQ), g->n); + VkCommandBuffer cbs[GROUP_CHUNKS]; + int rec = 0; + for (int i = 0; i < g->n; i++) cbs[i] = g->c[i].cb, rec += g->c[i].recorded; + vk_sync_append_fill(g->at, cbs, g->n); + if (ex->barrier_at == ex->recorded) ex->barrier_at += rec; // nothing since the chunks' closing barrier + ex->recorded += rec; + RW.behind_ns += kern_now_ns() - g->out_ns; + gfx_tl_note('h', 0, 0, 0); +} + +// The done groups not in their parts yet, oldest first; when every one handed out is, the submitting thread is no +// longer behind. +static void groups_put_done(VkExec *ex) { + uint64_t d = __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE); + while (Q.drained < d) Q.drained++, group_put(ex, &GQ[Q.drained % LIVE_GROUPS]); + if (Q.drained == Q.out) rw_behind = 0; +} + +static void group_drain(VkExec *ex, int why) { + if (Q.drained == Q.out) return; + group_wait(why, 1); + groups_put_done(ex); +} + +// Before the front pass of a new group: its slot holds a group put in its part (the oldest goes in first). +static void group_room(VkExec *ex) { + if (Q.out - Q.drained < LIVE_GROUPS) return; + uint64_t want = Q.drained + 1; + if (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < want) { + uint64_t t0 = kern_now_ns(); + gfx_tl_note('q', t0, (uint64_t)(uintptr_t)pw_name[PW_NEXT], 0); + group_take(RW.n); + while (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < want) plat_pause(); + gfx_tl_note('u', 0, 0, 0); + RW.pw_n[PW_NEXT]++, RW.pw_ns[PW_NEXT] += kern_now_ns() - t0; + } + groups_put_done(ex); +} + +// The ops the submitting thread replays past a group it handed out; before any other it waits for the group. The +// ops a chunk records, the game's barriers, dispatches, and events (a wait reads memory, a signal's value goes there +// from the completion thread): their lookups go again once the group is done (op_again), and they leave what the +// chunks read as it was (vk_exec_shared, vk_exec_epoch, vk_exec_upd_top). +static int pipe_op(uint32_t op) { + return run_op(op) || op == OP_BARRIER || op == OP_DISPATCH || op == OP_DISPATCH_IND || op == OP_WAIT_EVENT || + op == OP_SIGNAL_EVENT; +} + +// A lookup of op h would have made or taken back a texture entry before the group it went on past was done (the +// submitting thread's vk_texture_deferred): once it is, h goes again, as a chunk's op does in turn. +static void op_again(VkExec *ex, const op_hdr *h) { + vk_exec_pass_end(ex); + ex->rt_resolved = 0, ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + group_wait(PW_TEX, 1); + __atomic_add_fetch(&RW.redos, 1, __ATOMIC_RELAXED); + op_one(ex, h); } // The submitting thread, at op h of the walk: when a group large enough to split starts here, it is recorded by the @@ -1987,23 +2230,30 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { RW.skip_to[depth] = end, RW.serial_draws += draws, RW.small_end[why] += draws; return h; } + if (RW.pipe) { + // a part due goes to the GPU before the group, with the groups still recording in it: its GPU work would + // otherwise wait for this group's chunks too + if (split_draws && split_wanted()) group_drain(ex, PW_SPLIT), maybe_split(ex); + group_room(ex); + } + Group *g = &GQ[(Q.out + 1) % LIVE_GROUPS]; uint64_t t0 = kern_now_ns(); gfx_tl_note('g', 0, draws, (uint64_t)n); vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL - vk_exec_draw_group_begin(ex); + g->upd = vk_exec_draw_group_begin(ex); // the front pass uint32_t per = (draws + (uint32_t)n - 1) / (uint32_t)n, cd = 0; int nc = 1, cut = 0; - G.ncond = 0; - vk_exec_xfer_group(1); - Chunk *c = chunk_open(ex, h, 0); + g->ncond = 0; + vk_exec_xfer_group((int)(g - GQ)); + Chunk *c = chunk_open(g, ex, h, 0); for (const op_hdr *o = h; o != end;) { const void *a = nvn_op_payload(o); if (o->op == OP_JUMP) { o = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; } - if (cut) c = chunk_open(ex, o, nc++), cd = 0, cut = 0; + if (cut) c = chunk_open(g, ex, o, nc++), cd = 0, cut = 0; switch (o->op) { case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: case OP_DRAW_MESH: case OP_DRAW_MESH_IND: if (vk_exec_xfer_counting() && !ex->render_disabled) c->nslots++; // at most one query each @@ -2013,11 +2263,11 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { case OP_REPORT_COUNTER: case OP_RESET_COUNTER: case OP_RENDER_ENABLE: case OP_RENDER_COND: { int on = vk_exec_xfer_front(ex, o->op, a); if (o->op == OP_RENDER_COND) { - if (G.ncond == G.ccond) { - G.ccond = G.ccond ? G.ccond * 2 : 1024; - if (!(G.cond = realloc(G.cond, G.ccond))) rt_fatal("vk: out of memory (render conditions)"); + if (g->ncond == g->ccond) { + g->ccond = g->ccond ? g->ccond * 2 : 1024; + if (!(g->cond = realloc(g->cond, g->ccond))) rt_fatal("vk: out of memory (render conditions)"); } - G.cond[G.ncond++] = (uint8_t)on; + g->cond[g->ncond++] = (uint8_t)on; c->ncond++; } break; @@ -2027,35 +2277,42 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { c->nops++; o = nvn_op_next(o); } - vk_exec_xfer_group(0); // the segments stay listed for the workers until the next group - for (int i = 0; i < nc; i++) G.c[i].slot0 = vk_exec_xfer_slots(G.c[i].nslots); // in the order of the stream - // handed out: the workers and this thread take the chunks in order - for (int i = 0; i < nc; i++) G.c[i].done = 0; - G.hooks = 0; - __atomic_store_n(&G.n, nc, __ATOMIC_RELEASE); - __atomic_store_n(&G.next, 0, __ATOMIC_RELEASE); + vk_exec_xfer_group(-1); // the segments stay listed for the workers until the group is put in its part + for (int i = 0; i < nc; i++) g->c[i].slot0 = vk_exec_xfer_slots(g->c[i].nslots); // in the order of the stream + // handed out: the workers (and without RW.pipe this thread) take the chunks in order + for (int i = 0; i < nc; i++) g->c[i].done = 0; + g->hooks = 0, g->ndone = 0, g->n = nc; + __atomic_store_n(&g->seq, ++Q.out, __ATOMIC_RELEASE); + // the replay's command buffer so far goes into the part, then the chunks' places, and the replay goes on in a new + // one at once + ex->cb = vk_sync_append_later(nc, &g->at); + for (int i = 0; i < nc; i++) Q.q[(Q.put + (uint64_t)i) % QN] = &g->c[i]; uint64_t tf = kern_now_ns(); - __atomic_add_fetch(&G.gen, 1, __ATOMIC_SEQ_CST); + g->out_ns = tf; + __atomic_store_n(&Q.put, Q.put + (uint64_t)nc, __ATOMIC_SEQ_CST); for (int k = 0; k < RW.n; k++) if (__atomic_load_n(&RW.w[k]->sleeping, __ATOMIC_SEQ_CST)) plat_event_set(&RW.w[k]->wake); - group_take(RW.n); - uint64_t to = kern_now_ns(); - for (int i = 0; i < nc; i++) - while (!__atomic_load_n(&G.c[i].done, __ATOMIC_ACQUIRE)) plat_pause(); - uint64_t tw = kern_now_ns(); - RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.spin_ns += tw - to; - vk_exec_xfer_group_merge(nc); // the chunks' closed query slots into their segments - __atomic_store_n(&G.next, 1 << 30, __ATOMIC_RELEASE); - // the state the front pass left goes on in a new command buffer of the replay's, after the chunks' in the part - VkCommandBuffer cbs[GROUP_CHUNKS]; - int rec = 0; - for (int i = 0; i < nc; i++) cbs[i] = G.c[i].cb, rec += G.c[i].recorded; - ex->cb = vk_sync_append(cbs, nc); - gfx_tl_note('h', 0, 0, 0); - ex->recorded += rec, ex->barrier_at = ex->recorded; // the last chunk ended with a full barrier + ex->barrier_at = ex->recorded; // the chunks end with a full barrier ex->in_pass = 0, ex->nfbl = 0, ex->mark = NULL, ex->dirty = VK_EXEC_DIRTY_ALL, ex->rt_resolved = 0, ex->fb_fs = NULL; vk_exec_draw_new_cb(); - RW.groups++, RW.group_draws += draws, RW.chunks += (uint64_t)nc, RW.wait_ns += kern_now_ns() - t0; + RW.groups++, RW.group_draws += draws, RW.chunks += (uint64_t)nc; + if (RW.pipe) { + // the submitting thread takes chunks too, as the workers wake up, and goes on once none is left to take: the + // GPU waits for the group's part until its last chunk is done + rw_behind = 1; + group_take(RW.n); + uint64_t to = kern_now_ns(); + RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.wait_ns += to - t0; + return end; + } + group_take(RW.n); + uint64_t to = kern_now_ns(); + while (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < g->seq) plat_pause(); + uint64_t tw = kern_now_ns(); + RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.spin_ns += tw - to; + groups_put_done(ex); + vk_exec_draw_new_cb(); // the submitting thread's own chunks used its draw context + RW.wait_ns += kern_now_ns() - t0; return end; } @@ -2065,6 +2322,7 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); if (RW.skip_to[depth] == h) RW.skip_to[depth] = NULL; // the small group read ahead is behind + if (Q.drained < Q.out && Q.drained < __atomic_load_n(&Q.done, __ATOMIC_ACQUIRE)) groups_put_done(ex); if (RW.n > 0 && !RW.skip_to[depth] && !evo_in && h->op != OP_JUMP && run_op(h->op)) { const op_hdr *stop = group_start(ex, h, depth); if (stop != h) { @@ -2095,14 +2353,16 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { } default: if (RW.n > 0) { // the perf line's split of the replay outside groups + if (Q.drained < Q.out && !pipe_op(h->op)) RW.pd_op[h->op]++, group_drain(ex, PW_OP); int k = ser_kind(h->op); uint64_t t = kern_now_ns(); op_one(ex, h); + if (vk_texture_deferred_take()) op_again(ex, h); RW.ser_n[k]++, RW.ser_ns[k] += kern_now_ns() - t; } else op_one(ex, h); break; } - if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); + if (vk_exec_signalled && depth == 0 && !evo_in) evo_resume(ex); // their ops go through this walk's checks if (split_draws && split_wanted()) maybe_split(ex); h = nvn_op_next(h); } @@ -2161,6 +2421,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t VkExec *ex = &X; if (!x_ready) state_reset(ex), vk_dump_init(), x_ready = 1; if (RW.n < 0) rw_init(); + if (Q.drained != Q.out) rt_fatal("vk: a parallel group outlived its submission"); const tex_t *keep[9]; // the render targets carry over to this submission (set_rt compares them by pointer) memcpy(keep, ex->rt.color, sizeof ex->rt.color); keep[8] = ex->rt.depth; @@ -2224,6 +2485,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t AH.live = 0; evo_queue = 0; } + group_drain(ex, PW_END); vk_exec_pass_end(ex); if (RR.on > 0) runs_break(RB_SUBMIT); x_cur = NULL; @@ -2306,6 +2568,35 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t (double)RW.front_ns / ns, (double)RW.own_ns / ns, (double)RW.spin_ns / ns); memset(RW.ser_n, 0, sizeof RW.ser_n), memset(RW.ser_ns, 0, sizeof RW.ser_ns); RW.front_ns = RW.own_ns = RW.spin_ns = 0; + rt_log("perf: replay workers: the chunks took %.3f ms per submission, every thread's", + (double)__atomic_exchange_n(&RW.chunk_ns, 0, __ATOMIC_RELAXED) / ns); + if (RW.pipe) { + o = 0; + for (int k = 0; k < PW_N; k++) + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.2f (%.3f ms)", k ? "," : "", pw_name[k], + (double)RW.pw_n[k] / (double)n_subs, (double)RW.pw_ns[k] / ns); + rt_log("perf: replay workers: past a group the submitting thread went on %.3f ms per submission until " + "it was done; waits for it per submission:%s", (double)RW.behind_ns / ns, e); + o = 0; + for (int k = 0; k < 5; k++) { // the ops that waited the most often + int best = 0; + for (int op = 1; op < OP_COUNT_; op++) + if (RW.pd_op[op] > RW.pd_op[best]) best = op; + if (!RW.pd_op[best]) break; + o += snprintf(e + o, sizeof e - (size_t)o, "%s %s %.2f", k ? "," : "", nvn_op_name((uint32_t)best), + (double)RW.pd_op[best] / (double)n_subs); + RW.pd_op[best] = 0; + } + if (o) rt_log("perf: replay workers: ops that waited for a group, per submission:%s", e); + uint64_t td[3]; + vk_texture_deferred_stats(td); + rt_log("perf: replay workers: texture lookups put off until their turn, per submission: no entry " + "%.2f, a retired one %.2f, a released one %.2f", (double)td[0] / (double)n_subs, + (double)td[1] / (double)n_subs, (double)td[2] / (double)n_subs); + memset(RW.pw_n, 0, sizeof RW.pw_n), memset(RW.pw_ns, 0, sizeof RW.pw_ns); + memset(RW.pd_op, 0, sizeof RW.pd_op); + RW.behind_ns = 0; + } } RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index de4dd75d..47fa8696 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -155,20 +155,24 @@ void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence); void *vk_exec_draw_ctx_new(void); void vk_exec_draw_ctx_use(void *ctx); // vk_exec_draw.c, before the chunks of a parallel group are recorded (submitting thread): the stand-ins are made and the -// texture pool set this submission took is brought up to date, which the chunks then only bind. -void vk_exec_draw_group_begin(VkExec *ex); -// vk_exec_xfer.c, parallel groups: vk_exec_xfer_group turns the front pass on (it lists the group's occlusion segments) -// or off; vk_exec_xfer_front replays a counter report or reset, render enable or render condition in the front pass -// (returns the condition's decision, 1 renders); vk_exec_xfer_chunk_state is the occlusion counting at a chunk's first -// op, which vk_exec_xfer_worker hands to the worker side with the chunk's decisions (on), or turns it off. -void vk_exec_xfer_group(int on); +// texture pool set this submission took is brought up to date, which the chunks then only bind. Returns the inline +// uniform updates of the submission so far (vk_exec_upd_top). +uint32_t vk_exec_draw_group_begin(VkExec *ex); +// vk_exec.c: of the submission's inline uniform updates (all), those a draw on the calling thread may read. +uint32_t vk_exec_upd_top(uint32_t all); +// vk_exec_xfer.c, parallel groups: vk_exec_xfer_group turns the front pass of the group in slot `group` on (it lists +// the group's occlusion segments) or off (-1); vk_exec_xfer_front replays a counter report or reset, render enable or +// render condition in the front pass (returns the condition's decision, 1 renders); vk_exec_xfer_chunk_state is the +// occlusion counting at a chunk's first op, which vk_exec_xfer_worker hands to the worker side with the chunk's +// decisions and its group's slot (on), or turns it off. +void vk_exec_xfer_group(int group); int vk_exec_xfer_front(VkExec *fx, uint32_t op, const void *payload); uint64_t vk_exec_xfer_chunk_state(void); void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots, - int chunk); -// After every chunk of the group is done (submitting thread): the slots the chunks closed go into their segments, in -// the order of the chunks. -void vk_exec_xfer_group_merge(int nchunks); + int chunk, int group); +// After every chunk of the group in slot `group` is done (submitting thread, groups oldest first): the slots the +// chunks closed go into their segments, in the order of the chunks. +void vk_exec_xfer_group_merge(int group, int nchunks); // The front pass: whether a draw here counts samples (a query slot), and n slots of the ring in the order of the stream // for one chunk (the first is returned; vk_exec_xfer_worker hands them to the chunk's worker). int vk_exec_xfer_counting(void); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 43f8136b..01d28755 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -242,6 +242,21 @@ typedef struct { } MemoKey; // ---- the recording context +// bind_resources: the sampled-texture descriptors this context resolved, each with the generations and slot token it +// was resolved under: while its slot, the submission and the samplers are as they were (slot_still, as the texture +// descriptor memo TD checks, without the lock) it holds what TD would give, and a hit takes no texture lock. The +// replay workers' chunks waited for that lock at nearly every draw: 6.1 s of waits a minute in bind_resources alone +// (Hyrule Field, 3 workers, NX2NAVI_LOCK_TIME, 2026-11-12). +#define TM_N 512 +typedef struct { + uint64_t h, tok; + uint32_t gen, ep, sg; + uint16_t type; + uint8_t separate, valid; + VkImage image; + VkDescriptorImageInfo out; +} TexMemo; + // What the draws, binds and dispatches keep from one call to the next while they record a command buffer: the state // the command buffer holds (B), the signatures that let a draw skip its descriptor build or push (DS, RS, PB), the // previous draw's shaders and pipeline (SM, PM), the shader lookup's caches (AC, stagem, slotm) and the submission's @@ -280,6 +295,7 @@ typedef struct DrawCtx { struct { const void *d; uint64_t ind; VkDeviceSize at; } ma[64]; // mesh_ring_entry: entries filled ahead int ma_n, ma_i; // ... how many, and the next one a draw takes uint32_t ma_brk; // ... vk_exec_brk_now when they were filled + TexMemo TM[TM_N]; // bind_resources: sampled-texture descriptors (tm_take) VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views VkDescSlot bind_d[MAX_DESC]; VkBufferView bind_bv[MAX_DESC]; @@ -1083,7 +1099,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded set = shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)]; - int lk = vk_exec_in_chunk(); // only the chunks of a parallel group look up at once + int lk = vk_exec_shared(); // only while a parallel group records do several threads look up at once if (lk) plat_mutex_lock(&shader_lock); for (int w = 0; w < SHADER_WAYS && set[w].used; w++) { ShaderEnt *e = &set[w]; @@ -1111,7 +1127,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st, (unsigned long long)serial); else if (set) { // in front, the least recently used way drops out - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&shader_lock); memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set); set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s}; @@ -1377,7 +1393,8 @@ static void upd_note(const VkExec *ex, const p_update_ub *u) { // 1 when an update of this submission supplied [a, a+n). static int upd_read(const VkExec *ex, uint64_t a, void *out, size_t n) { if (UPD.fence != ex->fence) return 0; - for (uint32_t i = UPD.n, end = UPD.n > UPD_N ? UPD.n - UPD_N : 0; i-- > end;) { + uint32_t top = vk_exec_upd_top(UPD.n); // a replay worker's chunk: the updates before its group + for (uint32_t i = top, end = top > UPD_N ? top - UPD_N : 0; i-- > end;) { const typeof(UPD.u[0]) *e = &UPD.u[i % UPD_N]; if (a >= e->hi || a + n <= e->lo) continue; if (a < e->lo || a + n > e->hi) return 0; // partly covered: memory @@ -1787,10 +1804,14 @@ static struct { } TD[TD_N]; static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, - VkDescriptorImageInfo *out) { + VkDescriptorImageInfo *out, VkImage *img_out) { int td_on = td_mode(); - VkImage image; - if (!h || storage || !td_on) return (void)tex_info_full(ex, r, h, storage, separate, out, &image); + VkImage image = VK_NULL_HANDLE; + if (!h || storage || !td_on) { + tex_info_full(ex, r, h, storage, separate, out, &image); + if (img_out) *img_out = image; + return; + } uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 51]; if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->sgen == sg && @@ -1802,6 +1823,7 @@ static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int s *out = m->out; out->imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT : VK_IMAGE_LAYOUT_GENERAL; + if (img_out) *img_out = m->image; if (td_on == 2) { VkDescriptorImageInfo full; tex_info_full(ex, r, h, storage, separate, &full, &image); @@ -1817,6 +1839,7 @@ static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int s // entry never matches, and the next binding fills it *m = (typeof(*m)){.h = h, .tok = tok, .gen = gen, .epoch = ep, .sgen = sg, .type = r->type, .separate = (uint8_t)separate, .valid = 1, .reduced = (uint8_t)reduced, .image = image, .out = *out}; + if (img_out) *img_out = image; } // Returns whether the image bound was made without its top mip levels (texture detail, VkTexImage.skip). @@ -2470,9 +2493,16 @@ static void pool_sync(VkExec *ex); // A chunk of a parallel group (vk_exec.c "replay workers") binds the set as vk_exec_draw_group_begin left it, up to date // when the group began; the chunk that takes this submission's first set takes it in turn, under pool_mt. Elsewhere the -// replay syncs the pools at every draw that binds them, as before. +// replay syncs the pools at every draw that binds them, as before, under pool_mt past a group whose chunks still record +// (NX2NAVI_REPLAY_PIPE): the set is the submission's, read by the GPU when the part runs. static void pool_sync_draw(VkExec *ex) { - if (!vk_exec_in_chunk()) return pool_sync(ex); + if (!vk_exec_shared()) return pool_sync(ex); + if (!vk_exec_in_chunk()) { + plat_mutex_lock(&pool_mt); + pool_sync(ex); + plat_mutex_unlock(&pool_mt); + return; + } if (__atomic_load_n(&PL.cur_fence, __ATOMIC_ACQUIRE) == ex->fence) return; vk_exec_turn(); plat_mutex_lock(&pool_mt); @@ -2482,11 +2512,17 @@ static void pool_sync_draw(VkExec *ex) { static int mesh_ring_make(void); -void vk_exec_draw_group_begin(VkExec *ex) { +uint32_t vk_exec_draw_group_begin(VkExec *ex) { dummies_init(); if (vk_ctx()->has_mesh_shader && !vk_ctx()->mesh_fallback) mesh_ring_make(); // the chunks' indirect mesh draws if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); - if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) pool_sync(ex); // the set this submission took + if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) { // the set this submission took + int lk = vk_exec_shared(); // earlier groups may still record (NX2NAVI_REPLAY_PIPE) + if (lk) plat_mutex_lock(&pool_mt); + pool_sync(ex); + if (lk) plat_mutex_unlock(&pool_mt); + } + return UPD.fence == ex->fence ? UPD.n : 0; } static void pool_sync(VkExec *ex) { @@ -2749,7 +2785,8 @@ static void pool_sync_entries(VkExec *ex) { typedef struct { VkImage src, img; VmaAllocation mem; VkImageView view[2]; uint32_t w, h, layers; VkFormat fmt; } Twin; static Twin twins[TWINS]; static unsigned twin_next; -static int twin_dispatch; // op_dispatch*: the resources being bound are a dispatch's +// op_dispatch*: the resources being bound are a dispatch's, on this thread (the replay workers record draws meanwhile) +static _Thread_local int twin_dispatch; static VkBuffer twin_buf; static VmaAllocation twin_buf_mem; static VkDeviceSize twin_buf_cap; @@ -2899,8 +2936,9 @@ static Twin *twin_update(VkExec *ex, const VkTexImage *img) { // The image descriptor of slot h; in a dispatch, a compare-sampled colour image the driver cannot compare is swapped // for its depth twin; in a pass larger than it, the scene's light target, once DLSS scaled it this frame, for DLSS's // output (vk_dlaa.c). -static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) { - tex_info_lookup(ex, r, h, storage, separate, out); +static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out, + VkImage *img_out) { + tex_info_lookup(ex, r, h, storage, separate, out, img_out); if (h && !storage && vk_dlaa_swap_img && ex->in_pass && ex->pass_w > vk_dlaa_swap_img->width + 1 && vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL) == vk_dlaa_swap_img) out->imageView = vk_dlaa_swap_view, out->imageLayout = VK_IMAGE_LAYOUT_GENERAL; @@ -3116,7 +3154,7 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { DrawCtx *const cx = ex->dc; if (info == cx->rs_last) return cx->rs_last_l; - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&rs_lock); if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info); RsList *l = nvn_hm_get(&rs_lists, &info); @@ -3458,15 +3496,36 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } else { kind = 1; int locked = 0; // tex_info looks up with the texture lock held; kept descriptors (pb_take) need none + // the context's memo (TexMemo) for sampled textures while other threads record (the lock is free + // otherwise), outside a dispatch's depth twins and DLSS's stand-in image, with the generations read + // before any lookup below + int tm = !img && !r->exact && td_mode() == 1 && !vk_dlaa_swap_img && !twin_dispatch && vk_exec_shared(); + uint32_t tg = tm ? vk_texture_slots_gen() : 0, te = tm ? vk_texture_epoch() : 0; + uint32_t ts = tm ? vk_sampler_gen() : 0; for (uint32_t k = 0; k < cnt; k++) { if (pb_take(ex, b, nd + k, k, &o[k])) continue; - if (!locked) vk_texture_lock(), locked = 1; int slot = r->slot + (int)k; uint64_t h = 0; if (img) h = slot >= 0 && slot < 32 ? ex->img[st][slot] : 0; else h = slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0; uint64_t tt = bt_now(); - tex_info(ex, r, h, img, r->separate, &o[k].image); + TexMemo *m = tm && h ? &cx->TM[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 9 ^ + (uint64_t)r->separate) >> 55 & (TM_N - 1)] : NULL; + if (m && m->valid && m->h == h && m->type == r->type && m->separate == (uint8_t)r->separate && + m->ep == te && m->sg == ts && slot_still(&m->gen, m->tok, h, tg)) { + o[k].image = m->out; + o[k].image.imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) + ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT + : VK_IMAGE_LAYOUT_GENERAL; + } else { + if (!locked) vk_texture_lock(), locked = 1; + VkImage ti = VK_NULL_HANDLE; + uint64_t tok = m ? token_of(h) : 0; // before the lookup + tex_info(ex, r, h, img, r->separate, &o[k].image, &ti); + if (m) *m = (TexMemo){.h = h, .tok = tok, .gen = tg, .ep = te, .sg = ts, .type = r->type, + .separate = (uint8_t)r->separate, .valid = 1, .image = ti, + .out = o[k].image}; + } vk_exec_frame.bind_tex_ns += (uint64_t)(bt_now() - tt); o[k].w[2] = (uint32_t)o[k].image.imageLayout; // zero the tail padding pb_keep(cx, nd + k, &o[k]); @@ -4526,7 +4585,7 @@ static int mo_desc(VkExec *ex, const VkPipeInfo *info, MoDraw *q) { vk_texture_lock(); for (uint32_t k = 0; k < cnt; k++) { int slot = r->slot + (int)k; - tex_info(ex, r, slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0, 0, r->separate, &o[k].image); + tex_info(ex, r, slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0, 0, r->separate, &o[k].image, NULL); } vk_texture_unlock(); } else { @@ -4667,7 +4726,7 @@ static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSiz const GfxShader *vs; uint32_t banks; int listed = mvobj_pick(ex, &vs, &banks, t0); - int lk = listed && vk_exec_in_chunk(); + int lk = listed && vk_exec_shared(); if (lk) plat_mutex_lock(&mo_lock); if (listed) mvobj_draw_(ex, d, ib, ib_off, t0, vs, banks); if (t0) __atomic_add_fetch(&MO.ns_check, kern_now_ns() - t0, __ATOMIC_RELAXED); @@ -6240,6 +6299,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { vk_exec_frame.bind_calls++; } twin_dispatch = 0; + if (vk_texture_deferred()) return; // past a parallel group: the op goes again once it is done (vk_exec.c op_again) VkPassMark *m = dispatch_mark(ex, cs, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z); vkCmdDispatch(ex->cb, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z); vk_pass_mark_end(ex->cb, m); @@ -6279,6 +6339,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vk_exec_frame.bind_calls++; } twin_dispatch = 0; + if (vk_texture_deferred()) return; // as in op_dispatch if (cx->B.c0_valid[2]) { // the shader binds c0 // NVN2 programs read the group counts from c0 byte 0 (c0_info); at 0x200 their bank holds pushed constants // or the driver's tables, which the copy would overwrite diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index e3dde4fd..5e4c83f8 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -241,36 +241,43 @@ typedef struct { uint64_t addr, seq; int reset; volatile int32_t lost; uint32_t // times the result calls and a reset per slot, 3.5 ms more of the wait after a frame's first submission (RX 7900 XT, // Hyrule Field, 3 workers). A chunk that needs more than its block takes them from the ring. static _Thread_local struct { - int on, open, counting, reported, keep, chunk; + int on, open, counting, reported, keep, chunk, g; uint32_t seg, qnext, qend; const uint8_t *cond; uint32_t ncond, i; } XW = {.open = -1}; -static struct { int on; OqSegment **s; uint32_t n, cap; } GS; // the front pass's segments of the group +// GS: per group slot (vk_exec.c LIVE_GROUPS), the front pass's segments of the group. tail: with NX2NAVI_REPLAY_PIPE +// the submitting thread replays on past the group before its chunks are done: the first segment made after the +// group's front pass (tail_wait), by the next group's front pass or the replay, takes the queries the chunks close +// after the group's last report or reset, which in the stream come before that one's own. +#define GS_GROUPS 4 +static struct { int tail_wait; OqSegment **s, *tail; uint32_t n, cap; } GS[GS_GROUPS]; +static int gs_on = -1; // the group slot whose front pass runs #define CP_CHUNKS 32 -static struct { uint32_t *seg, *q, n, cap; } CP[CP_CHUNKS]; // per chunk index: closed slots and their segment +static struct { uint32_t *seg, *q, n, cap; } CP[GS_GROUPS][CP_CHUNKS]; // per chunk: closed slots and their segment -void vk_exec_xfer_group(int on) { // on: a new group's front pass begins; off: it ended, the list stays - if (on) GS.n = 0; - GS.on = on; +void vk_exec_xfer_group(int group) { // a group's front pass begins (its slot), or it ended (-1): the list stays + if (group >= GS_GROUPS) rt_fatal("vk: parallel group slot %d of %d", group, GS_GROUPS); + if (group >= 0) GS[group].n = 0, GS[group].tail_wait = 0, GS[group].tail = NULL; + else if (gs_on >= 0) GS[gs_on].tail_wait = 1; // until vk_exec_xfer_group_merge + gs_on = group; } int vk_exec_xfer_counting(void) { return oq_counting; } -uint32_t vk_exec_xfer_slots(uint32_t n) { - uint32_t first = oq_next; - oq_next += n; - return first; +uint32_t vk_exec_xfer_slots(uint32_t n) { // the workers of a group the submitting thread went on past take some too + return __atomic_fetch_add(&oq_next, n, __ATOMIC_RELAXED); } uint64_t vk_exec_xfer_chunk_state(void) { - return (uint64_t)GS.n << 8 | (uint64_t)(oq_counting & 1) | (uint64_t)(oq_reported & 1) << 1 | (uint64_t)(oq_keep & 1) << 2; + return (uint64_t)GS[gs_on].n << 8 | (uint64_t)(oq_counting & 1) | (uint64_t)(oq_reported & 1) << 1 | (uint64_t)(oq_keep & 1) << 2; } void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t state, uint32_t slot0, uint32_t nslots, - int chunk) { + int chunk, int group) { if (chunk < 0 || chunk >= CP_CHUNKS) rt_fatal("vk: replay worker chunk %d of at most %d", chunk, CP_CHUNKS); - XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0, XW.chunk = chunk; + if (group < 0 || group >= GS_GROUPS) rt_fatal("vk: parallel group slot %d of %d", group, GS_GROUPS); + XW.on = on, XW.open = -1, XW.cond = cond, XW.ncond = ncond, XW.i = 0, XW.chunk = chunk, XW.g = group; XW.qnext = slot0, XW.qend = slot0 + nslots; XW.counting = (int)(state & 1), XW.reported = (int)(state >> 1 & 1), XW.keep = (int)(state >> 2 & 1); XW.seg = (uint32_t)(state >> 8); @@ -278,7 +285,7 @@ void vk_exec_xfer_worker(int on, const uint8_t *cond, uint32_t ncond, uint64_t s // Adds closed slot q to the segment a worker's queries belong to now, or the pending list. static void seg_add(uint32_t q) { - __typeof__(CP[0]) *l = &CP[XW.chunk]; + __typeof__(CP[0][0]) *l = &CP[XW.g][XW.chunk]; if (l->n == l->cap) { l->cap = l->cap ? l->cap * 2 : 256; if (!(l->seg = realloc(l->seg, sizeof *l->seg * l->cap)) || !(l->q = realloc(l->q, sizeof *l->q * l->cap))) @@ -287,13 +294,21 @@ static void seg_add(uint32_t q) { l->seg[l->n] = XW.seg, l->q[l->n++] = q; } -void vk_exec_xfer_group_merge(int nchunks) { +void vk_exec_xfer_group_merge(int group, int nchunks) { + __typeof__(GS[0]) *gs = &GS[group]; for (int c = 0; c < nchunks && c < CP_CHUNKS; c++) { - __typeof__(CP[0]) *l = &CP[c]; + __typeof__(CP[0][0]) *l = &CP[group][c]; for (uint32_t i = 0; i < l->n; i++) { uint32_t q = l->q[i]; - if (l->seg[i] < GS.n) { - OqSegment *s = GS.s[l->seg[i]]; + if (l->seg[i] < gs->n) { + OqSegment *s = gs->s[l->seg[i]]; + if (s->n == s->cap) { + s->cap = s->cap ? s->cap * 2 : 16; + if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); + } + s->q[s->n++] = q; + } else if (gs->tail) { + OqSegment *s = gs->tail; if (s->n == s->cap) { s->cap = s->cap ? s->cap * 2 : 16; if (!(s->q = realloc(s->q, sizeof *s->q * s->cap))) rt_fatal("vk: out of memory"); @@ -309,6 +324,7 @@ void vk_exec_xfer_group_merge(int nchunks) { } l->n = 0; } + gs->tail_wait = 0, gs->tail = NULL; } static int oq_ready(void) { @@ -356,17 +372,16 @@ static void oq_close(VkExec *ex) { void vk_exec_oq_draw(VkExec *ex) { int *open = XW.on ? &XW.open : &oq_open; if (!(XW.on ? XW.counting : oq_counting) || *open >= 0) return; - // a worker takes the next slot of its chunk's block, past it one from the ring at once; a busy one is lost, not - // tried again - uint32_t q = (!XW.on ? oq_next : XW.qnext != XW.qend ? XW.qnext++ : __atomic_fetch_add(&oq_next, 1, __ATOMIC_RELAXED)) % OQ_N; + // a worker takes the next slot of its chunk's block, past it one from the ring at once (as the replay does, while + // a group's workers may still take some); a busy one is lost, not tried again + uint32_t q = (XW.on && XW.qnext != XW.qend ? XW.qnext++ : __atomic_fetch_add(&oq_next, 1, __ATOMIC_RELAXED)) % OQ_N; if (__atomic_load_n(&oq_busy[q], __ATOMIC_ACQUIRE)) { warn_once(W_QUERY, "occlusion query ring full: draws left uncounted"); - if (XW.on && XW.seg < GS.n) __atomic_store_n(&GS.s[XW.seg]->lost, 1, __ATOMIC_RELAXED); + if (XW.on && XW.seg < GS[XW.g].n) __atomic_store_n(&GS[XW.g].s[XW.seg]->lost, 1, __ATOMIC_RELAXED); else __atomic_store_n(&oq_lost, 1, __ATOMIC_RELAXED); return; } __atomic_store_n(&oq_busy[q], 1, __ATOMIC_RELAXED); - if (!XW.on) oq_next++; vkCmdBeginQuery(ex->cb, oq_pool, q, oq_flags); ex->recorded++; *open = (int)q; @@ -500,13 +515,16 @@ static void oq_segment(VkExec *ex, uint64_t addr, int reset) { s->cap = oq_nseg; s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg; if (oq_nseg) memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg); - if (GS.on) { // the front pass: the workers add the queries of the group's ops before it - if (GS.n == GS.cap) { - GS.cap = GS.cap ? GS.cap * 2 : 64; - if (!(GS.s = realloc(GS.s, sizeof *GS.s * GS.cap))) rt_fatal("vk: out of memory"); + if (gs_on >= 0) { // the front pass: the workers add the queries of the group's ops before it + __typeof__(GS[0]) *gs = &GS[gs_on]; + if (gs->n == gs->cap) { + gs->cap = gs->cap ? gs->cap * 2 : 64; + if (!(gs->s = realloc(gs->s, sizeof *gs->s * gs->cap))) rt_fatal("vk: out of memory"); } - GS.s[GS.n++] = s; + gs->s[gs->n++] = s; } + for (int g = 0; g < GS_GROUPS; g++) // the first made after a group's front pass + if (GS[g].tail_wait && !GS[g].tail) GS[g].tail = s; oq_nseg = 0, oq_lost = 0; vk_completion_add_result(ex->fence, oq_cb, s); } diff --git a/runtime/vk_hdr.c b/runtime/vk_hdr.c index 108d552b..43ef0152 100644 --- a/runtime/vk_hdr.c +++ b/runtime/vk_hdr.c @@ -755,7 +755,8 @@ void vk_hdr_shader_swap(uint32_t **spirv, uint64_t *words) { int vk_hdr_track; // vk_exec_draw.c calls vk_hdr_draw: set at the first present with NX2NAVI_HDR=1 static volatile int32_t g_active; // the swapchain is HDR10 (gfx_present.c vk_hdr_set_active) static PlatMutex rec_lock = PLAT_MUTEX_INIT; -static struct { VkImage win, src; VkFormat fmt; } recs[4]; // per presented window image: what the last draw copied +// per presented window image: what the last draw copied, and where in the stream (vk_exec_stream_pos) +static struct { VkImage win, src; VkFormat fmt; uint64_t pos; } recs[4]; static int nrecs; void vk_hdr_set_active(int on) { plat_xchg32(&g_active, on); } @@ -773,7 +774,8 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) { for (int i = 0; i < 4; i++) // unlocked first look: most draws go elsewhere (a stale slot only costs the lock below) known |= __atomic_load_n(&recs[i].win, __ATOMIC_RELAXED) == t->image; if (!known) return; - vk_exec_turn(); // a replay worker's chunk: the last such draw of the stream decides, as in the replay + // the last such draw of the stream decides, as in the replay, also when replay workers record draws at once + uint64_t pos = vk_exec_stream_pos(); VkImage src = VK_NULL_HANDLE; VkFormat fmt = VK_FORMAT_UNDEFINED; int ntex = 0, slot = -1; @@ -790,7 +792,7 @@ void vk_hdr_draw(VkExec *ex, const VkPipeInfo *info) { } plat_mutex_lock(&rec_lock); for (int i = 0; i < nrecs; i++) - if (recs[i].win == t->image) recs[i].src = src, recs[i].fmt = fmt; + if (recs[i].win == t->image && pos >= recs[i].pos) recs[i].src = src, recs[i].fmt = fmt, recs[i].pos = pos; plat_mutex_unlock(&rec_lock); } @@ -801,7 +803,7 @@ void vk_hdr_present_source(const VkTexImage *win, VkImage *image, VkFormat *form int r = -1; for (int i = 0; i < nrecs; i++) if (recs[i].win == win->image) r = i; - if (r < 0 && nrecs < 4) r = nrecs++, recs[r].win = win->image, recs[r].src = VK_NULL_HANDLE; + if (r < 0 && nrecs < 4) r = nrecs++, recs[r].win = win->image, recs[r].src = VK_NULL_HANDLE, recs[r].pos = 0; // kept until the next draw into the window texture: a present again without a new copy shows the same picture if (r >= 0 && recs[r].src && vk_hdr_active()) *image = recs[r].src, *format = recs[r].fmt; plat_mutex_unlock(&rec_lock); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 7fc26090..1571b8b8 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2609,7 +2609,7 @@ void vk_mem_use(VkMemSpan *sp) { } // a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors) if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) return; - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); mem_use_locked(sp); if (lk) plat_mutex_unlock(&use_lock); @@ -2634,7 +2634,7 @@ void vk_mem_pick(VkMemSpan *sp) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON) return; if (r->acc_sub != msub) { - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); if (r->acc_sub != msub) mirror_sync(r); if (lk) plat_mutex_unlock(&use_lock); @@ -3256,13 +3256,13 @@ static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) { range_add(&wbl, &nwbl, &cwbl, r, sp->offset + o, size - o); // what found no snapshot room } void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size) { - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); gpu_wrote(sp, size, 0); if (lk) plat_mutex_unlock(&use_lock); } void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size) { - int lk = vk_exec_in_chunk(); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); gpu_wrote(sp, size, 1); if (lk) plat_mutex_unlock(&use_lock); diff --git a/runtime/vk_mvprev.c b/runtime/vk_mvprev.c index 682fd6d9..31c506cc 100644 --- a/runtime/vk_mvprev.c +++ b/runtime/vk_mvprev.c @@ -138,7 +138,7 @@ static PlatMutex draw_lock = PLAT_MUTEX_INIT; // the replay workers' scene dra static void mvprev_draw_(VkExec *ex) { const VkExecTarget *z = &ex->depth; if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT) return; - int lk = vk_exec_in_chunk(); // only the chunks of a parallel group (vk_exec.c) note at once + int lk = vk_exec_shared(); // only while a parallel group (vk_exec.c) records do several threads note at once if (lk) plat_mutex_lock(&draw_lock); mvprev_draw_locked(ex); if (lk) plat_mutex_unlock(&draw_lock); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 37a5591d..11b3db00 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -456,8 +456,9 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order // NX2NAVI_GPU_TIMELINE=: from that presented frame on, TL_FRAMES frames of what the CPU and the GPU did go to the // log once they are complete, in milliseconds from the first of those presents: each replay's begin and end and its // thread, each timed part's submit, GPU start and end (calibrated timestamps, so with NX2NAVI_TRACE=perf), the waits for -// the GPU after a frame's first submission, the replay workers' groups and the presents. It tells whether the GPU waits -// for the replay, for a wait, or for the CPU behind one. Off (and one load per note) without the setting. +// the GPU after a frame's first submission, the replay workers' groups, chunks and the submitting thread's waits for +// them, and the presents. It tells whether the GPU waits for the replay, for a wait, or for the CPU behind one. Off +// (and one load per note) without the setting. #define TL_CAP 16384 #define TL_FRAMES 4 typedef struct { uint64_t t, a, b; uint32_t tid; int kind; } TlEv; @@ -502,7 +503,12 @@ static void tl_dump(void) { case 'w': rt_log("vk: tl %8.3f t%-5u wait ends", at, e->tid); break; case 'g': rt_log("vk: tl %8.3f t%-5u group of %llu draws in %llu chunks", at, e->tid, (unsigned long long)e->a, (unsigned long long)e->b); break; - case 'h': rt_log("vk: tl %8.3f t%-5u group recorded", at, e->tid); break; + case 'h': rt_log("vk: tl %8.3f t%-5u group recorded and in the part", at, e->tid); break; + case 'q': rt_log("vk: tl %8.3f t%-5u waits for its group: %s", at, e->tid, (const char *)(uintptr_t)e->a); break; + case 'u': rt_log("vk: tl %8.3f t%-5u group done, goes on", at, e->tid); break; + case 'c': rt_log("vk: tl %8.3f t%-5u records chunk %llu (%llu ops)", at, e->tid, (unsigned long long)e->a, + (unsigned long long)e->b); break; + case 'd': rt_log("vk: tl %8.3f t%-5u chunk %llu done", at, e->tid, (unsigned long long)e->a); break; case 'S': rt_log("vk: tl %8.3f t%-5u part submitted, value %llu", at, e->tid, (unsigned long long)e->a); break; case 'G': rt_log("vk: tl %8.3f gpu part runs %.3f ms (submitted %.3f ms before)", at, (double)(e->a - e->t) / 1e6, (double)(int64_t)(e->t - e->b) / 1e6); break; @@ -783,25 +789,30 @@ static uint32_t part_list(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, V uint32_t n = 0; VkCommandBufferSubmitInfo one = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}; if (pro) one.commandBuffer = pro, cbi[n++] = one; - for (int i = 0; i < npart_cbs; i++) one.commandBuffer = part_cbs[i], cbi[n++] = one; + for (int i = 0; i < npart_cbs; i++) { + if (!part_cbs[i]) rt_fatal("vk: a replay worker's command buffer is missing from a part"); + one.commandBuffer = part_cbs[i], cbi[n++] = one; + } if (with_last) one.commandBuffer = last, cbi[n++] = one; npart_cbs = 0; return n; } // The replay workers (vk_exec.c "replay workers"): the replay's command buffer being recorded ends and joins the -// part's list, then the n command buffers the workers recorded, in that order, and a new command buffer of the same -// part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers' commands run -// between the replay's commands before and after. vk_sync_append_room: whether the list has room for n more. +// part's list, then n places for the command buffers the workers record, in that order, and a new command buffer of +// the same part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers' +// commands run between the replay's commands before and after. The places are filled (vk_sync_append_fill) before the +// part is submitted, which part_list checks. vk_sync_append_room: whether the list has room for n more. int vk_sync_append_room(int n) { return npart_cbs + n + 1 <= PART_CBS; } -VkCommandBuffer vk_sync_append(const VkCommandBuffer *cbs, int n) { +VkCommandBuffer vk_sync_append_later(int n, int *at) { if (!vk_sync_append_room(n)) rt_fatal("vk: a submission part of more than %d command buffers", PART_CBS); VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of it is in cb, ended by its recorder part_cbs[npart_cbs++] = cb; - for (int i = 0; i < n; i++) part_cbs[npart_cbs++] = cbs[i]; + *at = npart_cbs; + for (int i = 0; i < n; i++) part_cbs[npart_cbs++] = VK_NULL_HANDLE; CmdSlot *s = take_slot(c, cur_fence); // the ended one keeps its slot until the value is reached VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; @@ -810,10 +821,16 @@ VkCommandBuffer vk_sync_append(const VkCommandBuffer *cbs, int n) { return s->cb; } +void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n) { + if (at < 0 || at + n > npart_cbs) rt_fatal("vk: replay worker command buffers past the part's list"); + for (int i = 0; i < n; i++) part_cbs[at + i] = cbs[i]; +} + // The replay workers' command buffers: each worker has a command pool of its own (a pool belongs to one thread at a // time) and claims command buffers from it for the submission being recorded, as the replay claims its slots. Called -// on the worker while the submitting thread holds sub_lock and waits for it, so cur_fence stays put. -#define SYNC_WORKERS 16 +// on the worker while the submitting thread holds sub_lock and records on (NX2NAVI_REPLAY_PIPE) or waits for it: the +// part with the worker's command buffer is not submitted before it is done, so cur_fence stays put. +#define SYNC_WORKERS 17 // the replay workers (at most 16) and the submitting thread static struct { VkCommandPool pool; CmdSlot *slots; int n; } WP[SYNC_WORKERS]; VkCommandBuffer vk_sync_worker_begin(int w) { diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 151072ec..6c36441c 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -3720,6 +3720,10 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { // returns nothing and sets this; the chunk replays the op again once every earlier chunk is done, so entries are made, // initialised and taken back in the order of the stream, as the replay would. static _Thread_local int tex_deferred; +static volatile uint64_t n_deferred[3]; // by what the lookup found: no entry, a retired one, a released one +void vk_texture_deferred_stats(uint64_t out[3]) { + for (int i = 0; i < 3; i++) out[i] = __atomic_exchange_n(&n_deferred[i], 0, __ATOMIC_RELAXED); +} int vk_texture_deferred(void) { return tex_deferred; } int vk_texture_deferred_take(void) { int d = tex_deferred; @@ -3727,11 +3731,26 @@ int vk_texture_deferred_take(void) { return d; } +// A lookup of k that finds e (NULL: no entry) and needs no turn (vk_exec_turn_due): it returns nothing without changing +// anything (the pool refresh asking for a held key or a retired entry, a creation that failed lately: the texture pool +// refresh looks its unresolved slots up at every sync), or it is the submitting thread's past a group (vk_exec_behind) +// and takes back a released entry whose image stayed its own. That keeps its content and is ordered after the +// group's chunks anyway: one of them that looks the entry up later finds it in use, as it would have after taking it +// back itself. A transient render target the frame releases and takes back at every pass that draws it did that after +// nearly every group (Hyrule Field, 2026-11-12). +static int lookup_quiet(const Tex *e, const TexKey *k) { + if (e && e->retiring) return pool_ctx; + if (e) return !e->superseded && vk_exec_behind(); + if (holds.n && nvn_hm_get(&holds, k)) return pool_ctx; + return create_held(k); +} + static Tex *entry_of(const tex_t *t) { TexKey k; make_key(t, &k); Tex *e = nvn_hm_get(&by_key, &k); - if ((!e || e->retiring || e->released) && vk_exec_turn_due()) { + if ((!e || e->retiring || e->released) && vk_exec_turn_due() && !lookup_quiet(e, &k)) { + __atomic_add_fetch(&n_deferred[!e ? 0 : e->retiring ? 1 : 2], 1, __ATOMIC_RELAXED); tex_deferred = 1; return NULL; } diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index e9e8b4ab..c9a6ff9c 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -44,8 +44,9 @@ typedef struct Arena { } Arena; typedef struct { uint64_t lo, hi; } Rng; -typedef struct { uint64_t lo, hi, off; int dropped; } Reg; -typedef struct { uint64_t lo, hi, fence; } Wrote; +// dropped: 0, or the epoch of its first drop in the stream (vk_exec_epoch); epoch: of the write +typedef struct { uint64_t lo, hi, off, dropped; } Reg; +typedef struct { uint64_t lo, hi, fence, epoch; } Wrote; static struct { int mode; // rt_knob: 0 off, 1 on, 2 ab @@ -422,7 +423,7 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI if (i >= U.nreg || addr < U.reg[i].lo || addr >= U.reg[i].hi) i = first_above(addr); if (i >= U.nreg) return 0; const Reg *r = &U.reg[i]; - if (r->dropped || addr < r->lo || addr + n > r->hi) return 0; + if ((r->dropped && r->dropped <= vk_exec_epoch(0)) || addr < r->lo || addr + n > r->hi) return 0; uint64_t off = r->off + (addr - r->lo); if (off % align) return 0; *out = (VkDescriptorBufferInfo){U.arena->buf, off, n}; @@ -433,15 +434,22 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI static PlatMutex drop_lock = PLAT_MUTEX_INIT; // the replay workers' shaders write storage at once +// The replay workers (vk_exec.c NX2NAVI_REPLAY_PIPE) record a group's draws while the submitting thread goes on past +// it: a drop holds from its place in the stream on (vk_exec_epoch), so a chunk does not see the drops of ops after its +// group, and the submitting thread sees every one. void vk_usnap_drop(uint64_t addr, uint64_t n) { if (!U.mode || !n) return; - int lk = vk_exec_in_chunk(); + uint64_t ep = vk_exec_epoch(1); + int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&drop_lock); if (U.nwr == U.cwr) U.wr = grow(U.wr, &U.cwr, sizeof *U.wr); - U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence}; + U.wr[U.nwr++] = (Wrote){addr, addr + n, U.fence, ep}; if (U.on) for (int i = first_above(addr); i < U.nreg && U.reg[i].lo < addr + n; i++) - if (!U.reg[i].dropped) U.reg[i].dropped = 1, U.drops++, __atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED); + if (!U.reg[i].dropped || ep < U.reg[i].dropped) { + if (!U.reg[i].dropped) U.drops++; + U.reg[i].dropped = ep, __atomic_add_fetch(&U.gen, 1, __ATOMIC_RELAXED); + } if (lk) plat_mutex_unlock(&drop_lock); } @@ -450,8 +458,9 @@ uint64_t vk_usnap_gen(void) { return U.gen; } // A GPU write of a submission not known to be complete touches [lo, hi). A linear scan: vk_usnap_drop appends after // wr_prune sorted the list, so wr_hits's binary search does not hold here; only called on rare paths. static int wr_overlaps(uint64_t lo, uint64_t hi) { + uint64_t ep = vk_exec_epoch(0); for (int i = 0; i < U.nwr; i++) - if (U.wr[i].lo < hi && lo < U.wr[i].hi) return 1; + if (U.wr[i].lo < hi && lo < U.wr[i].hi && U.wr[i].epoch <= ep) return 1; return 0; } @@ -464,7 +473,12 @@ static int cpu_written(uint64_t addr, uint64_t n) { // the same test vk_usnap_ int vk_usnap_read(uint64_t addr, uint64_t n, void *dst) { const uint8_t *s = vk_usnap_bytes(addr, n); if (s) return memcpy(dst, s, n), 1; - if (!n || !cpu_written(addr, n) || wr_overlaps(addr, addr + n)) return 0; + if (!n || !cpu_written(addr, n)) return 0; + int lk = vk_exec_shared(); // a chunk's drop may grow the list + if (lk) plat_mutex_lock(&drop_lock); + int w = wr_overlaps(addr, addr + n); + if (lk) plat_mutex_unlock(&drop_lock); + if (w) return 0; memcpy(dst, (const void *)(uintptr_t)addr, n); return 2; }