From 7bfdd8a4b749f8ab2d1702c647b7398821536634 Mon Sep 17 00:00:00 2001 From: epicmeme546 <178368684+epicmeme546@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:50:39 -0500 Subject: [PATCH] Keep the draw replay's per-recording state in one context a thread reaches through a thread-local pointer. vk_exec_draw.c kept what a draw, bind or dispatch carries to the next one in file statics: the state the command buffer holds, the descriptor signatures and skips, the per-binding reuse, the shader and pipeline memos, the shader lookup's caches and the submission's transient resources. They now live in one DrawCtx, and every thread starts with the replay's own, so the replay behaves as before. A later stage can give each recording thread a context of its own. vk_exec.c's inline commands (vk_exec_inline_begin) find the replay state the calling thread records with through a thread-local pointer too, instead of the file's single state and a flag. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec.c | 23 +- runtime/vk_exec_draw.c | 591 ++++++++++++++++++++++------------------- 2 files changed, 328 insertions(+), 286 deletions(-) diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 48c703e2..a725699f 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -43,7 +43,9 @@ typedef struct { static VkExec X; static int x_ready; -static _Thread_local int x_here; // this thread is inside vk_exec_record (vk_exec_inline_begin) +// The replay state this thread records with now (inside vk_exec_record, or a test's vk_exec_probe_inline), NULL when +// it records none: vk_exec_inline_begin hands out its command buffer. +static _Thread_local VkExec *x_cur; // ---- NX2NAVI_TRACE=gfx frame counters @@ -51,7 +53,7 @@ VkExecFrame vk_exec_frame; static struct { const uint8_t *mem; uint32_t w, h, fmt, n; } fw[64]; // textures written this frame static int nfw; -uint64_t vk_exec_recording_fence(void) { return x_here ? X.fence : 0; } +uint64_t vk_exec_recording_fence(void) { return x_cur ? x_cur->fence : 0; } void vk_exec_frame_written(const tex_t *t) { if (!(rt_trace & RT_TRACE_GFX) || !t) return; @@ -772,22 +774,23 @@ void vk_exec_pass_end(VkExec *ex) { } VkCommandBuffer vk_exec_inline_begin(int *reopen) { - VkExec *ex = &X; + VkExec *ex = x_cur; *reopen = 0; - if (!x_here || !ex->cb) return VK_NULL_HANDLE; + if (!ex || !ex->cb) return VK_NULL_HANDLE; *reopen = ex->in_pass; vk_exec_barrier(ex); // ends the pass; feedback-layout targets go back to GENERAL return ex->cb; } void vk_exec_inline_end(int reopen) { - X.recorded++; - if (reopen) vk_exec_pass_begin(&X); + VkExec *ex = x_cur; // set: the caller's vk_exec_inline_begin handed out its command buffer + ex->recorded++; + if (reopen) vk_exec_pass_begin(ex); } void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence) { X.cb = cb, X.fence = fence; - x_here = cb != VK_NULL_HANDLE; + x_cur = cb != VK_NULL_HANDLE ? &X : NULL; } typedef struct { VkBuffer b; VmaAllocation a; } InlineBuf; @@ -804,7 +807,7 @@ VkBuffer vk_exec_inline_buffer(VkDeviceSize size) { .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; if (!t || vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &t->b, &t->a, NULL) != VK_SUCCESS) { free(t); return VK_NULL_HANDLE; } - vk_completion_add(X.fence, free_inline_buf, t); + vk_completion_add(x_cur->fence, free_inline_buf, t); // inside vk_exec_inline_begin and _end return t->b; } @@ -1696,7 +1699,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t split_draws = part_start = 0; split_at = split_n < 0 ? 0 : split_n ? split_n : INT_MAX; // 0 until maybe_split reads the settings part_t0 = kern_now_ns(); - x_here = 1; + x_cur = ex; int evo = evo_enabled() && gfx_submit_queue; int slot = evo ? evq_slot(gfx_submit_queue) : -1; if (evo && slot < 0) evo = 0; @@ -1740,7 +1743,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t evo_queue = 0; } vk_exec_pass_end(ex); - x_here = 0; + x_cur = NULL; vk_exec_event_sync(0); if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes // NX2NAVI_WB_SPLIT: the write-backs in a part of their own, after one that signals the executed timeline diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index fa20d2c8..a768a7a9 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -125,9 +125,6 @@ typedef struct { int nwritten, cwritten; } CbRes; -static CbRes *cur; -static uint64_t cur_fence; - // Descriptor pools and full-size transient chunks go back to these lists when their command buffer completes, instead // of being destroyed and created again for the next one: each creation is a driver memory allocation (on Windows an // allocation, a map and a paging-fence wait on the CPU), about ten of them per frame inside the replay. @@ -138,8 +135,8 @@ static VkDescriptorPool spare_pool[SPARE_POOLS]; static Chunk spare_chunk[SPARE_CHUNKS]; static int nspare_pool, nspare_chunk; -// State of the command buffer being recorded that the Vulkan command buffer itself holds. -static struct { +// State of the command buffer being recorded that the Vulkan command buffer itself holds (DrawCtx.B). +typedef struct { VkPipeline pipe[2]; // bound per bind point: 0 graphics, 1 compute VkPipeState st; // the state vk_pipeline_set_dynamic last recorded int st_valid; @@ -167,14 +164,14 @@ static struct { VkDescriptorSetLayout ds_last_layout[2]; int ds_pushed[2]; // set number holds the push of DS.psig under ds_layout int res_valid; // the graphics sets bound hold what the inputs in RS give (bind_resources) -} B; +} CbState; -// Contents of B.ds_last: the set's packed descriptors (VkDescSlot), which with its layout determine every write. A draw -// whose allocated set would hold the same reuses the set instead of allocating and writing one (a per-draw +// Contents of B.ds_last (DrawCtx.DS): the set's packed descriptors (VkDescSlot), which with its layout determine every +// write. A draw whose allocated set would hold the same reuses the set instead of allocating and writing one (a per-draw // cost). psig: the same for the last push per set number; a push equal to it under the same layout is not recorded // again (the descriptors stay bound across pipeline binds of that layout). #define DS_SIG_BYTES (MAX_DESC * sizeof(VkDescSlot)) -static struct { uint32_t n[2], pn[2]; uint8_t sig[2][DS_SIG_BYTES], psig[2][DS_SIG_BYTES]; } DS; +typedef struct { uint32_t n[2], pn[2]; uint8_t sig[2][DS_SIG_BYTES], psig[2][DS_SIG_BYTES]; } DsSig; // NX2NAVI_BIND_STATS counters (bind_stats). static struct { @@ -182,13 +179,12 @@ static struct { uint64_t draws, bind[2], desc[2], push, push_skip, set_new, set_reuse, ac_miss, same_prog, same_all, bind_skip; } BS = {-1}; -// The inputs of the last graphics bind_resources (bind_sig) and what its build counted for the traffic census, which a -// skipped build counts again. RS_BYTES bounds the bound-slot bytes a pipeline reads (RsList): per binding at most one -// c0 storage table, otherwise one bound range or handle per descriptor. +// The inputs of the last graphics bind_resources (bind_sig, DrawCtx.RS) and what its build counted for the traffic +// census, which a skipped build counts again. RS_BYTES bounds the bound-slot bytes a pipeline reads (RsList): per +// binding at most one c0 storage table, otherwise one bound range or handle per descriptor. #define RS_BYTES (MAX_BINDINGS * sizeof(((VkExec *)0)->ssbo[0]) + MAX_DESC * sizeof(p_bind_buf)) typedef struct { int cls, use; uint64_t bytes; } RsCount; -static struct { - int on; // NX2NAVI_BIND_SKIP: 1 skip (default), 2 check, 0 off +typedef struct { uint32_t hn; // words of h the last signature filled; 0: none uint64_t h[24]; // generations, feedback images, support inputs, base instance const VkPipeInfo *info; // the pipeline whose RsList ranges body holds, as of bind op generation gen @@ -201,7 +197,8 @@ static struct { int cmds; // check mode: pushes, binds and set writes the build recorded uint32_t ckn; // check mode: the last build's packed descriptors (bytes in ck) uint8_t ck[DS_SIG_BYTES]; -} RS = {-1}; +} RsState; +static int rs_on = -1; // NX2NAVI_BIND_SKIP: 1 skip (default), 2 check, 0 off // The bound-slot bytes of VkExec a pipeline's descriptors read, as ranges merged where they touch, built once per // pipeline info: c0's storage table, the uniform and storage ranges of the bound slots and the texture and image @@ -213,11 +210,77 @@ typedef struct { uint32_t off, len; } RsRange; typedef struct { int st; const GfxShaderRes *res; } RsPtr; typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; RsRange r[]; } RsList; static nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown -static const VkPipeInfo *rs_last; // the last lookup -static const RsList *rs_last_l; -static uint32_t c0_dim[3], c0_binst; // the dispatch size and base instance of the draw or dispatch being bound -static uint64_t bind_gen = 1; // bumped by every bind op that writes ex->prog, ubo, ssbo, tex or img +// Types of the context's caches and memos; each is described where it is used. +#define STAGEM_N 512 // find_stage +#define SLOTM_N 4096 // slot_mask_of +#define PB_N 8 // per-binding descriptor reuse (pb_take) +#define PB_DESC 192 +#define PB_G 17 +typedef struct { + uint64_t tex[32], img[32], ttok[32], itok[32]; + uint32_t tgen[32], igen[32]; + uint8_t tt[32], it[32], ifmt[32]; +} AcSlots; +typedef struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } StageMemo; +typedef struct { uint64_t serial; int st; uint32_t m; } SlotMask; +typedef struct { int cls, use, has; uint64_t n; } PbCount; +typedef struct { + const VkPipeInfo *info; + uint64_t g[PB_G]; + uint8_t ok[PB_DESC]; // descriptor j holds a result that may be reused + uint64_t in[PB_DESC][4]; // the values it was resolved from (pb_key) + VkDescSlot d[PB_DESC]; + PbCount cnt[PB_DESC]; // what its resolution counted for the census +} PbEntry; +typedef struct { + const prog_t *prog; + uint64_t serial; + uint32_t gen, tex_serial; + uint64_t tex[2][32], img[2][32]; + VkPipeState pipe; +} MemoKey; + +// ---- the recording context +// What the draws, binds and dispatches keep from one call to the next while they record a command buffer: the state +// the command buffer holds (B), the signatures that let a draw skip its descriptor build or push (DS, RS, PB), the +// previous draw's shaders and pipeline (SM, PM), the shader lookup's caches (AC, stagem, slotm) and the submission's +// transient resources (res). A thread reaches the one it records with through dc; every thread starts with dc_main, +// the replay's, which is the only one while the replay records on a single thread. Shared tables (image views, +// shaders, descriptor lists, the texture pool sets, the stand-ins) stay file statics. +typedef struct DrawCtx { + CbRes *res; // transient resources of the submission being recorded (cb_res) + uint64_t res_fence; // the submission they belong to + CbState B; + DsSig DS; + RsState RS; + const VkPipeInfo *rs_last; // rs_list's last lookup + const RsList *rs_last_l; + uint32_t c0_dim[3], c0_binst; // the dispatch size and base instance of the draw or dispatch being bound + uint64_t bind_gen; // bumped by every bind op that writes ex->prog, ubo, ssbo, tex or img + AcSlots AC[8]; // answers_for: texture and image types per stage + StageMemo stagem[STAGEM_N]; // find_stage: the shader serving each NVN stage, per program serial + SlotMask slotm[SLOTM_N]; // slot_mask_of: the slots a (program, stage) translation reads + int sf_mvprev; // shader_for asks for the vertex shader's previous-frame variant (motion) + int mfb_stage; // the mesh fallback binds this NVN stage's resources (nvn_stage_of) + const GfxShader *mfb_cur_vs; // the mesh fallback's vertex stage (prepare_draw_ex) + PbCount *pb_cap; // per-binding reuse: where the resolving descriptor's census count goes + int pb_spoiled; // ... and it may not be kept + PbEntry PB[PB_N]; + PbEntry *pb_e; // the build running, NULL when it keeps nothing + uint64_t pb_key_cur[4]; // pb_take to pb_keep: the descriptor's values + int pb_key_ok; // ... and whether it may be kept at all + PbCount pb_cnt_cur; + int mo_ovr_on; // per-object motion: mo_ovr replaces uniform bindings (ubo_info) + VkDescriptorBufferInfo mo_ovr[2][32]; // [vertex, fragment][slot]; buffer 0: not replaced + struct { int valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM; // the shader memo (shader_memo_on) + struct { int valid; VkPipeState s; const VkPipeInfo *info; } PM; // the pipeline memo (pipe_memo_on) + VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views + VkDescSlot bind_d[MAX_DESC]; + VkBufferView bind_bv[MAX_DESC]; +} DrawCtx; +static DrawCtx dc_main = {.bind_gen = 1}; +static _Thread_local DrawCtx *dc = &dc_main; static void cb_done(void *arg) { // completion thread CbRes *r = arg; @@ -243,11 +306,11 @@ static void cb_done(void *arg) { // completion thread free(r->written), free(r->pools), free(r->chunks), free(r); } -void vk_exec_draw_new_cb(void) { memset(&B, 0, sizeof B); } +void vk_exec_draw_new_cb(void) { memset(&dc->B, 0, sizeof dc->B); } void vk_exec_draw_shaders(const GfxShader **vs, const GfxShader **fs) { - *vs = B.st_valid ? B.st.vs : NULL; - *fs = B.st_valid ? B.st.fs : NULL; + *vs = dc->B.st_valid ? dc->B.st.vs : NULL; + *fs = dc->B.st_valid ? dc->B.st.fs : NULL; } // Marks a dispatch about to be recorded (vk.h vk_pass_mark_begin); the caller ends it after the dispatch. @@ -261,13 +324,14 @@ static VkPassMark *dispatch_mark(VkExec *ex, const GfxShader *cs, uint32_t x, ui } static CbRes *cb_res(VkExec *ex) { - if (cur && cur_fence == ex->fence) return cur; - cur = calloc(1, sizeof *cur); - if (!cur) rt_fatal("vk: out of memory"); - cur_fence = ex->fence; - memset(&B, 0, sizeof B); - vk_completion_add(ex->fence, cb_done, cur); - return cur; + DrawCtx *d = dc; + if (d->res && d->res_fence == ex->fence) return d->res; + CbRes *r = calloc(1, sizeof *r); + if (!r) rt_fatal("vk: out of memory"); + d->res = r, d->res_fence = ex->fence; + memset(&d->B, 0, sizeof d->B); + vk_completion_add(ex->fence, cb_done, r); + return r; } static uint8_t *scratch(VkExec *ex, uint64_t size, VkBuffer *buf, VkDeviceSize *off) { @@ -469,7 +533,7 @@ void vk_exec_draw_shutdown(void) { } memset(SD, 0, sizeof SD); nvn_hm_clear(&rs_lists, free); - rs_last = NULL, rs_last_l = NULL; + dc_main.rs_last = NULL, dc_main.rs_last_l = NULL; // the replay's context: no other records now for (int i = 0; i < nspare_pool; i++) vkDestroyDescriptorPool(c->device, spare_pool[i], NULL); for (int i = 0; i < nspare_chunk; i++) vmaDestroyBuffer(c->vma, spare_chunk[i].buf, spare_chunk[i].alloc); nspare_pool = nspare_chunk = 0; @@ -829,13 +893,10 @@ static int find_stage_scan(const prog_t *p, int nvn_stage, const uint8_t **ctl, // Which shader of a program serves each NVN stage, and its code length, per program serial (new at every // nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs -// several times a draw. Direct-mapped; a collision only scans again. -#define STAGEM_N 512 -static struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } stagem[STAGEM_N]; - +// several times a draw. Direct-mapped (DrawCtx.stagem); a collision only scans again. static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { if (!p->serial || (unsigned)nvn_stage >= 6) return find_stage_scan(p, nvn_stage, ctl, code, len); - typeof(stagem[0]) *m = &stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55]; + StageMemo *m = &dc->stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55]; if (m->serial != p->serial) { m->serial = p->serial; memset(m->idx, -1, sizeof m->idx); @@ -855,8 +916,7 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const // Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it // was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve -// otherwise (slot_still). -static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32], ifmt[32]; } AC[8]; +// otherwise (slot_still). DrawCtx.AC, per stage. // vk_texture_slot_token of the slot a texture or image handle names; an empty handle always resolves to nothing. static uint64_t token_of(uint64_t h) { return h ? vk_texture_slot_token((int)(h & 0xfffff)) : 1; } @@ -875,20 +935,17 @@ static int shader_memo_on(void); // The slots of (program, stage) whose types its translation reads (gfx_shader_slot_mask), learned from its first // shader; ~0u while unknown. The types of other slots select the same shader, so answers_for leaves them zero instead -// of looking up all 64 handles. Direct-mapped by serial and stage; a collision only relearns. -#define SLOTM_N 4096 -static struct { uint64_t serial; int st; uint32_t m; } slotm[SLOTM_N]; - +// of looking up all 64 handles. Direct-mapped by serial and stage (DrawCtx.slotm); a collision only relearns. static uint32_t slotm_idx(uint64_t serial, int st) { return (uint32_t)((serial * 8 + (uint64_t)st) * 0x9E3779B97F4A7C15ull >> 52); } static uint32_t slot_mask_of(uint64_t serial, int st) { - const typeof(slotm[0]) *e = &slotm[slotm_idx(serial, st)]; + const SlotMask *e = &dc->slotm[slotm_idx(serial, st)]; return serial && e->serial == serial && e->st == st ? e->m : ~0u; } static void slot_mask_learn(uint64_t serial, int st, const GfxShader *s) { if (!serial || slot_mask_of(serial, st) != ~0u) return; - slotm[slotm_idx(serial, st)] = (typeof(slotm[0])){serial, st, gfx_shader_slot_mask(s)}; + dc->slotm[slotm_idx(serial, st)] = (SlotMask){serial, st, gfx_shader_slot_mask(s)}; } static uint8_t type_of_handle(uint64_t h) { // texture lock held @@ -925,7 +982,7 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G memcpy(&l, ctl_local(ctl) + 4 * k, 4); a->local[k] = (uint16_t)(l ? l : 1); } - typeof(AC[0]) *c = &AC[st]; + AcSlots *c = &dc->AC[st]; // The generation and tokens are read before the lookups (one that creates an entry changes them) and unlocked: a // registration racing with them is ordered after this draw, and a slot stored under older ones is only looked up // again. @@ -953,7 +1010,7 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G // stages, vk_pipeline_async_on): a shader not translated yet is queued and NULL returned with *pending = 1. await_ms: // a shader queued earlier is waited for instead, that long at most (gfx_shader_await; UINT32_MAX: until it is done), // and *pending = 1 says the wait ended first. -static int sf_mvprev; // shader_for asks for the vertex shader's last-frame variant (per-object motion) +// DrawCtx.sf_mvprev: shader_for asks for the vertex shader's last-frame variant (per-object motion). // Probes only (mesh_fb_probe.c): the shader of the program stage whose control section is ctl, in place of a // translation. The probes build their programs from modules made here, which no translator has an input for. @@ -974,7 +1031,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin GfxShaderAnswers a; uint64_t serial = ex->prog->serial; answers_for(ex, st, ctl, slot_mask_of(serial, st), &a); - a.mvprev = (uint8_t)(sf_mvprev && st == NVN_STAGE_VS); + a.mvprev = (uint8_t)(dc->sf_mvprev && st == NVN_STAGE_VS); ShaderEnt *set = NULL; if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded @@ -1011,11 +1068,11 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin // ---- descriptors // The mesh fallback (below, "mesh fallback") runs a program's task and mesh stages as compute modules: while it binds -// one's resources, a compute module's bindings are those of this NVN stage (6 task, 7 mesh), not of the compute stage. -static int mfb_stage; +// one's resources, a compute module's bindings are those of this NVN stage (6 task, 7 mesh), not of the compute stage +// (DrawCtx.mfb_stage). static int nvn_stage_of(int gfx_stage) { - if (gfx_stage == GFX_STAGE_CS && mfb_stage) return mfb_stage; + if (gfx_stage == GFX_STAGE_CS && dc->mfb_stage) return dc->mfb_stage; return gfx_stage == GFX_STAGE_VS ? NVN_STAGE_VS : gfx_stage == GFX_STAGE_FS ? NVN_STAGE_FS : gfx_stage == GFX_STAGE_TS ? NVN_STAGE_TASK : gfx_stage == GFX_STAGE_MS ? NVN_STAGE_MESH : NVN_STAGE_CS; } @@ -1026,26 +1083,23 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) { // Per-binding descriptor reuse (bind_resources, PB): while a descriptor that may be kept resolves, pb_cap is where its // census count goes, so a later reuse counts the same bytes again; pb_spoiled is set when the resolution copied guest -// bytes into transient memory or counted more than once, and such a descriptor is not kept. -typedef struct { int cls, use, has; uint64_t n; } PbCount; -static PbCount *pb_cap; -static int pb_spoiled; +// bytes into transient memory or counted more than once, and such a descriptor is not kept (both in DrawCtx). // vk_mem_count, also kept per (class, use) while a graphics build runs (RS). static void count_use(int cls, int use, uint64_t n) { vk_mem_count(cls, use, n); - if (pb_cap) { - if (pb_cap->has) pb_spoiled = 1; - else *pb_cap = (PbCount){cls, use, 1, n}; + if (dc->pb_cap) { + if (dc->pb_cap->has) dc->pb_spoiled = 1; + else *dc->pb_cap = (PbCount){cls, use, 1, n}; } - if (!RS.rec) return; - for (int i = 0; i < RS.ncnt; i++) - if (RS.cnt[i].cls == cls && RS.cnt[i].use == use) { - RS.cnt[i].bytes += n; + if (!dc->RS.rec) return; + for (int i = 0; i < dc->RS.ncnt; i++) + if (dc->RS.cnt[i].cls == cls && dc->RS.cnt[i].use == use) { + dc->RS.cnt[i].bytes += n; return; } - if (RS.ncnt < (int)(sizeof RS.cnt / sizeof RS.cnt[0])) RS.cnt[RS.ncnt++] = (RsCount){cls, use, n}; - else RS.cnt_over = 1; + if (dc->RS.ncnt < (int)(sizeof dc->RS.cnt / sizeof dc->RS.cnt[0])) dc->RS.cnt[dc->RS.ncnt++] = (RsCount){cls, use, n}; + else dc->RS.cnt_over = 1; } // A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when @@ -1069,8 +1123,8 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, if (sp.alloc) return 0; VkBuffer b; VkDeviceSize o; - RS.volatile_ = 1; - pb_spoiled = 1; + dc->RS.volatile_ = 1; + dc->pb_spoiled = 1; memcpy(scratch(ex, n, &b, &o), (const void *)(uintptr_t)addr, n); vk_mem_count(sp.cls, VK_MUSE_CPU_READ, n); vk_mem_count(VK_MCLS_SCRATCH, use, n); @@ -1130,7 +1184,6 @@ static int c0_on(void) { // NX2NAVI_FIX_C0=0: c0 reads zeros instead of the dr // c0 from replay scratch; a bank equal to the stage's previous one reuses its range, so it costs no copy. static void nvn2_cs_tables(const uint8_t *ctl, uint32_t *ubos, uint32_t *ssbos); static uint32_t nvn2_cs_tables_at(const uint8_t *ctl); -static uint32_t cs_push_bytes; // the size of the last PushComputeConstants (OP_PUSH_CC), replay thread static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : 2; @@ -1138,25 +1191,25 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { // graphics: equal inputs give the same bank, so neither the fill nor the 2 KiB compare is needed. // A vertex or fragment bank depends on the storage slots and the base instance alone (the rest of its inputs record // is zero): those two are compared without building and comparing the whole 1.8 KiB record on every draw. - if (k < 2 && B.c0_valid[k] && B.c0_in[k].binst == c0_binst && - !memcmp(B.c0_in[k].ssbo, ex->ssbo[st], sizeof B.c0_in[k].ssbo)) - return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; - typeof(B.c0_in[0]) in; + if (k < 2 && dc->B.c0_valid[k] && dc->B.c0_in[k].binst == dc->c0_binst && + !memcmp(dc->B.c0_in[k].ssbo, ex->ssbo[st], sizeof dc->B.c0_in[k].ssbo)) + return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; + typeof(dc->B.c0_in[0]) in; memset(&in, 0, sizeof in); memcpy(in.ssbo, ex->ssbo[st], sizeof in.ssbo); - if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, c0_dim, sizeof in.dim); - else in.binst = c0_binst; + if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, dc->c0_dim, sizeof in.dim); + else in.binst = dc->c0_binst; // NVN2 compute: the bank holds the push constants from byte 0x10 (then the program's handle table, which the // host does not read: textures and images are bound by slot). const uint8_t *pctl, *pcode; uint32_t plen; int nvn2_push = st == NVN_STAGE_CS && ex->prog && find_stage(ex->prog, st, &pctl, &pcode, &plen) && ctl_nvn2(pctl); if (nvn2_push) memcpy(in.push, ex->cs_push, sizeof in.push); - if (B.c0_valid[k] && !memcmp(&in, &B.c0_in[k], sizeof in)) - return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; - B.c0_in[k] = in; + if (dc->B.c0_valid[k] && !memcmp(&in, &dc->B.c0_in[k], sizeof in)) + return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; + dc->B.c0_in[k] = in; uint8_t tmp[VK_C0_BYTES]; - vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], c0_dim, c0_binst); + vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], dc->c0_dim, dc->c0_binst); if (nvn2_push) { // the pushed constants from 0x10, the driver's tables after them (push_ptr, nvn2_cs_tables_at) uint32_t nu, ns, ta = nvn2_cs_tables_at(pctl); @@ -1166,7 +1219,7 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { // Bytes 0-0xb of an NVN2 compute bank are the dispatch's group counts (x, y, z). The fog pass after the // integration (passes 0115/0116 of a Kokiri frame) maps each thread to tid / (groups * 16 - 1): with zeros // there every thread read the same point. An indirect dispatch leaves them 0 (op_dispatch_indirect). - memcpy(tmp, c0_dim, 12); + memcpy(tmp, dc->c0_dim, 12); for (uint32_t k = 0; k < nu + ns && ta + 16 * k + 12 <= VK_C0_BYTES; k++) { const p_bind_buf *sb = k < nu ? (k < 18 ? &ex->ubo[st][k] : NULL) : (k - nu < 16 ? &ex->ssbo[st][k - nu] : NULL); if (!sb) continue; @@ -1174,12 +1227,12 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { memcpy(tmp + ta + 16 * k, &sb->addr, 8), memcpy(tmp + ta + 16 * k + 8, &sz, 4); } } - if (!B.c0_valid[k] || memcmp(tmp, B.c0[k], VK_C0_BYTES)) { - memcpy(scratch(ex, VK_C0_BYTES, &B.c0_buf[k], &B.c0_off[k]), tmp, VK_C0_BYTES); - memcpy(B.c0[k], tmp, VK_C0_BYTES); - B.c0_valid[k] = 1; + if (!dc->B.c0_valid[k] || memcmp(tmp, dc->B.c0[k], VK_C0_BYTES)) { + memcpy(scratch(ex, VK_C0_BYTES, &dc->B.c0_buf[k], &dc->B.c0_off[k]), tmp, VK_C0_BYTES); + memcpy(dc->B.c0[k], tmp, VK_C0_BYTES); + dc->B.c0_valid[k] = 1; } - return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; + return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; } // The buffer range limits, read once: the device's properties are a cold cache line on every binding of every draw. @@ -1193,13 +1246,13 @@ static void limits_init(void) { } // Per-object motion (section at the end): while it records a draw again, these replace the uniform bindings of the -// vertex stage's slots 18-20 (last frame's camera, bones and place) and the fragment stage's slot 3 (its parameters). -static int mo_ovr_on; -static VkDescriptorBufferInfo mo_ovr[2][32]; // [vertex, fragment][slot]; buffer 0: not replaced +// vertex stage's slots 18-20 (last frame's camera, bones and place) and the fragment stage's slot 3 (its parameters) +// (DrawCtx.mo_ovr). static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { - if (mo_ovr_on && (st == NVN_STAGE_VS || st == NVN_STAGE_FS) && slot >= 0 && slot < 32 && mo_ovr[st][slot].buffer) - return mo_ovr[st][slot]; + if (dc->mo_ovr_on && (st == NVN_STAGE_VS || st == NVN_STAGE_FS) && slot >= 0 && slot < 32 && + dc->mo_ovr[st][slot].buffer) + return dc->mo_ovr[st][slot]; if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); uint64_t max = LIM.ubo_max, align = LIM.ubo_align; VkDescriptorBufferInfo bi; @@ -2955,12 +3008,12 @@ static int push_skip_on(void) { // NX2NAVI_PUSH_SKIP=0: every graphics draw pu // NX2NAVI_BIND_SKIP=0 builds every draw; =check builds every draw and stops when a draw with equal inputs gets other // writes, census counts or descriptor commands. static int bind_skip_on(void) { - if (RS.on < 0) { + if (rs_on < 0) { const char *e = cfg_value("NX2NAVI_BIND_SKIP"); - RS.on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; - if (RS.on != 1) rt_log("vk_exec: descriptor build skip %s", RS.on ? "check" : "off"); + rs_on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; + if (rs_on != 1) rt_log("vk_exec: descriptor build skip %s", rs_on ? "check" : "off"); } - return RS.on; + return rs_on; } static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, size_t len) { @@ -2970,7 +3023,7 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si } static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { - if (info == rs_last) return rs_last_l; + if (info == dc->rs_last) return dc->rs_last_l; if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info); RsList *l = nvn_hm_get(&rs_lists, &info); if (!l) { @@ -3020,7 +3073,7 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { memcpy(l->r, tmp, sizeof *tmp * n); nvn_hm_put(&rs_lists, &info, l); } - rs_last = info, rs_last_l = l; + dc->rs_last = info, dc->rs_last_l = l; return l; } @@ -3029,8 +3082,8 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { // Check mode compares them anyway and stops when the bind op generation said they could not differ but they do. static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { const RsList *l = rs_list(ex, info); - if (!l->ok) return RS.hn = 0, RS.info = NULL, 0; - uint64_t *h = RS.h, diff = 0; + if (!l->ok) return dc->RS.hn = 0, dc->RS.info = NULL, 0; + uint64_t *h = dc->RS.h, diff = 0; uint32_t n = 0; #define PUT(v) do { uint64_t v_ = (uint64_t)(v); diff |= h[n] ^ v_; h[n++] = v_; } while (0) PUT((uintptr_t)info); @@ -3049,16 +3102,16 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { PUT(a); PUT(ps); } - if (l->c0) PUT(c0_binst); + if (l->c0) PUT(dc->c0_binst); for (int i = 0; i < l->np; i++) PUT(l->p[i].res->ptr ? ptr_addr(ex, l->p[i].st, l->p[i].res) : ubo_ptr(ex, l->p[i].st, l->p[i].res)); #undef PUT - diff |= n != RS.hn; - RS.hn = n; - int fast = RS.info == info && RS.gen == bind_gen; - if (fast && RS.on != 2) return !diff; + diff |= n != dc->RS.hn; + dc->RS.hn = n; + int fast = dc->RS.info == info && dc->RS.gen == dc->bind_gen; + if (fast && rs_on != 2) return !diff; const uint8_t *e = (const uint8_t *)ex; - uint8_t *d = RS.body; + uint8_t *d = dc->RS.body; int bd = 0; for (uint32_t i = 0; i < l->n; i++) { const RsRange *r = &l->r[i]; @@ -3067,7 +3120,7 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { d += r->len; } if (fast && bd) rt_fatal("vk_exec: descriptor build skip check: bound slots changed without a bind op"); - RS.info = info, RS.gen = bind_gen; + dc->RS.info = info, dc->RS.gen = dc->bind_gen; return !diff && !bd; } @@ -3091,22 +3144,7 @@ static inline uint64_t bt_now(void) { return rt_trace & RT_TRACE_GFX ? plat_now_ // resolution that copied guest bytes into transient memory is never kept. Off while per-object motion replaces uniform // bindings (mo_ovr_on). NX2NAVI_BIND_REUSE=0 resolves every descriptor; =check resolves every descriptor and stops on // one that resolves differently from its kept result with the same inputs. -#define PB_N 8 -#define PB_DESC 192 -#define PB_G 17 -typedef struct { - const VkPipeInfo *info; - uint64_t g[PB_G]; - uint8_t ok[PB_DESC]; // descriptor j holds a result that may be reused - uint64_t in[PB_DESC][4]; // the values it was resolved from (pb_key) - VkDescSlot d[PB_DESC]; - PbCount cnt[PB_DESC]; // what its resolution counted for the census -} PbEntry; -static PbEntry PB[PB_N]; -static PbEntry *pb_e; // the build running, NULL when it keeps nothing -static uint64_t pb_key_cur[4]; // pb_take to pb_keep: the descriptor's values, and whether it may be kept at all -static int pb_key_ok; -static PbCount pb_cnt_cur; +// The entries (PbEntry, DrawCtx.PB) and the build running (DrawCtx.pb_e) belong to the recording context. static int pb_mode = -1; static int pb_on(void) { @@ -3136,7 +3174,7 @@ static void pb_globals(const VkExec *ex, uint64_t g[PB_G]) { // info's entry, emptied when it held another pipeline's results or the globals moved. static PbEntry *pb_entry(const VkPipeInfo *info, const uint64_t g[PB_G]) { uintptr_t p = (uintptr_t)info; - PbEntry *e = &PB[(p >> 6 ^ p >> 13) % PB_N]; + PbEntry *e = &dc->PB[(p >> 6 ^ p >> 13) % PB_N]; if (e->info != info || memcmp(e->g, g, sizeof e->g)) { e->info = info; memcpy(e->g, g, sizeof e->g); @@ -3177,38 +3215,39 @@ static int pb_key(const VkExec *ex, const VkPipeBinding *b, uint32_t k, uint64_t // Before descriptor j (descriptor k of binding b) resolves: 1 when its kept result serves, *o filled and its census // counted again. 0: resolve it, then pb_keep. static int pb_take(const VkExec *ex, const VkPipeBinding *b, uint32_t j, uint32_t k, VkDescSlot *o) { - pb_key_ok = 0; - if (!pb_e || j >= PB_DESC || !pb_key(ex, b, k, pb_key_cur)) return 0; - pb_key_ok = 1; - if (pb_mode != 2 && pb_e->ok[j] && !memcmp(pb_e->in[j], pb_key_cur, sizeof pb_key_cur)) { - const PbCount *c = &pb_e->cnt[j]; - *o = pb_e->d[j]; + dc->pb_key_ok = 0; + if (!dc->pb_e || j >= PB_DESC || !pb_key(ex, b, k, dc->pb_key_cur)) return 0; + dc->pb_key_ok = 1; + if (pb_mode != 2 && dc->pb_e->ok[j] && !memcmp(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur)) { + const PbCount *c = &dc->pb_e->cnt[j]; + *o = dc->pb_e->d[j]; if (c->has) count_use(c->cls, c->use, c->n); vk_exec_frame.bind_reused++; return 1; } - pb_cnt_cur = (PbCount){0}; - pb_cap = &pb_cnt_cur, pb_spoiled = 0; + dc->pb_cnt_cur = (PbCount){0}; + dc->pb_cap = &dc->pb_cnt_cur, dc->pb_spoiled = 0; return 0; } // After descriptor j resolved to *o. static void pb_keep(uint32_t j, const VkDescSlot *o) { - pb_cap = NULL; - if (!pb_e || j >= PB_DESC) return; + dc->pb_cap = NULL; + if (!dc->pb_e || j >= PB_DESC) return; vk_exec_frame.bind_resolved++; - if (!pb_key_ok) { - pb_e->ok[j] = 0; + if (!dc->pb_key_ok) { + dc->pb_e->ok[j] = 0; return; } - if (pb_mode == 2 && pb_e->ok[j] && !pb_spoiled && !memcmp(pb_e->in[j], pb_key_cur, sizeof pb_key_cur) && - memcmp(&pb_e->d[j], o, sizeof *o)) + if (pb_mode == 2 && dc->pb_e->ok[j] && !dc->pb_spoiled && + !memcmp(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur) && + memcmp(&dc->pb_e->d[j], o, sizeof *o)) rt_fatal("vk_exec: per-binding descriptor reuse check: descriptor %u resolves differently from its kept result " "with the same inputs", j); - pb_e->ok[j] = (uint8_t)!pb_spoiled; - memcpy(pb_e->in[j], pb_key_cur, sizeof pb_key_cur); - pb_e->d[j] = *o; - pb_e->cnt[j] = pb_cnt_cur; + dc->pb_e->ok[j] = (uint8_t)!dc->pb_spoiled; + memcpy(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur); + dc->pb_e->d[j] = *o; + dc->pb_e->cnt[j] = dc->pb_cnt_cur; } // Writes allocated set s from its packed descriptors d: through the layout's update template, or with the write @@ -3231,32 +3270,32 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c // The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update // template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. static void bind_resources(VkExec *ex, const VkPipeInfo *info) { - static VkWriteDescriptorSet w[MAX_BINDINGS]; - static VkDescSlot d[MAX_DESC]; - static VkBufferView bv[MAX_DESC]; + VkWriteDescriptorSet *w = dc->bind_w; + VkDescSlot *d = dc->bind_d; + VkBufferView *bv = dc->bind_bv; if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS); bind_stats(info); int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS; - int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = B.res_valid; + int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = dc->B.res_valid; if (skip) { uint64_t sig_t0 = bt_now(); same = bind_sig(ex, info); vk_exec_frame.bind_sig_ns += (uint64_t)(bt_now() - sig_t0); if (same && was_valid && skip == 1) { - for (int i = 0; i < RS.ncnt; i++) vk_mem_count(RS.cnt[i].cls, RS.cnt[i].use, RS.cnt[i].bytes); + for (int i = 0; i < dc->RS.ncnt; i++) vk_mem_count(dc->RS.cnt[i].cls, dc->RS.cnt[i].use, dc->RS.cnt[i].bytes); BS.bind_skip++; ex->recorded++; return; } - B.res_valid = 0; // until the build below ends - if (skip == 2) memcpy(RS.pcnt, RS.cnt, sizeof RS.cnt), RS.pncnt = RS.ncnt; - RS.rec = 1, RS.ncnt = 0, RS.cnt_over = 0, RS.volatile_ = 0, RS.cmds = 0; + dc->B.res_valid = 0; // until the build below ends + if (skip == 2) memcpy(dc->RS.pcnt, dc->RS.cnt, sizeof dc->RS.cnt), dc->RS.pncnt = dc->RS.ncnt; + dc->RS.rec = 1, dc->RS.ncnt = 0, dc->RS.cnt_over = 0, dc->RS.volatile_ = 0, dc->RS.cmds = 0; } - pb_e = NULL; - if (gfx && !mo_ovr_on && pb_on()) { + dc->pb_e = NULL; + if (gfx && !dc->mo_ovr_on && pb_on()) { uint64_t g[PB_G]; pb_globals(ex, g); - pb_e = pb_entry(info, g); + dc->pb_e = pb_entry(info, g); } uint32_t n_sets = info->n_sets ? info->n_sets : 1; uint32_t lo[2] = {0, 0}, hi[2] = {0, 0}, d0[2] = {0, 0}; // bindings [lo, hi) and first descriptor per set @@ -3279,19 +3318,20 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { if (b->kind == VK_PIPE_SUPPORT) { float in[3]; support_in(ex, in); - if (!B.sup_valid || memcmp(in, B.sup_in, sizeof in)) { // support_fill reads only these + if (!dc->B.sup_valid || memcmp(in, dc->B.sup_in, sizeof in)) { // support_fill reads only these uint8_t tmp[SUPPORT_BYTES]; support_fill(tmp, in[0], in[1], in[2]); - memcpy(B.sup_in, in, sizeof in); - if (!B.sup_valid || memcmp(tmp, B.support, SUPPORT_BYTES)) { // a new buffer only when the contents changed + memcpy(dc->B.sup_in, in, sizeof in); + // a new buffer only when the contents changed + if (!dc->B.sup_valid || memcmp(tmp, dc->B.support, SUPPORT_BYTES)) { VkBuffer sb; VkDeviceSize so; memcpy(scratch(ex, SUPPORT_BYTES, &sb, &so), tmp, SUPPORT_BYTES); - memcpy(B.support, tmp, SUPPORT_BYTES); - B.sup_buf = sb, B.sup_off = so, B.sup_valid = 1; + memcpy(dc->B.support, tmp, SUPPORT_BYTES); + dc->B.sup_buf = sb, dc->B.sup_off = so, dc->B.sup_valid = 1; } } - o->buffer = (VkDescriptorBufferInfo){B.sup_buf, B.sup_off, SUPPORT_BYTES}; + o->buffer = (VkDescriptorBufferInfo){dc->B.sup_buf, dc->B.sup_off, SUPPORT_BYTES}; } else if (b->kind == GFX_RES_UBO) { uint64_t bt = bt_now(); for (uint32_t k = 0; k < cnt; k++) { @@ -3347,7 +3387,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } nd += cnt; } - pb_e = NULL; + dc->pb_e = NULL; uint64_t push_t0 = bt_now(); // An empty set 1 (a fragment stage without resources: depth and shadow draws) starts past the last binding, which // the loop never reaches: d0[1] stayed 0 and set 0 was pushed with 0 bytes, so the driver read stale descriptors @@ -3358,10 +3398,10 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { // layout lacks (false VUID-vkCmdDrawIndexed-None-08114, then an access violation inside the layer). A bind at set 1 // disturbs set 0 by its canonical id, which drops the layer's push set: the push after it gets a new one. int set1_first = 0; - if (gfx && info->layout != B.ds_layout) { // another layout: sets bound before may be disturbed - B.ds_layout = info->layout; - B.ds_bound[0] = B.ds_bound[1] = VK_NULL_HANDLE; - B.ds_pushed[0] = B.ds_pushed[1] = 0; + if (gfx && info->layout != dc->B.ds_layout) { // another layout: sets bound before may be disturbed + dc->B.ds_layout = info->layout; + dc->B.ds_bound[0] = dc->B.ds_bound[1] = VK_NULL_HANDLE; + dc->B.ds_pushed[0] = dc->B.ds_pushed[1] = 0; set1_first = n_sets == 2 && vk_ctx()->validation; } for (uint32_t k = 0; k < n_sets; k++) { @@ -3372,24 +3412,24 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { if (set1_first) { // bound only to drop the layer's push set (comment above) VkDescriptorSet set = alloc_set(ex, info->set_layout1); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 1, 1, &set, 0, NULL); - B.ds_bound[1] = set, B.ds_pushed[1] = 0; - RS.cmds++; + dc->B.ds_bound[1] = set, dc->B.ds_pushed[1] = 0; + dc->RS.cmds++; } continue; } if (s ? info->push1 : info->push) { uint32_t n = gfx && push_skip_on() ? dbytes : 0; - if (n && B.ds_pushed[s] && DS.pn[s] == n && !memcmp(DS.psig[s], ds, n)) { + if (n && dc->B.ds_pushed[s] && dc->DS.pn[s] == n && !memcmp(dc->DS.psig[s], ds, n)) { BS.push_skip++; continue; } if (info->tpl[s]) rec_push_tpl(vk_pipeline_push_tpl_fn(), ex->cb, info->tpl[s], info->layout, s, ds, dbytes); else rec_push(vk_pipeline_push_fn(), ex->cb, info->bind_point, info->layout, s, i1 - i0, w + i0); BS.push++; - RS.cmds++; + dc->RS.cmds++; if (gfx) { - B.ds_bound[s] = VK_NULL_HANDLE, B.ds_pushed[s] = n != 0; - if (n) DS.pn[s] = n, memcpy(DS.psig[s], ds, n); + dc->B.ds_bound[s] = VK_NULL_HANDLE, dc->B.ds_pushed[s] = n != 0; + if (n) dc->DS.pn[s] = n, memcpy(dc->DS.psig[s], ds, n); } } else if (!gfx || s > 1) { VkDescriptorSet set = alloc_set(ex, s ? info->set_layout1 : info->set_layout); @@ -3397,49 +3437,50 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL); } else { VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout; - VkDescriptorSet set = B.ds_last[s]; - if (!set || B.ds_last_layout[s] != sl || DS.n[s] != dbytes || memcmp(DS.sig[s], ds, dbytes)) { + VkDescriptorSet set = dc->B.ds_last[s]; + if (!set || dc->B.ds_last_layout[s] != sl || dc->DS.n[s] != dbytes || memcmp(dc->DS.sig[s], ds, dbytes)) { set = alloc_set(ex, sl); write_set(info, s, set, ds, w, i0, i1); - B.ds_last[s] = set, B.ds_last_layout[s] = sl, DS.n[s] = dbytes; - memcpy(DS.sig[s], ds, dbytes); + dc->B.ds_last[s] = set, dc->B.ds_last_layout[s] = sl, dc->DS.n[s] = dbytes; + memcpy(dc->DS.sig[s], ds, dbytes); BS.set_new++; - RS.cmds++; + dc->RS.cmds++; } else { BS.set_reuse++; } - if (B.ds_bound[s] != set) { + if (dc->B.ds_bound[s] != set) { vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL); - B.ds_bound[s] = set, B.ds_pushed[s] = 0; - RS.cmds++; + dc->B.ds_bound[s] = set, dc->B.ds_pushed[s] = 0; + dc->RS.cmds++; } } } if (info->pool) { // set 2: the texture and sampler pools pool_sync(ex); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 2, 1, &PL.set, 0, NULL); - RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again + dc->RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again } if (skip) { - RS.rec = 0; + dc->RS.rec = 0; // An equal next draw records nothing when every push set holds its push (NX2NAVI_PUSH_SKIP on) and nothing made // the build unrepeatable; allocated sets are always bound to the one written last. - int valid = !RS.volatile_ && !RS.cnt_over && RS.hn; + int valid = !dc->RS.volatile_ && !dc->RS.cnt_over && dc->RS.hn; for (uint32_t s = 0; s < n_sets; s++) - if (hi[s] > lo[s] && (s ? info->push1 : info->push) && !B.ds_pushed[s]) valid = 0; + if (hi[s] > lo[s] && (s ? info->push1 : info->push) && !dc->B.ds_pushed[s]) valid = 0; if (skip == 2) { // with the pipeline info equal (bind_sig), the packed descriptors determine every write uint32_t n = (uint32_t)(nd * sizeof *d); - int wr = n != RS.ckn || memcmp(RS.ck, d, n); - int cn = RS.ncnt != RS.pncnt || memcmp(RS.cnt, RS.pcnt, sizeof RS.cnt[0] * (size_t)RS.ncnt); - if (same && was_valid && (wr || cn || RS.cmds)) + int wr = n != dc->RS.ckn || memcmp(dc->RS.ck, d, n); + int cn = dc->RS.ncnt != dc->RS.pncnt || + memcmp(dc->RS.cnt, dc->RS.pcnt, sizeof dc->RS.cnt[0] * (size_t)dc->RS.ncnt); + if (same && was_valid && (wr || cn || dc->RS.cmds)) rt_fatal("vk_exec: descriptor build skip check: a draw of program %llu with the inputs of the previous one " "gets %s", (unsigned long long)ex->prog->serial, - RS.cmds ? "descriptor commands" : wr ? "other writes" : "other census counts"); + dc->RS.cmds ? "descriptor commands" : wr ? "other writes" : "other census counts"); if (same && was_valid) BS.bind_skip++; // the draws a skip would have spared, compared instead - RS.ckn = n; - memcpy(RS.ck, d, n); + dc->RS.ckn = n; + memcpy(dc->RS.ck, d, n); } - B.res_valid = valid; + dc->B.res_valid = valid; } vk_exec_frame.bind_push_ns += (uint64_t)(bt_now() - push_t0); ex->recorded++; @@ -3449,11 +3490,11 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { static void bind_pipeline(VkExec *ex, VkPipelineBindPoint bp, VkPipeline p) { int k = bp == VK_PIPELINE_BIND_POINT_COMPUTE; - if (B.pipe[k] == p) return; + if (dc->B.pipe[k] == p) return; vkCmdBindPipeline(ex->cb, bp, p); ex->recorded++; - B.pipe[k] = p; - if (!k) B.st_valid = 0; + dc->B.pipe[k] = p; + if (!k) dc->B.st_valid = 0; } static VkViewport draw_viewport(VkExec *ex) { @@ -3511,6 +3552,7 @@ static void record_fixed(VkExec *ex) { } static void bind_vertex_buffers(VkExec *ex) { + CbState *held = &dc->B; VkBuffer vb[17]; VkDeviceSize vo[17], vsz[17], vstr[17]; int vcls[17] = {0}; @@ -3519,20 +3561,21 @@ static void bind_vertex_buffers(VkExec *ex) { // A stream whose range is the one its bound buffer was resolved from, with no pool or mirror change since // (vk_mem_gen, read before any resolve), resolves to that buffer again: most draws change one stream or none. uint64_t gen = vk_mem_gen(); - int same_gen = B.vb_valid && B.vgen == gen; + int same_gen = held->vb_valid && held->vgen == gen; // Most draws change no stream at all: with every range as it was bound and no pool or mirror change, the loop // below would take each buffer from B again and return at the compare after it. - if (same_gen && !memcmp(B.vin, ex->vbuf, sizeof B.vin)) { + if (same_gen && !memcmp(held->vin, ex->vbuf, sizeof held->vin)) { int same = 1; - for (int b = 0; !dyn && b < 16 && same; b++) same = B.vstr[b] == ex->pipe.vstream[b].stride; + for (int b = 0; !dyn && b < 16 && same; b++) same = held->vstr[b] == ex->pipe.vstream[b].stride; if (same) return; } for (int b = 0; b < 17; b++) { vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0; if (b == VK_PIPE_ZERO_BINDING) continue; const NVNbufferRange *r = &ex->vbuf[b]; - if (same_gen && B.vin[b].address == r->address && B.vin[b].size == r->size) { - vb[b] = B.vb[b], vo[b] = B.vo[b], vsz[b] = B.vsz[b], vcls[b] = B.vcls[b], vpool[b] = B.vpool[b]; + if (same_gen && held->vin[b].address == r->address && held->vin[b].size == r->size) { + vb[b] = held->vb[b], vo[b] = held->vo[b], vsz[b] = held->vsz[b]; + vcls[b] = held->vcls[b], vpool[b] = held->vpool[b]; continue; } VkMemSpan sp; @@ -3541,11 +3584,12 @@ static void bind_vertex_buffers(VkExec *ex) { uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail; vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool; } - memcpy(B.vcls, vcls, sizeof vcls), memcpy(B.vpool, vpool, sizeof vpool); - memcpy(B.vin, ex->vbuf, sizeof B.vin); - B.vgen = gen; - if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) && - (dyn || !memcmp(B.vstr, vstr, sizeof vstr))) + memcpy(held->vcls, vcls, sizeof vcls), memcpy(held->vpool, vpool, sizeof vpool); + memcpy(held->vin, ex->vbuf, sizeof held->vin); + held->vgen = gen; + if (held->vb_valid && !memcmp(held->vb, vb, sizeof vb) && !memcmp(held->vo, vo, sizeof vo) && + !memcmp(held->vsz, vsz, sizeof vsz) && + (dyn || !memcmp(held->vstr, vstr, sizeof vstr))) return; vkCmdBindVertexBuffers2(ex->cb, 0, 17, vb, vo, vsz, dyn ? NULL : vstr); ex->recorded++; @@ -3554,8 +3598,9 @@ static void bind_vertex_buffers(VkExec *ex) { if (rt_trace & RT_TRACE_PERF) for (int b = 0; b < 17; b++) if (vpool[b]) vk_mem_bound_note(vpool[b]); - memcpy(B.vb, vb, sizeof vb), memcpy(B.vo, vo, sizeof vo), memcpy(B.vsz, vsz, sizeof vsz), memcpy(B.vstr, vstr, sizeof vstr); - B.vb_valid = 1; + memcpy(held->vb, vb, sizeof vb), memcpy(held->vo, vo, sizeof vo), memcpy(held->vsz, vsz, sizeof vsz); + memcpy(held->vstr, vstr, sizeof vstr); + held->vb_valid = 1; } // NX2NAVI_FOLIAGE_SHADOW_CULL=[f] (0 to 3, default 1; 3 turns it off): the foliage's mesh draws into the sun's shadow cascades @@ -3717,14 +3762,8 @@ static uint64_t *link_budget(void) { // texture type in answers_for, and stops on a difference. // tex_serial (bumped by every texture and image bind) equal to the key's means the handles were not rebound since // they were compared, so the 1 KiB compare is skipped; a rebind to the same handles still matches by the compare. -typedef struct { - const prog_t *prog; - uint64_t serial; - uint32_t gen, tex_serial; - uint64_t tex[2][32], img[2][32]; - VkPipeState pipe; -} MemoKey; -static struct { int on, valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM = {-1}; +// The memo and its key (MemoKey) are DrawCtx.SM. +static int sm_on = -1; static void memo_key(const VkExec *ex, uint32_t gen, MemoKey *k) { k->prog = ex->prog, k->serial = ex->prog->serial, k->gen = gen, k->tex_serial = ex->tex_serial; @@ -3745,18 +3784,18 @@ static int memo_match(const VkExec *ex, uint32_t gen, MemoKey *k) { } static int shader_memo_on(void) { - if (SM.on < 0) { + if (sm_on < 0) { const char *e = cfg_value("NX2NAVI_SHADER_MEMO"); - SM.on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; + sm_on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; } - return SM.on; + return sm_on; } -static struct { int on, valid; VkPipeState s; const VkPipeInfo *info; } PM = {-1}; +static int pm_on = -1; // the memo is DrawCtx.PM static int pipe_memo_on(void) { - if (PM.on < 0) PM.on = cfg_bool("NX2NAVI_PIPE_MEMO"); - return PM.on; + if (pm_on < 0) pm_on = cfg_bool("NX2NAVI_PIPE_MEMO"); + return pm_on; } // Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped. @@ -3896,8 +3935,7 @@ static void prog_lead(const prog_t *p, int mesh) { // mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing // vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. 2: a mesh draw of -// the mesh fallback (mfb_draw): the vertex stage is mfb_cur_vs, in an ordinary pipeline without vertex input. -static const GfxShader *mfb_cur_vs; +// the mesh fallback (mfb_draw): the vertex stage is DrawCtx.mfb_cur_vs, in an ordinary pipeline without vertex input. static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) { int fb = mesh == 2; // from here on mesh says what the draw is, and !fb that its pipeline is a mesh pipeline @@ -3922,8 +3960,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int int has_ts = 0, ts_wait = 0, vst = mesh ? NVN_STAGE_MESH : NVN_STAGE_VS; const GfxShader *vs, *fs, *ts = NULL; uint32_t gen = memo ? vk_texture_slots_gen() : 0; // before shader_for, whose lookups may bump it - if (memo && SM.valid && memo_match(ex, gen, &SM.k)) { - vs = SM.vs, fs = SM.fs, has_vs = 1, has_fs = SM.has_fs; + if (memo && dc->SM.valid && memo_match(ex, gen, &dc->SM.k)) { + vs = dc->SM.vs, fs = dc->SM.fs, has_vs = 1, has_fs = dc->SM.has_fs; if (memo == 2) { int hv, hf, wv, wf; const GfxShader *v2 = shader_for(ex, NVN_STAGE_VS, &hv, &wv, 0), *f2 = shader_for(ex, NVN_STAGE_FS, &hf, &wf, 0); @@ -3931,13 +3969,13 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int rt_fatal("vk_exec: shader memo of program %llu differs from the lookup", (unsigned long long)ex->prog->serial); } } else { - SM.valid = 0; - if (fb) vs = mfb_cur_vs, has_vs = 1; + dc->SM.valid = 0; + if (fb) vs = dc->mfb_cur_vs, has_vs = 1; else vs = shader_for(ex, vst, &has_vs, &vs_wait, 0); if (mesh && !fb) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0); fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, 0); if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs)) - memo_key(ex, gen, &SM.k), SM.vs = vs, SM.fs = fs, SM.has_fs = has_fs, SM.valid = 1; + memo_key(ex, gen, &dc->SM.k), dc->SM.vs = vs, dc->SM.fs = fs, dc->SM.has_fs = has_fs, dc->SM.valid = 1; } if (!has_vs) return draw_skipped(ex, mesh, SK_NO_VERTEX_STAGE); // A draw into a target the previous frame did not render (or into none) waits for a pending shader or pipeline: @@ -4010,8 +4048,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int const char *why = NULL; uint64_t w0 = 0, m0 = 0; // m0: when the lookup missed (wait_note) int kind = WK_LINK; - if (pipe_memo_on() && PM.valid && !memcmp(&PM.s, &s, sizeof s)) { - info = PM.info, pending = 0; + if (pipe_memo_on() && dc->PM.valid && !memcmp(&dc->PM.s, &s, sizeof s)) { + info = dc->PM.info, pending = 0; p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); } else { // Nearly every draw finds its pipeline, so the first lookup waits for nothing and the clock is read only for a @@ -4060,8 +4098,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int pool_charge(last, b0); } if (m0) wait_note(kind, m0, pending), draw_stats_tick(); - PM.valid = !pending && p && info; - if (PM.valid) PM.s = s, PM.info = info; + dc->PM.valid = !pending && p && info; + if (dc->PM.valid) dc->PM.s = s, dc->PM.info = info; } uint64_t waited = w0 ? kern_now_ns() - w0 : 0; static int wait_logs; @@ -4079,7 +4117,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int if (!p || !info) return draw_skipped(ex, mesh, SK_PIPE_FAILED); if (info->pool && !pool_on()) return draw_skipped(ex, mesh, SK_POOL_OFF); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p); - if (!B.st_valid || memcmp(&B.st, &s, sizeof s)) { + if (!dc->B.st_valid || memcmp(&dc->B.st, &s, sizeof s)) { // Only the dynamic states whose values changed (vk_pipeline_set_dynamic_delta); NX2NAVI_DYN_DELTA=0 records // them all after every pipeline bind, as before. static int delta = -1; @@ -4087,11 +4125,11 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int delta = cfg_bool("NX2NAVI_DYN_DELTA"); if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)"); } - if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh && !fb, + if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &dc->B.dyn, mesh && !fb, &vk_exec_frame.dyn_left); else vk_pipeline_set_dynamic(ex->cb, &s); ex->recorded++; - B.st = s, B.st_valid = 1; + dc->B.st = s, dc->B.st_valid = 1; if (ex->mark && vk_exec_frame.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; } if (!mesh) { @@ -4509,7 +4547,7 @@ static void mo_emit(VkCommandBuffer cb) { static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0); static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off) { - if (MO.failed || !vk_mvprev_on() || !B.st_valid) return; + if (MO.failed || !vk_mvprev_on() || !dc->B.st_valid) return; static int timed = -1; if (timed < 0) timed = rt_knob("MVOBJ_TIMES", "0"); // NX2NAVI_MVOBJ_TIMES=1: the summary's times (the clock reads cost) if (!timed) return mvobj_draw_(ex, d, ib, ib_off, 0); @@ -4523,13 +4561,13 @@ static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSiz static void mvobj_banks(VkExec *ex) { static int on = -1; if (on < 0) on = rt_knob("MVOBJ_BANKS", "0"); - if (!on || !B.st_valid) return; + if (!on || !dc->B.st_valid) return; uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (frame < 2999 || frame > 3000) return; static struct { const GfxShader *vs; int bank; uint64_t frame; float w[64]; int n; } tab[4096]; static int logged; const GfxShaderRes *ur; - int nu = gfx_shader_resources(B.st.vs, GFX_RES_UBO, &ur); + int nu = gfx_shader_resources(dc->B.st.vs, GFX_RES_UBO, &ur); for (int i = 0; i < nu; i++) { int bank = ur[i].slot; if (bank < 3 || bank == 4 || bank == 5 || bank == 7 || bank > 16) continue; @@ -4537,16 +4575,17 @@ static void mvobj_banks(VkExec *ex) { uint32_t n = u->size < 256 ? (uint32_t)u->size / 4 : 64; const float *now = n ? (const float *)vk_usnap_bytes(u->addr, n * 4) : NULL; if (!now) continue; - uint32_t k = (uint32_t)(((uintptr_t)B.st.vs ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull >> 52); + uint32_t k = + (uint32_t)(((uintptr_t)dc->B.st.vs ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull >> 52); if (frame == 2999) { - tab[k].vs = B.st.vs, tab[k].bank = bank, tab[k].frame = frame, tab[k].n = (int)n; + tab[k].vs = dc->B.st.vs, tab[k].bank = bank, tab[k].frame = frame, tab[k].n = (int)n; memcpy(tab[k].w, now, n * 4); - } else if (tab[k].vs == B.st.vs && tab[k].bank == bank && tab[k].frame == 2999 && logged < 80) { + } else if (tab[k].vs == dc->B.st.vs && tab[k].bank == bank && tab[k].frame == 2999 && logged < 80) { char line[700]; int o = 0, nd = 0; for (int j = 0; j < tab[k].n && j < (int)n && o < 600; j++) if (memcmp(&tab[k].w[j], &now[j], 4)) nd++, o += snprintf(line + o, sizeof line - o, " [%d] %g>%g", j, tab[k].w[j], now[j]); - if (nd) logged++, rt_log("mvbanks: vs %p c%d size %#llx: %d words%s", (const void *)B.st.vs, bank, + if (nd) logged++, rt_log("mvbanks: vs %p c%d size %#llx: %d words%s", (const void *)dc->B.st.vs, bank, (unsigned long long)u->size, nd, line); tab[k].frame = 0; } @@ -4641,14 +4680,15 @@ static int mo_prev_cached(int index, const p_bind_buf *u, VkDescriptorBufferInfo static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0) { mvobj_banks(ex); const VkExecTarget *z = &ex->depth; - if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || B.st.samples > 1 || B.st.raster_discard) + if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || dc->B.st.samples > 1 || + dc->B.st.raster_discard) return; // solid geometry only, which tests and writes the scene depth: effects, particles and screen quads write none, and // drawn again they would put their motion on the surfaces behind them (DLSS takes those from the surface) - if (!B.st.ds.dt || !B.st.ds.dw) return; + if (!dc->B.st.ds.dt || !dc->B.st.ds.dw) return; // a vertex shader that reads a user bank besides the camera's (c4) may move on its own const GfxShaderRes *ur; - int nu = gfx_shader_resources(B.st.vs, GFX_RES_UBO, &ur), reads = 0; + int nu = gfx_shader_resources(dc->B.st.vs, GFX_RES_UBO, &ur), reads = 0; for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4; if (!reads) return; MO.cand++; @@ -4660,20 +4700,20 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi // the variant: the translation of the same code and answers, kept per shader once found; it names the banks the // position depends on (0: none, the shader is skipped from then on) static struct { const GfxShader *of, *vs; uint32_t banks; } vmemo[1024]; - uint32_t vi = (uint32_t)((uintptr_t)B.st.vs * 0x9E3779B97F4A7C15ull >> 54); - if (vmemo[vi].of != B.st.vs) { + uint32_t vi = (uint32_t)((uintptr_t)dc->B.st.vs * 0x9E3779B97F4A7C15ull >> 54); + if (vmemo[vi].of != dc->B.st.vs) { int has, pending = 0; - sf_mvprev = 1; + dc->sf_mvprev = 1; const GfxShader *v = shader_for(ex, NVN_STAGE_VS, &has, &pending, 0); - sf_mvprev = 0; + dc->sf_mvprev = 0; if (!v || pending) { MO.building++; return; } GfxShaderIo io[32]; - uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(B.st.vs, io); + uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(dc->B.st.vs, io); uint32_t banks = extra ? gfx_shader_mvprev_banks(v) & 0x1fff8u : 0; - vmemo[vi].of = B.st.vs, vmemo[vi].vs = v, vmemo[vi].banks = banks; + vmemo[vi].of = dc->B.st.vs, vmemo[vi].vs = v, vmemo[vi].banks = banks; vk_mvprev_want((banks & ~(1u << 4)) >> 3); // binding index = bank - 3 } const GfxShader *vs = vmemo[vi].vs; @@ -4714,7 +4754,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi exact = e && *e ? strtol(e, NULL, 16) : -1; } if (exact >= 0 && banks != (uint32_t)exact) return; - mo_mask_note(banks, B.st.vs); + mo_mask_note(banks, dc->B.st.vs); // last frame's camera and blocks; held still when none of the position's blocks changed (most of the scenery) VkDescriptorBufferInfo cam, prevb[17]; float jit[2]; @@ -4753,7 +4793,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi } if (t0) MO.ns_part[1] += kern_now_ns() - tp, MO.n_part[1]++, tp = kern_now_ns(); GfxShaderIo io[32]; - uint32_t extra = gfx_shader_outputs(vs, io) & ~gfx_shader_outputs(B.st.vs, io); + uint32_t extra = gfx_shader_outputs(vs, io) & ~gfx_shader_outputs(dc->B.st.vs, io); const GfxShaderRes *r; if (gfx_shader_resources(vs, GFX_RES_IMG, &r)) { MO.storage++; @@ -4764,7 +4804,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi MO.no_fs++; return; } - VkPipeState ms = B.st; + VkPipeState ms = dc->B.st; ms.vs = vs, ms.fs = fs, ms.feedback = 0; memset(ms.color_fmt, 0, sizeof ms.color_fmt); ms.color_fmt[0] = NVN_FORMAT_RG16F; @@ -4776,7 +4816,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi // scene depth passes, anything nearer hides it (NX2NAVI_MVOBJ_EQUAL=1: EQUAL, for comparison) static int eq = -1; if (eq < 0) eq = rt_knob("MVOBJ_EQUAL", "0"); - int df = B.st.ds.df; + int df = dc->B.st.ds.df; df = eq ? NVN_DEPTH_FUNC_EQUAL : df == NVN_DEPTH_FUNC_LESS ? NVN_DEPTH_FUNC_LEQUAL : df == NVN_DEPTH_FUNC_GREATER ? NVN_DEPTH_FUNC_GEQUAL : df == NVN_DEPTH_FUNC_LEQUAL || df == NVN_DEPTH_FUNC_GEQUAL ? df : NVN_DEPTH_FUNC_EQUAL; @@ -4826,13 +4866,13 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi MQ.par = (VkDescriptorBufferInfo){pbuf, poff, 16}; memcpy(MQ.par_in, par_in, sizeof par_in); } - memset(mo_ovr, 0, sizeof mo_ovr); + memset(dc->mo_ovr, 0, sizeof dc->mo_ovr); for (int bk = 3; bk <= 16; bk++) // last frame's block, or this frame's where none was found - mo_ovr[NVN_STAGE_VS][bk + 15] = bk == 4 ? cam : have >> bk & 1 ? prevb[bk] : ubo_info(ex, NVN_STAGE_VS, bk); - mo_ovr[NVN_STAGE_FS][3] = MQ.par; - mo_ovr_on = 1; + dc->mo_ovr[NVN_STAGE_VS][bk + 15] = bk == 4 ? cam : have >> bk & 1 ? prevb[bk] : ubo_info(ex, NVN_STAGE_VS, bk); + dc->mo_ovr[NVN_STAGE_FS][3] = MQ.par; + dc->mo_ovr_on = 1; int bound = mo_desc(ex, info, q); - mo_ovr_on = 0; + dc->mo_ovr_on = 0; if (!bound) { MO.storage++; return; @@ -5165,9 +5205,9 @@ static int mfb_dispatch(VkExec *ex, const GfxShader *cs, int stage, VkBuffer rec VkPipeline p = vk_pipeline_compute(cs, &info); if (!p || !info || (info->pool && !pool_on())) return 0; bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); - mfb_stage = stage; + dc->mfb_stage = stage; bind_resources(ex, info); - mfb_stage = 0; + dc->mfb_stage = 0; VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0); vkCmdDispatchIndirect(ex->cb, rec, at); vk_pass_mark_end(ex->cb, m); @@ -5223,7 +5263,7 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { if (!mfb_dispatch(ex, c->cs_task, NVN_STAGE_TASK, rec, at)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; } vk_exec_barrier(ex); vk_meshfb_scan(ex->cb, at, c->ti.payload_bytes / 4, c->ti.threads, slot_words, verts); - B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own + dc->B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own ex->recorded += 4; vk_exec_barrier(ex); } @@ -5232,8 +5272,8 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { vk_meshfb_note(ex->cb, at); ex->recorded++; vk_exec_barrier(ex); - c0_binst = 0; - mfb_cur_vs = vs; + dc->c0_binst = 0; + dc->mfb_cur_vs = vs; int prim = c->mi.ipp == 1 ? NVN_DRAW_PRIMITIVE_POINTS : c->mi.ipp == 2 ? NVN_DRAW_PRIMITIVE_LINES : NVN_DRAW_PRIMITIVE_TRIANGLES; if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 2)) return; vk_exec_frame.draws++; @@ -5252,7 +5292,7 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { } PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); - c0_binst = 0; + dc->c0_binst = 0; if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first); vk_exec_frame.draws++; @@ -5417,7 +5457,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { if (!vk_mem_resolve(d->ind, 12, &ip) || ip.offset % 4) return warn_once(W_INDIRECT, "indirect mesh draw arguments at %#llx are not backed or not aligned: skipped", (unsigned long long)d->ind); - c0_binst = 0; + dc->c0_binst = 0; VK_MEM_USE(ip); if (fallback) { mfb_draw(ex, 0, &ip); @@ -5453,7 +5493,7 @@ static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType } static void op_draw(VkExec *ex, const p_draw *d) { - c0_binst = (uint32_t)d->base_instance; + dc->c0_binst = (uint32_t)d->base_instance; if (d->count <= 0 || !prepare_draw(ex, d->prim, d->count, 0)) return; vk_exec_frame.draws++; uint32_t inst = d->instances > 0 ? (uint32_t)d->instances : 1, first = (uint32_t)d->first, n = (uint32_t)d->count; @@ -5502,7 +5542,7 @@ static int restart_for(const VkExec *ex, int prim, int itype) { } static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { - c0_binst = (uint32_t)d->base_instance; + dc->c0_binst = (uint32_t)d->base_instance; static const uint32_t isize[3] = {1, 2, 4}; if (d->itype < 0 || d->itype > 2) return warn_once(W_INDEX, "index type %d is unknown: draw skipped", d->itype); int restart = restart_for(ex, d->prim, d->itype); @@ -5542,7 +5582,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { // dispatch). The base instance stays out of c0 (0x644): c0 cannot be patched by a transfer // inside a pass; gl_BaseInstance of the Vulkan draw is the argument's. Byte indices need the count to be widened: skipped. static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) { - c0_binst = 0; + dc->c0_binst = 0; if (d->itype != NVN_INDEX_TYPE_UNSIGNED_SHORT && d->itype != NVN_INDEX_TYPE_UNSIGNED_INT) return warn_once(W_INDEX, "indirect draw with index type %d: skipped", d->itype); VkMemSpan ib, ip; @@ -5803,7 +5843,7 @@ static void untranslated_dispatch_fallback(VkExec *ex) { // push did not go through the recorder (vk_rec.h rec_push; PORT-HANDOFF 7.12 item 11), which it now does. VkImage tm = e5 && dst_view ? vk_tonemap_record(ex->cb, e5, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, dw, dh) : VK_NULL_HANDLE; if (tm) { // the tonemapped picture, already at the target's size, then the same blit as below - B.pipe[1] = VK_NULL_HANDLE; // the replay must bind its compute pipeline again + dc->B.pipe[1] = VK_NULL_HANDLE; // the replay must bind its compute pipeline again si = tm, sw = dw, sh = dh; bl.srcOffsets[1] = (VkOffset3D){(int32_t)sw, (int32_t)sh, 1}; } @@ -5891,7 +5931,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS vk_exec_barrier_dispatch(ex); // ends the pass; earlier writes are visible to the shader bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); - c0_dim[0] = (uint32_t)d->x, c0_dim[1] = (uint32_t)d->y, c0_dim[2] = (uint32_t)d->z; + dc->c0_dim[0] = (uint32_t)d->x, dc->c0_dim[1] = (uint32_t)d->y, dc->c0_dim[2] = (uint32_t)d->z; twin_dispatch = 1; { // the replay's binding path is the CPU cost the frame profile needs to separate from recording uint64_t bt = bt_now(); @@ -5927,8 +5967,8 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vk_exec_barrier_dispatch(ex); VK_MEM_USE(ip); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); - c0_dim[0] = c0_dim[1] = c0_dim[2] = 0; - B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads + dc->c0_dim[0] = dc->c0_dim[1] = dc->c0_dim[2] = 0; + dc->B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads twin_dispatch = 1; { // the replay's binding path is the CPU cost the frame profile needs to separate from recording uint64_t bt = bt_now(); @@ -5937,17 +5977,17 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vk_exec_frame.bind_calls++; } twin_dispatch = 0; - if (B.c0_valid[2]) { // the shader binds c0 + if (dc->B.c0_valid[2]) { // the shader binds c0 // NVN2 programs read the group counts from c0 byte 0 (c0_info); at 0x200 their bank holds pushed constants // or the driver's tables, which the copy would overwrite const uint8_t *pctl, *pcode; uint32_t plen; int nvn2 = find_stage(ex->prog, NVN_STAGE_CS, &pctl, &pcode, &plen) && ctl_nvn2(pctl); - VkBufferCopy cp = {ip.offset, B.c0_off[2] + (nvn2 ? 0 : 0x200), sizeof(VkDispatchIndirectCommand)}; - vkCmdCopyBuffer(ex->cb, ip.buffer, B.c0_buf[2], 1, &cp); + VkBufferCopy cp = {ip.offset, dc->B.c0_off[2] + (nvn2 ? 0 : 0x200), sizeof(VkDispatchIndirectCommand)}; + vkCmdCopyBuffer(ex->cb, ip.buffer, dc->B.c0_buf[2], 1, &cp); ex->recorded++; vk_exec_barrier(ex); - B.c0_valid[2] = 0; // the range no longer holds B.c0[2] + dc->B.c0_valid[2] = 0; // the range no longer holds B.c0[2] } VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0); vkCmdDispatchIndirect(ex->cb, ip.buffer, ip.offset); @@ -5995,7 +6035,7 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { case OP_BIND_PROGRAM: { const p_bind_program *b = payload; ex->prog = b->prog, ex->prog_stages = b->stages; - bind_gen++; + dc->bind_gen++; break; } case OP_BIND_STATE: { @@ -6044,7 +6084,7 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { rt_log("vk: uniform buffer bind to index %d (bank %d), stage %d, %llu bytes: beyond Maxwell's banks, dropped", b->index, b->index + 3, b->stage, (unsigned long long)b->size); } - bind_gen++; + dc->bind_gen++; break; } case OP_BIND_SSBO: { @@ -6057,14 +6097,14 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { rt_log("vk: storage buffer bind, stage %d slot %d: %#llx, %llu bytes", b->stage, b->index, (unsigned long long)b->addr, (unsigned long long)b->size); } - bind_gen++; + dc->bind_gen++; break; } case OP_BIND_TEX: { const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 8 ? ex->tex[b->stage] : NULL, 128, b); ex->tex_serial++; - bind_gen++; + dc->bind_gen++; break; } case OP_BIND_SAMP: { @@ -6076,15 +6116,14 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 8 ? ex->img[b->stage] : NULL, 32, b); ex->tex_serial++; // the shader memo's handle check (feedback_scan reads only textures: a rescan at most) - bind_gen++; + dc->bind_gen++; break; } case OP_UPDATE_UB: op_update_ub(ex, payload); break; case OP_PUSH_CC: { const p_push_cc *p = payload; if (p->offset + (uint64_t)p->size <= sizeof ex->cs_push) memcpy(ex->cs_push + p->offset, p->data, p->size); - cs_push_bytes = p->offset + p->size; - bind_gen++; // storage buffers may come from these words (push_ptr) + dc->bind_gen++; // storage buffers may come from these words (push_ptr) break; } case OP_DRAW: op_draw(ex, payload); break;