diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 48c703e2..a725699f 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -43,7 +43,9 @@ typedef struct { static VkExec X; static int x_ready; -static _Thread_local int x_here; // this thread is inside vk_exec_record (vk_exec_inline_begin) +// The replay state this thread records with now (inside vk_exec_record, or a test's vk_exec_probe_inline), NULL when +// it records none: vk_exec_inline_begin hands out its command buffer. +static _Thread_local VkExec *x_cur; // ---- NX2NAVI_TRACE=gfx frame counters @@ -51,7 +53,7 @@ VkExecFrame vk_exec_frame; static struct { const uint8_t *mem; uint32_t w, h, fmt, n; } fw[64]; // textures written this frame static int nfw; -uint64_t vk_exec_recording_fence(void) { return x_here ? X.fence : 0; } +uint64_t vk_exec_recording_fence(void) { return x_cur ? x_cur->fence : 0; } void vk_exec_frame_written(const tex_t *t) { if (!(rt_trace & RT_TRACE_GFX) || !t) return; @@ -772,22 +774,23 @@ void vk_exec_pass_end(VkExec *ex) { } VkCommandBuffer vk_exec_inline_begin(int *reopen) { - VkExec *ex = &X; + VkExec *ex = x_cur; *reopen = 0; - if (!x_here || !ex->cb) return VK_NULL_HANDLE; + if (!ex || !ex->cb) return VK_NULL_HANDLE; *reopen = ex->in_pass; vk_exec_barrier(ex); // ends the pass; feedback-layout targets go back to GENERAL return ex->cb; } void vk_exec_inline_end(int reopen) { - X.recorded++; - if (reopen) vk_exec_pass_begin(&X); + VkExec *ex = x_cur; // set: the caller's vk_exec_inline_begin handed out its command buffer + ex->recorded++; + if (reopen) vk_exec_pass_begin(ex); } void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence) { X.cb = cb, X.fence = fence; - x_here = cb != VK_NULL_HANDLE; + x_cur = cb != VK_NULL_HANDLE ? &X : NULL; } typedef struct { VkBuffer b; VmaAllocation a; } InlineBuf; @@ -804,7 +807,7 @@ VkBuffer vk_exec_inline_buffer(VkDeviceSize size) { .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; if (!t || vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &t->b, &t->a, NULL) != VK_SUCCESS) { free(t); return VK_NULL_HANDLE; } - vk_completion_add(X.fence, free_inline_buf, t); + vk_completion_add(x_cur->fence, free_inline_buf, t); // inside vk_exec_inline_begin and _end return t->b; } @@ -1696,7 +1699,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t split_draws = part_start = 0; split_at = split_n < 0 ? 0 : split_n ? split_n : INT_MAX; // 0 until maybe_split reads the settings part_t0 = kern_now_ns(); - x_here = 1; + x_cur = ex; int evo = evo_enabled() && gfx_submit_queue; int slot = evo ? evq_slot(gfx_submit_queue) : -1; if (evo && slot < 0) evo = 0; @@ -1740,7 +1743,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t evo_queue = 0; } vk_exec_pass_end(ex); - x_here = 0; + x_cur = NULL; vk_exec_event_sync(0); if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes // NX2NAVI_WB_SPLIT: the write-backs in a part of their own, after one that signals the executed timeline diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index fa20d2c8..a768a7a9 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -125,9 +125,6 @@ typedef struct { int nwritten, cwritten; } CbRes; -static CbRes *cur; -static uint64_t cur_fence; - // Descriptor pools and full-size transient chunks go back to these lists when their command buffer completes, instead // of being destroyed and created again for the next one: each creation is a driver memory allocation (on Windows an // allocation, a map and a paging-fence wait on the CPU), about ten of them per frame inside the replay. @@ -138,8 +135,8 @@ static VkDescriptorPool spare_pool[SPARE_POOLS]; static Chunk spare_chunk[SPARE_CHUNKS]; static int nspare_pool, nspare_chunk; -// State of the command buffer being recorded that the Vulkan command buffer itself holds. -static struct { +// State of the command buffer being recorded that the Vulkan command buffer itself holds (DrawCtx.B). +typedef struct { VkPipeline pipe[2]; // bound per bind point: 0 graphics, 1 compute VkPipeState st; // the state vk_pipeline_set_dynamic last recorded int st_valid; @@ -167,14 +164,14 @@ static struct { VkDescriptorSetLayout ds_last_layout[2]; int ds_pushed[2]; // set number holds the push of DS.psig under ds_layout int res_valid; // the graphics sets bound hold what the inputs in RS give (bind_resources) -} B; +} CbState; -// Contents of B.ds_last: the set's packed descriptors (VkDescSlot), which with its layout determine every write. A draw -// whose allocated set would hold the same reuses the set instead of allocating and writing one (a per-draw +// Contents of B.ds_last (DrawCtx.DS): the set's packed descriptors (VkDescSlot), which with its layout determine every +// write. A draw whose allocated set would hold the same reuses the set instead of allocating and writing one (a per-draw // cost). psig: the same for the last push per set number; a push equal to it under the same layout is not recorded // again (the descriptors stay bound across pipeline binds of that layout). #define DS_SIG_BYTES (MAX_DESC * sizeof(VkDescSlot)) -static struct { uint32_t n[2], pn[2]; uint8_t sig[2][DS_SIG_BYTES], psig[2][DS_SIG_BYTES]; } DS; +typedef struct { uint32_t n[2], pn[2]; uint8_t sig[2][DS_SIG_BYTES], psig[2][DS_SIG_BYTES]; } DsSig; // NX2NAVI_BIND_STATS counters (bind_stats). static struct { @@ -182,13 +179,12 @@ static struct { uint64_t draws, bind[2], desc[2], push, push_skip, set_new, set_reuse, ac_miss, same_prog, same_all, bind_skip; } BS = {-1}; -// The inputs of the last graphics bind_resources (bind_sig) and what its build counted for the traffic census, which a -// skipped build counts again. RS_BYTES bounds the bound-slot bytes a pipeline reads (RsList): per binding at most one -// c0 storage table, otherwise one bound range or handle per descriptor. +// The inputs of the last graphics bind_resources (bind_sig, DrawCtx.RS) and what its build counted for the traffic +// census, which a skipped build counts again. RS_BYTES bounds the bound-slot bytes a pipeline reads (RsList): per +// binding at most one c0 storage table, otherwise one bound range or handle per descriptor. #define RS_BYTES (MAX_BINDINGS * sizeof(((VkExec *)0)->ssbo[0]) + MAX_DESC * sizeof(p_bind_buf)) typedef struct { int cls, use; uint64_t bytes; } RsCount; -static struct { - int on; // NX2NAVI_BIND_SKIP: 1 skip (default), 2 check, 0 off +typedef struct { uint32_t hn; // words of h the last signature filled; 0: none uint64_t h[24]; // generations, feedback images, support inputs, base instance const VkPipeInfo *info; // the pipeline whose RsList ranges body holds, as of bind op generation gen @@ -201,7 +197,8 @@ static struct { int cmds; // check mode: pushes, binds and set writes the build recorded uint32_t ckn; // check mode: the last build's packed descriptors (bytes in ck) uint8_t ck[DS_SIG_BYTES]; -} RS = {-1}; +} RsState; +static int rs_on = -1; // NX2NAVI_BIND_SKIP: 1 skip (default), 2 check, 0 off // The bound-slot bytes of VkExec a pipeline's descriptors read, as ranges merged where they touch, built once per // pipeline info: c0's storage table, the uniform and storage ranges of the bound slots and the texture and image @@ -213,11 +210,77 @@ typedef struct { uint32_t off, len; } RsRange; typedef struct { int st; const GfxShaderRes *res; } RsPtr; typedef struct { uint32_t n, bytes; uint8_t sup, c0, ok, np; RsPtr p[RS_PTRS]; RsRange r[]; } RsList; static nvn_hmap rs_lists; // VkPipeInfo pointer -> RsList, until shutdown -static const VkPipeInfo *rs_last; // the last lookup -static const RsList *rs_last_l; -static uint32_t c0_dim[3], c0_binst; // the dispatch size and base instance of the draw or dispatch being bound -static uint64_t bind_gen = 1; // bumped by every bind op that writes ex->prog, ubo, ssbo, tex or img +// Types of the context's caches and memos; each is described where it is used. +#define STAGEM_N 512 // find_stage +#define SLOTM_N 4096 // slot_mask_of +#define PB_N 8 // per-binding descriptor reuse (pb_take) +#define PB_DESC 192 +#define PB_G 17 +typedef struct { + uint64_t tex[32], img[32], ttok[32], itok[32]; + uint32_t tgen[32], igen[32]; + uint8_t tt[32], it[32], ifmt[32]; +} AcSlots; +typedef struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } StageMemo; +typedef struct { uint64_t serial; int st; uint32_t m; } SlotMask; +typedef struct { int cls, use, has; uint64_t n; } PbCount; +typedef struct { + const VkPipeInfo *info; + uint64_t g[PB_G]; + uint8_t ok[PB_DESC]; // descriptor j holds a result that may be reused + uint64_t in[PB_DESC][4]; // the values it was resolved from (pb_key) + VkDescSlot d[PB_DESC]; + PbCount cnt[PB_DESC]; // what its resolution counted for the census +} PbEntry; +typedef struct { + const prog_t *prog; + uint64_t serial; + uint32_t gen, tex_serial; + uint64_t tex[2][32], img[2][32]; + VkPipeState pipe; +} MemoKey; + +// ---- the recording context +// What the draws, binds and dispatches keep from one call to the next while they record a command buffer: the state +// the command buffer holds (B), the signatures that let a draw skip its descriptor build or push (DS, RS, PB), the +// previous draw's shaders and pipeline (SM, PM), the shader lookup's caches (AC, stagem, slotm) and the submission's +// transient resources (res). A thread reaches the one it records with through dc; every thread starts with dc_main, +// the replay's, which is the only one while the replay records on a single thread. Shared tables (image views, +// shaders, descriptor lists, the texture pool sets, the stand-ins) stay file statics. +typedef struct DrawCtx { + CbRes *res; // transient resources of the submission being recorded (cb_res) + uint64_t res_fence; // the submission they belong to + CbState B; + DsSig DS; + RsState RS; + const VkPipeInfo *rs_last; // rs_list's last lookup + const RsList *rs_last_l; + uint32_t c0_dim[3], c0_binst; // the dispatch size and base instance of the draw or dispatch being bound + uint64_t bind_gen; // bumped by every bind op that writes ex->prog, ubo, ssbo, tex or img + AcSlots AC[8]; // answers_for: texture and image types per stage + StageMemo stagem[STAGEM_N]; // find_stage: the shader serving each NVN stage, per program serial + SlotMask slotm[SLOTM_N]; // slot_mask_of: the slots a (program, stage) translation reads + int sf_mvprev; // shader_for asks for the vertex shader's previous-frame variant (motion) + int mfb_stage; // the mesh fallback binds this NVN stage's resources (nvn_stage_of) + const GfxShader *mfb_cur_vs; // the mesh fallback's vertex stage (prepare_draw_ex) + PbCount *pb_cap; // per-binding reuse: where the resolving descriptor's census count goes + int pb_spoiled; // ... and it may not be kept + PbEntry PB[PB_N]; + PbEntry *pb_e; // the build running, NULL when it keeps nothing + uint64_t pb_key_cur[4]; // pb_take to pb_keep: the descriptor's values + int pb_key_ok; // ... and whether it may be kept at all + PbCount pb_cnt_cur; + int mo_ovr_on; // per-object motion: mo_ovr replaces uniform bindings (ubo_info) + VkDescriptorBufferInfo mo_ovr[2][32]; // [vertex, fragment][slot]; buffer 0: not replaced + struct { int valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM; // the shader memo (shader_memo_on) + struct { int valid; VkPipeState s; const VkPipeInfo *info; } PM; // the pipeline memo (pipe_memo_on) + VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views + VkDescSlot bind_d[MAX_DESC]; + VkBufferView bind_bv[MAX_DESC]; +} DrawCtx; +static DrawCtx dc_main = {.bind_gen = 1}; +static _Thread_local DrawCtx *dc = &dc_main; static void cb_done(void *arg) { // completion thread CbRes *r = arg; @@ -243,11 +306,11 @@ static void cb_done(void *arg) { // completion thread free(r->written), free(r->pools), free(r->chunks), free(r); } -void vk_exec_draw_new_cb(void) { memset(&B, 0, sizeof B); } +void vk_exec_draw_new_cb(void) { memset(&dc->B, 0, sizeof dc->B); } void vk_exec_draw_shaders(const GfxShader **vs, const GfxShader **fs) { - *vs = B.st_valid ? B.st.vs : NULL; - *fs = B.st_valid ? B.st.fs : NULL; + *vs = dc->B.st_valid ? dc->B.st.vs : NULL; + *fs = dc->B.st_valid ? dc->B.st.fs : NULL; } // Marks a dispatch about to be recorded (vk.h vk_pass_mark_begin); the caller ends it after the dispatch. @@ -261,13 +324,14 @@ static VkPassMark *dispatch_mark(VkExec *ex, const GfxShader *cs, uint32_t x, ui } static CbRes *cb_res(VkExec *ex) { - if (cur && cur_fence == ex->fence) return cur; - cur = calloc(1, sizeof *cur); - if (!cur) rt_fatal("vk: out of memory"); - cur_fence = ex->fence; - memset(&B, 0, sizeof B); - vk_completion_add(ex->fence, cb_done, cur); - return cur; + DrawCtx *d = dc; + if (d->res && d->res_fence == ex->fence) return d->res; + CbRes *r = calloc(1, sizeof *r); + if (!r) rt_fatal("vk: out of memory"); + d->res = r, d->res_fence = ex->fence; + memset(&d->B, 0, sizeof d->B); + vk_completion_add(ex->fence, cb_done, r); + return r; } static uint8_t *scratch(VkExec *ex, uint64_t size, VkBuffer *buf, VkDeviceSize *off) { @@ -469,7 +533,7 @@ void vk_exec_draw_shutdown(void) { } memset(SD, 0, sizeof SD); nvn_hm_clear(&rs_lists, free); - rs_last = NULL, rs_last_l = NULL; + dc_main.rs_last = NULL, dc_main.rs_last_l = NULL; // the replay's context: no other records now for (int i = 0; i < nspare_pool; i++) vkDestroyDescriptorPool(c->device, spare_pool[i], NULL); for (int i = 0; i < nspare_chunk; i++) vmaDestroyBuffer(c->vma, spare_chunk[i].buf, spare_chunk[i].alloc); nspare_pool = nspare_chunk = 0; @@ -829,13 +893,10 @@ static int find_stage_scan(const prog_t *p, int nvn_stage, const uint8_t **ctl, // Which shader of a program serves each NVN stage, and its code length, per program serial (new at every // nvnProgramSetShaders): the scan reads two words of every stage's control block, cold guest cache lines, and runs -// several times a draw. Direct-mapped; a collision only scans again. -#define STAGEM_N 512 -static struct { uint64_t serial; int8_t idx[6]; uint32_t len[6]; } stagem[STAGEM_N]; - +// several times a draw. Direct-mapped (DrawCtx.stagem); a collision only scans again. static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { if (!p->serial || (unsigned)nvn_stage >= 6) return find_stage_scan(p, nvn_stage, ctl, code, len); - typeof(stagem[0]) *m = &stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55]; + StageMemo *m = &dc->stagem[p->serial * 0x9E3779B97F4A7C15ull >> 55]; if (m->serial != p->serial) { m->serial = p->serial; memset(m->idx, -1, sizeof m->idx); @@ -855,8 +916,7 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const // Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it // was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve -// otherwise (slot_still). -static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32], ifmt[32]; } AC[8]; +// otherwise (slot_still). DrawCtx.AC, per stage. // vk_texture_slot_token of the slot a texture or image handle names; an empty handle always resolves to nothing. static uint64_t token_of(uint64_t h) { return h ? vk_texture_slot_token((int)(h & 0xfffff)) : 1; } @@ -875,20 +935,17 @@ static int shader_memo_on(void); // The slots of (program, stage) whose types its translation reads (gfx_shader_slot_mask), learned from its first // shader; ~0u while unknown. The types of other slots select the same shader, so answers_for leaves them zero instead -// of looking up all 64 handles. Direct-mapped by serial and stage; a collision only relearns. -#define SLOTM_N 4096 -static struct { uint64_t serial; int st; uint32_t m; } slotm[SLOTM_N]; - +// of looking up all 64 handles. Direct-mapped by serial and stage (DrawCtx.slotm); a collision only relearns. static uint32_t slotm_idx(uint64_t serial, int st) { return (uint32_t)((serial * 8 + (uint64_t)st) * 0x9E3779B97F4A7C15ull >> 52); } static uint32_t slot_mask_of(uint64_t serial, int st) { - const typeof(slotm[0]) *e = &slotm[slotm_idx(serial, st)]; + const SlotMask *e = &dc->slotm[slotm_idx(serial, st)]; return serial && e->serial == serial && e->st == st ? e->m : ~0u; } static void slot_mask_learn(uint64_t serial, int st, const GfxShader *s) { if (!serial || slot_mask_of(serial, st) != ~0u) return; - slotm[slotm_idx(serial, st)] = (typeof(slotm[0])){serial, st, gfx_shader_slot_mask(s)}; + dc->slotm[slotm_idx(serial, st)] = (SlotMask){serial, st, gfx_shader_slot_mask(s)}; } static uint8_t type_of_handle(uint64_t h) { // texture lock held @@ -925,7 +982,7 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G memcpy(&l, ctl_local(ctl) + 4 * k, 4); a->local[k] = (uint16_t)(l ? l : 1); } - typeof(AC[0]) *c = &AC[st]; + AcSlots *c = &dc->AC[st]; // The generation and tokens are read before the lookups (one that creates an entry changes them) and unlocked: a // registration racing with them is ordered after this draw, and a slot stored under older ones is only looked up // again. @@ -953,7 +1010,7 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G // stages, vk_pipeline_async_on): a shader not translated yet is queued and NULL returned with *pending = 1. await_ms: // a shader queued earlier is waited for instead, that long at most (gfx_shader_await; UINT32_MAX: until it is done), // and *pending = 1 says the wait ended first. -static int sf_mvprev; // shader_for asks for the vertex shader's last-frame variant (per-object motion) +// DrawCtx.sf_mvprev: shader_for asks for the vertex shader's last-frame variant (per-object motion). // Probes only (mesh_fb_probe.c): the shader of the program stage whose control section is ctl, in place of a // translation. The probes build their programs from modules made here, which no translator has an input for. @@ -974,7 +1031,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin GfxShaderAnswers a; uint64_t serial = ex->prog->serial; answers_for(ex, st, ctl, slot_mask_of(serial, st), &a); - a.mvprev = (uint8_t)(sf_mvprev && st == NVN_STAGE_VS); + a.mvprev = (uint8_t)(dc->sf_mvprev && st == NVN_STAGE_VS); ShaderEnt *set = NULL; if (serial) { uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded @@ -1011,11 +1068,11 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin // ---- descriptors // The mesh fallback (below, "mesh fallback") runs a program's task and mesh stages as compute modules: while it binds -// one's resources, a compute module's bindings are those of this NVN stage (6 task, 7 mesh), not of the compute stage. -static int mfb_stage; +// one's resources, a compute module's bindings are those of this NVN stage (6 task, 7 mesh), not of the compute stage +// (DrawCtx.mfb_stage). static int nvn_stage_of(int gfx_stage) { - if (gfx_stage == GFX_STAGE_CS && mfb_stage) return mfb_stage; + if (gfx_stage == GFX_STAGE_CS && dc->mfb_stage) return dc->mfb_stage; return gfx_stage == GFX_STAGE_VS ? NVN_STAGE_VS : gfx_stage == GFX_STAGE_FS ? NVN_STAGE_FS : gfx_stage == GFX_STAGE_TS ? NVN_STAGE_TASK : gfx_stage == GFX_STAGE_MS ? NVN_STAGE_MESH : NVN_STAGE_CS; } @@ -1026,26 +1083,23 @@ static VkDescriptorBufferInfo zero_info(uint64_t max) { // Per-binding descriptor reuse (bind_resources, PB): while a descriptor that may be kept resolves, pb_cap is where its // census count goes, so a later reuse counts the same bytes again; pb_spoiled is set when the resolution copied guest -// bytes into transient memory or counted more than once, and such a descriptor is not kept. -typedef struct { int cls, use, has; uint64_t n; } PbCount; -static PbCount *pb_cap; -static int pb_spoiled; +// bytes into transient memory or counted more than once, and such a descriptor is not kept (both in DrawCtx). // vk_mem_count, also kept per (class, use) while a graphics build runs (RS). static void count_use(int cls, int use, uint64_t n) { vk_mem_count(cls, use, n); - if (pb_cap) { - if (pb_cap->has) pb_spoiled = 1; - else *pb_cap = (PbCount){cls, use, 1, n}; + if (dc->pb_cap) { + if (dc->pb_cap->has) dc->pb_spoiled = 1; + else *dc->pb_cap = (PbCount){cls, use, 1, n}; } - if (!RS.rec) return; - for (int i = 0; i < RS.ncnt; i++) - if (RS.cnt[i].cls == cls && RS.cnt[i].use == use) { - RS.cnt[i].bytes += n; + if (!dc->RS.rec) return; + for (int i = 0; i < dc->RS.ncnt; i++) + if (dc->RS.cnt[i].cls == cls && dc->RS.cnt[i].use == use) { + dc->RS.cnt[i].bytes += n; return; } - if (RS.ncnt < (int)(sizeof RS.cnt / sizeof RS.cnt[0])) RS.cnt[RS.ncnt++] = (RsCount){cls, use, n}; - else RS.cnt_over = 1; + if (dc->RS.ncnt < (int)(sizeof dc->RS.cnt / sizeof dc->RS.cnt[0])) dc->RS.cnt[dc->RS.ncnt++] = (RsCount){cls, use, n}; + else dc->RS.cnt_over = 1; } // A buffer descriptor for [addr, addr+size) of guest memory: its backing (a pool's VRAM mirror when it has one) when @@ -1069,8 +1123,8 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, if (sp.alloc) return 0; VkBuffer b; VkDeviceSize o; - RS.volatile_ = 1; - pb_spoiled = 1; + dc->RS.volatile_ = 1; + dc->pb_spoiled = 1; memcpy(scratch(ex, n, &b, &o), (const void *)(uintptr_t)addr, n); vk_mem_count(sp.cls, VK_MUSE_CPU_READ, n); vk_mem_count(VK_MCLS_SCRATCH, use, n); @@ -1130,7 +1184,6 @@ static int c0_on(void) { // NX2NAVI_FIX_C0=0: c0 reads zeros instead of the dr // c0 from replay scratch; a bank equal to the stage's previous one reuses its range, so it costs no copy. static void nvn2_cs_tables(const uint8_t *ctl, uint32_t *ubos, uint32_t *ssbos); static uint32_t nvn2_cs_tables_at(const uint8_t *ctl); -static uint32_t cs_push_bytes; // the size of the last PushComputeConstants (OP_PUSH_CC), replay thread static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : 2; @@ -1138,25 +1191,25 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { // graphics: equal inputs give the same bank, so neither the fill nor the 2 KiB compare is needed. // A vertex or fragment bank depends on the storage slots and the base instance alone (the rest of its inputs record // is zero): those two are compared without building and comparing the whole 1.8 KiB record on every draw. - if (k < 2 && B.c0_valid[k] && B.c0_in[k].binst == c0_binst && - !memcmp(B.c0_in[k].ssbo, ex->ssbo[st], sizeof B.c0_in[k].ssbo)) - return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; - typeof(B.c0_in[0]) in; + if (k < 2 && dc->B.c0_valid[k] && dc->B.c0_in[k].binst == dc->c0_binst && + !memcmp(dc->B.c0_in[k].ssbo, ex->ssbo[st], sizeof dc->B.c0_in[k].ssbo)) + return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; + typeof(dc->B.c0_in[0]) in; memset(&in, 0, sizeof in); memcpy(in.ssbo, ex->ssbo[st], sizeof in.ssbo); - if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, c0_dim, sizeof in.dim); - else in.binst = c0_binst; + if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, dc->c0_dim, sizeof in.dim); + else in.binst = dc->c0_binst; // NVN2 compute: the bank holds the push constants from byte 0x10 (then the program's handle table, which the // host does not read: textures and images are bound by slot). const uint8_t *pctl, *pcode; uint32_t plen; int nvn2_push = st == NVN_STAGE_CS && ex->prog && find_stage(ex->prog, st, &pctl, &pcode, &plen) && ctl_nvn2(pctl); if (nvn2_push) memcpy(in.push, ex->cs_push, sizeof in.push); - if (B.c0_valid[k] && !memcmp(&in, &B.c0_in[k], sizeof in)) - return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; - B.c0_in[k] = in; + if (dc->B.c0_valid[k] && !memcmp(&in, &dc->B.c0_in[k], sizeof in)) + return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; + dc->B.c0_in[k] = in; uint8_t tmp[VK_C0_BYTES]; - vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], c0_dim, c0_binst); + vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], dc->c0_dim, dc->c0_binst); if (nvn2_push) { // the pushed constants from 0x10, the driver's tables after them (push_ptr, nvn2_cs_tables_at) uint32_t nu, ns, ta = nvn2_cs_tables_at(pctl); @@ -1166,7 +1219,7 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { // Bytes 0-0xb of an NVN2 compute bank are the dispatch's group counts (x, y, z). The fog pass after the // integration (passes 0115/0116 of a Kokiri frame) maps each thread to tid / (groups * 16 - 1): with zeros // there every thread read the same point. An indirect dispatch leaves them 0 (op_dispatch_indirect). - memcpy(tmp, c0_dim, 12); + memcpy(tmp, dc->c0_dim, 12); for (uint32_t k = 0; k < nu + ns && ta + 16 * k + 12 <= VK_C0_BYTES; k++) { const p_bind_buf *sb = k < nu ? (k < 18 ? &ex->ubo[st][k] : NULL) : (k - nu < 16 ? &ex->ssbo[st][k - nu] : NULL); if (!sb) continue; @@ -1174,12 +1227,12 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { memcpy(tmp + ta + 16 * k, &sb->addr, 8), memcpy(tmp + ta + 16 * k + 8, &sz, 4); } } - if (!B.c0_valid[k] || memcmp(tmp, B.c0[k], VK_C0_BYTES)) { - memcpy(scratch(ex, VK_C0_BYTES, &B.c0_buf[k], &B.c0_off[k]), tmp, VK_C0_BYTES); - memcpy(B.c0[k], tmp, VK_C0_BYTES); - B.c0_valid[k] = 1; + if (!dc->B.c0_valid[k] || memcmp(tmp, dc->B.c0[k], VK_C0_BYTES)) { + memcpy(scratch(ex, VK_C0_BYTES, &dc->B.c0_buf[k], &dc->B.c0_off[k]), tmp, VK_C0_BYTES); + memcpy(dc->B.c0[k], tmp, VK_C0_BYTES); + dc->B.c0_valid[k] = 1; } - return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; + return (VkDescriptorBufferInfo){dc->B.c0_buf[k], dc->B.c0_off[k], VK_C0_BYTES}; } // The buffer range limits, read once: the device's properties are a cold cache line on every binding of every draw. @@ -1193,13 +1246,13 @@ static void limits_init(void) { } // Per-object motion (section at the end): while it records a draw again, these replace the uniform bindings of the -// vertex stage's slots 18-20 (last frame's camera, bones and place) and the fragment stage's slot 3 (its parameters). -static int mo_ovr_on; -static VkDescriptorBufferInfo mo_ovr[2][32]; // [vertex, fragment][slot]; buffer 0: not replaced +// vertex stage's slots 18-20 (last frame's camera, bones and place) and the fragment stage's slot 3 (its parameters) +// (DrawCtx.mo_ovr). static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { - if (mo_ovr_on && (st == NVN_STAGE_VS || st == NVN_STAGE_FS) && slot >= 0 && slot < 32 && mo_ovr[st][slot].buffer) - return mo_ovr[st][slot]; + if (dc->mo_ovr_on && (st == NVN_STAGE_VS || st == NVN_STAGE_FS) && slot >= 0 && slot < 32 && + dc->mo_ovr[st][slot].buffer) + return dc->mo_ovr[st][slot]; if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); uint64_t max = LIM.ubo_max, align = LIM.ubo_align; VkDescriptorBufferInfo bi; @@ -2955,12 +3008,12 @@ static int push_skip_on(void) { // NX2NAVI_PUSH_SKIP=0: every graphics draw pu // NX2NAVI_BIND_SKIP=0 builds every draw; =check builds every draw and stops when a draw with equal inputs gets other // writes, census counts or descriptor commands. static int bind_skip_on(void) { - if (RS.on < 0) { + if (rs_on < 0) { const char *e = cfg_value("NX2NAVI_BIND_SKIP"); - RS.on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; - if (RS.on != 1) rt_log("vk_exec: descriptor build skip %s", RS.on ? "check" : "off"); + rs_on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; + if (rs_on != 1) rt_log("vk_exec: descriptor build skip %s", rs_on ? "check" : "off"); } - return RS.on; + return rs_on; } static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, size_t len) { @@ -2970,7 +3023,7 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si } static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { - if (info == rs_last) return rs_last_l; + if (info == dc->rs_last) return dc->rs_last_l; if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info); RsList *l = nvn_hm_get(&rs_lists, &info); if (!l) { @@ -3020,7 +3073,7 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { memcpy(l->r, tmp, sizeof *tmp * n); nvn_hm_put(&rs_lists, &info, l); } - rs_last = info, rs_last_l = l; + dc->rs_last = info, dc->rs_last_l = l; return l; } @@ -3029,8 +3082,8 @@ static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) { // Check mode compares them anyway and stops when the bind op generation said they could not differ but they do. static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { const RsList *l = rs_list(ex, info); - if (!l->ok) return RS.hn = 0, RS.info = NULL, 0; - uint64_t *h = RS.h, diff = 0; + if (!l->ok) return dc->RS.hn = 0, dc->RS.info = NULL, 0; + uint64_t *h = dc->RS.h, diff = 0; uint32_t n = 0; #define PUT(v) do { uint64_t v_ = (uint64_t)(v); diff |= h[n] ^ v_; h[n++] = v_; } while (0) PUT((uintptr_t)info); @@ -3049,16 +3102,16 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { PUT(a); PUT(ps); } - if (l->c0) PUT(c0_binst); + if (l->c0) PUT(dc->c0_binst); for (int i = 0; i < l->np; i++) PUT(l->p[i].res->ptr ? ptr_addr(ex, l->p[i].st, l->p[i].res) : ubo_ptr(ex, l->p[i].st, l->p[i].res)); #undef PUT - diff |= n != RS.hn; - RS.hn = n; - int fast = RS.info == info && RS.gen == bind_gen; - if (fast && RS.on != 2) return !diff; + diff |= n != dc->RS.hn; + dc->RS.hn = n; + int fast = dc->RS.info == info && dc->RS.gen == dc->bind_gen; + if (fast && rs_on != 2) return !diff; const uint8_t *e = (const uint8_t *)ex; - uint8_t *d = RS.body; + uint8_t *d = dc->RS.body; int bd = 0; for (uint32_t i = 0; i < l->n; i++) { const RsRange *r = &l->r[i]; @@ -3067,7 +3120,7 @@ static int bind_sig(const VkExec *ex, const VkPipeInfo *info) { d += r->len; } if (fast && bd) rt_fatal("vk_exec: descriptor build skip check: bound slots changed without a bind op"); - RS.info = info, RS.gen = bind_gen; + dc->RS.info = info, dc->RS.gen = dc->bind_gen; return !diff && !bd; } @@ -3091,22 +3144,7 @@ static inline uint64_t bt_now(void) { return rt_trace & RT_TRACE_GFX ? plat_now_ // resolution that copied guest bytes into transient memory is never kept. Off while per-object motion replaces uniform // bindings (mo_ovr_on). NX2NAVI_BIND_REUSE=0 resolves every descriptor; =check resolves every descriptor and stops on // one that resolves differently from its kept result with the same inputs. -#define PB_N 8 -#define PB_DESC 192 -#define PB_G 17 -typedef struct { - const VkPipeInfo *info; - uint64_t g[PB_G]; - uint8_t ok[PB_DESC]; // descriptor j holds a result that may be reused - uint64_t in[PB_DESC][4]; // the values it was resolved from (pb_key) - VkDescSlot d[PB_DESC]; - PbCount cnt[PB_DESC]; // what its resolution counted for the census -} PbEntry; -static PbEntry PB[PB_N]; -static PbEntry *pb_e; // the build running, NULL when it keeps nothing -static uint64_t pb_key_cur[4]; // pb_take to pb_keep: the descriptor's values, and whether it may be kept at all -static int pb_key_ok; -static PbCount pb_cnt_cur; +// The entries (PbEntry, DrawCtx.PB) and the build running (DrawCtx.pb_e) belong to the recording context. static int pb_mode = -1; static int pb_on(void) { @@ -3136,7 +3174,7 @@ static void pb_globals(const VkExec *ex, uint64_t g[PB_G]) { // info's entry, emptied when it held another pipeline's results or the globals moved. static PbEntry *pb_entry(const VkPipeInfo *info, const uint64_t g[PB_G]) { uintptr_t p = (uintptr_t)info; - PbEntry *e = &PB[(p >> 6 ^ p >> 13) % PB_N]; + PbEntry *e = &dc->PB[(p >> 6 ^ p >> 13) % PB_N]; if (e->info != info || memcmp(e->g, g, sizeof e->g)) { e->info = info; memcpy(e->g, g, sizeof e->g); @@ -3177,38 +3215,39 @@ static int pb_key(const VkExec *ex, const VkPipeBinding *b, uint32_t k, uint64_t // Before descriptor j (descriptor k of binding b) resolves: 1 when its kept result serves, *o filled and its census // counted again. 0: resolve it, then pb_keep. static int pb_take(const VkExec *ex, const VkPipeBinding *b, uint32_t j, uint32_t k, VkDescSlot *o) { - pb_key_ok = 0; - if (!pb_e || j >= PB_DESC || !pb_key(ex, b, k, pb_key_cur)) return 0; - pb_key_ok = 1; - if (pb_mode != 2 && pb_e->ok[j] && !memcmp(pb_e->in[j], pb_key_cur, sizeof pb_key_cur)) { - const PbCount *c = &pb_e->cnt[j]; - *o = pb_e->d[j]; + dc->pb_key_ok = 0; + if (!dc->pb_e || j >= PB_DESC || !pb_key(ex, b, k, dc->pb_key_cur)) return 0; + dc->pb_key_ok = 1; + if (pb_mode != 2 && dc->pb_e->ok[j] && !memcmp(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur)) { + const PbCount *c = &dc->pb_e->cnt[j]; + *o = dc->pb_e->d[j]; if (c->has) count_use(c->cls, c->use, c->n); vk_exec_frame.bind_reused++; return 1; } - pb_cnt_cur = (PbCount){0}; - pb_cap = &pb_cnt_cur, pb_spoiled = 0; + dc->pb_cnt_cur = (PbCount){0}; + dc->pb_cap = &dc->pb_cnt_cur, dc->pb_spoiled = 0; return 0; } // After descriptor j resolved to *o. static void pb_keep(uint32_t j, const VkDescSlot *o) { - pb_cap = NULL; - if (!pb_e || j >= PB_DESC) return; + dc->pb_cap = NULL; + if (!dc->pb_e || j >= PB_DESC) return; vk_exec_frame.bind_resolved++; - if (!pb_key_ok) { - pb_e->ok[j] = 0; + if (!dc->pb_key_ok) { + dc->pb_e->ok[j] = 0; return; } - if (pb_mode == 2 && pb_e->ok[j] && !pb_spoiled && !memcmp(pb_e->in[j], pb_key_cur, sizeof pb_key_cur) && - memcmp(&pb_e->d[j], o, sizeof *o)) + if (pb_mode == 2 && dc->pb_e->ok[j] && !dc->pb_spoiled && + !memcmp(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur) && + memcmp(&dc->pb_e->d[j], o, sizeof *o)) rt_fatal("vk_exec: per-binding descriptor reuse check: descriptor %u resolves differently from its kept result " "with the same inputs", j); - pb_e->ok[j] = (uint8_t)!pb_spoiled; - memcpy(pb_e->in[j], pb_key_cur, sizeof pb_key_cur); - pb_e->d[j] = *o; - pb_e->cnt[j] = pb_cnt_cur; + dc->pb_e->ok[j] = (uint8_t)!dc->pb_spoiled; + memcpy(dc->pb_e->in[j], dc->pb_key_cur, sizeof dc->pb_key_cur); + dc->pb_e->d[j] = *o; + dc->pb_e->cnt[j] = dc->pb_cnt_cur; } // Writes allocated set s from its packed descriptors d: through the layout's update template, or with the write @@ -3231,32 +3270,32 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c // The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update // template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. static void bind_resources(VkExec *ex, const VkPipeInfo *info) { - static VkWriteDescriptorSet w[MAX_BINDINGS]; - static VkDescSlot d[MAX_DESC]; - static VkBufferView bv[MAX_DESC]; + VkWriteDescriptorSet *w = dc->bind_w; + VkDescSlot *d = dc->bind_d; + VkBufferView *bv = dc->bind_bv; if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS); bind_stats(info); int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS; - int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = B.res_valid; + int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = dc->B.res_valid; if (skip) { uint64_t sig_t0 = bt_now(); same = bind_sig(ex, info); vk_exec_frame.bind_sig_ns += (uint64_t)(bt_now() - sig_t0); if (same && was_valid && skip == 1) { - for (int i = 0; i < RS.ncnt; i++) vk_mem_count(RS.cnt[i].cls, RS.cnt[i].use, RS.cnt[i].bytes); + for (int i = 0; i < dc->RS.ncnt; i++) vk_mem_count(dc->RS.cnt[i].cls, dc->RS.cnt[i].use, dc->RS.cnt[i].bytes); BS.bind_skip++; ex->recorded++; return; } - B.res_valid = 0; // until the build below ends - if (skip == 2) memcpy(RS.pcnt, RS.cnt, sizeof RS.cnt), RS.pncnt = RS.ncnt; - RS.rec = 1, RS.ncnt = 0, RS.cnt_over = 0, RS.volatile_ = 0, RS.cmds = 0; + dc->B.res_valid = 0; // until the build below ends + if (skip == 2) memcpy(dc->RS.pcnt, dc->RS.cnt, sizeof dc->RS.cnt), dc->RS.pncnt = dc->RS.ncnt; + dc->RS.rec = 1, dc->RS.ncnt = 0, dc->RS.cnt_over = 0, dc->RS.volatile_ = 0, dc->RS.cmds = 0; } - pb_e = NULL; - if (gfx && !mo_ovr_on && pb_on()) { + dc->pb_e = NULL; + if (gfx && !dc->mo_ovr_on && pb_on()) { uint64_t g[PB_G]; pb_globals(ex, g); - pb_e = pb_entry(info, g); + dc->pb_e = pb_entry(info, g); } uint32_t n_sets = info->n_sets ? info->n_sets : 1; uint32_t lo[2] = {0, 0}, hi[2] = {0, 0}, d0[2] = {0, 0}; // bindings [lo, hi) and first descriptor per set @@ -3279,19 +3318,20 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { if (b->kind == VK_PIPE_SUPPORT) { float in[3]; support_in(ex, in); - if (!B.sup_valid || memcmp(in, B.sup_in, sizeof in)) { // support_fill reads only these + if (!dc->B.sup_valid || memcmp(in, dc->B.sup_in, sizeof in)) { // support_fill reads only these uint8_t tmp[SUPPORT_BYTES]; support_fill(tmp, in[0], in[1], in[2]); - memcpy(B.sup_in, in, sizeof in); - if (!B.sup_valid || memcmp(tmp, B.support, SUPPORT_BYTES)) { // a new buffer only when the contents changed + memcpy(dc->B.sup_in, in, sizeof in); + // a new buffer only when the contents changed + if (!dc->B.sup_valid || memcmp(tmp, dc->B.support, SUPPORT_BYTES)) { VkBuffer sb; VkDeviceSize so; memcpy(scratch(ex, SUPPORT_BYTES, &sb, &so), tmp, SUPPORT_BYTES); - memcpy(B.support, tmp, SUPPORT_BYTES); - B.sup_buf = sb, B.sup_off = so, B.sup_valid = 1; + memcpy(dc->B.support, tmp, SUPPORT_BYTES); + dc->B.sup_buf = sb, dc->B.sup_off = so, dc->B.sup_valid = 1; } } - o->buffer = (VkDescriptorBufferInfo){B.sup_buf, B.sup_off, SUPPORT_BYTES}; + o->buffer = (VkDescriptorBufferInfo){dc->B.sup_buf, dc->B.sup_off, SUPPORT_BYTES}; } else if (b->kind == GFX_RES_UBO) { uint64_t bt = bt_now(); for (uint32_t k = 0; k < cnt; k++) { @@ -3347,7 +3387,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } nd += cnt; } - pb_e = NULL; + dc->pb_e = NULL; uint64_t push_t0 = bt_now(); // An empty set 1 (a fragment stage without resources: depth and shadow draws) starts past the last binding, which // the loop never reaches: d0[1] stayed 0 and set 0 was pushed with 0 bytes, so the driver read stale descriptors @@ -3358,10 +3398,10 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { // layout lacks (false VUID-vkCmdDrawIndexed-None-08114, then an access violation inside the layer). A bind at set 1 // disturbs set 0 by its canonical id, which drops the layer's push set: the push after it gets a new one. int set1_first = 0; - if (gfx && info->layout != B.ds_layout) { // another layout: sets bound before may be disturbed - B.ds_layout = info->layout; - B.ds_bound[0] = B.ds_bound[1] = VK_NULL_HANDLE; - B.ds_pushed[0] = B.ds_pushed[1] = 0; + if (gfx && info->layout != dc->B.ds_layout) { // another layout: sets bound before may be disturbed + dc->B.ds_layout = info->layout; + dc->B.ds_bound[0] = dc->B.ds_bound[1] = VK_NULL_HANDLE; + dc->B.ds_pushed[0] = dc->B.ds_pushed[1] = 0; set1_first = n_sets == 2 && vk_ctx()->validation; } for (uint32_t k = 0; k < n_sets; k++) { @@ -3372,24 +3412,24 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { if (set1_first) { // bound only to drop the layer's push set (comment above) VkDescriptorSet set = alloc_set(ex, info->set_layout1); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 1, 1, &set, 0, NULL); - B.ds_bound[1] = set, B.ds_pushed[1] = 0; - RS.cmds++; + dc->B.ds_bound[1] = set, dc->B.ds_pushed[1] = 0; + dc->RS.cmds++; } continue; } if (s ? info->push1 : info->push) { uint32_t n = gfx && push_skip_on() ? dbytes : 0; - if (n && B.ds_pushed[s] && DS.pn[s] == n && !memcmp(DS.psig[s], ds, n)) { + if (n && dc->B.ds_pushed[s] && dc->DS.pn[s] == n && !memcmp(dc->DS.psig[s], ds, n)) { BS.push_skip++; continue; } if (info->tpl[s]) rec_push_tpl(vk_pipeline_push_tpl_fn(), ex->cb, info->tpl[s], info->layout, s, ds, dbytes); else rec_push(vk_pipeline_push_fn(), ex->cb, info->bind_point, info->layout, s, i1 - i0, w + i0); BS.push++; - RS.cmds++; + dc->RS.cmds++; if (gfx) { - B.ds_bound[s] = VK_NULL_HANDLE, B.ds_pushed[s] = n != 0; - if (n) DS.pn[s] = n, memcpy(DS.psig[s], ds, n); + dc->B.ds_bound[s] = VK_NULL_HANDLE, dc->B.ds_pushed[s] = n != 0; + if (n) dc->DS.pn[s] = n, memcpy(dc->DS.psig[s], ds, n); } } else if (!gfx || s > 1) { VkDescriptorSet set = alloc_set(ex, s ? info->set_layout1 : info->set_layout); @@ -3397,49 +3437,50 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL); } else { VkDescriptorSetLayout sl = s ? info->set_layout1 : info->set_layout; - VkDescriptorSet set = B.ds_last[s]; - if (!set || B.ds_last_layout[s] != sl || DS.n[s] != dbytes || memcmp(DS.sig[s], ds, dbytes)) { + VkDescriptorSet set = dc->B.ds_last[s]; + if (!set || dc->B.ds_last_layout[s] != sl || dc->DS.n[s] != dbytes || memcmp(dc->DS.sig[s], ds, dbytes)) { set = alloc_set(ex, sl); write_set(info, s, set, ds, w, i0, i1); - B.ds_last[s] = set, B.ds_last_layout[s] = sl, DS.n[s] = dbytes; - memcpy(DS.sig[s], ds, dbytes); + dc->B.ds_last[s] = set, dc->B.ds_last_layout[s] = sl, dc->DS.n[s] = dbytes; + memcpy(dc->DS.sig[s], ds, dbytes); BS.set_new++; - RS.cmds++; + dc->RS.cmds++; } else { BS.set_reuse++; } - if (B.ds_bound[s] != set) { + if (dc->B.ds_bound[s] != set) { vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, s, 1, &set, 0, NULL); - B.ds_bound[s] = set, B.ds_pushed[s] = 0; - RS.cmds++; + dc->B.ds_bound[s] = set, dc->B.ds_pushed[s] = 0; + dc->RS.cmds++; } } } if (info->pool) { // set 2: the texture and sampler pools pool_sync(ex); vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 2, 1, &PL.set, 0, NULL); - RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again + dc->RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again } if (skip) { - RS.rec = 0; + dc->RS.rec = 0; // An equal next draw records nothing when every push set holds its push (NX2NAVI_PUSH_SKIP on) and nothing made // the build unrepeatable; allocated sets are always bound to the one written last. - int valid = !RS.volatile_ && !RS.cnt_over && RS.hn; + int valid = !dc->RS.volatile_ && !dc->RS.cnt_over && dc->RS.hn; for (uint32_t s = 0; s < n_sets; s++) - if (hi[s] > lo[s] && (s ? info->push1 : info->push) && !B.ds_pushed[s]) valid = 0; + if (hi[s] > lo[s] && (s ? info->push1 : info->push) && !dc->B.ds_pushed[s]) valid = 0; if (skip == 2) { // with the pipeline info equal (bind_sig), the packed descriptors determine every write uint32_t n = (uint32_t)(nd * sizeof *d); - int wr = n != RS.ckn || memcmp(RS.ck, d, n); - int cn = RS.ncnt != RS.pncnt || memcmp(RS.cnt, RS.pcnt, sizeof RS.cnt[0] * (size_t)RS.ncnt); - if (same && was_valid && (wr || cn || RS.cmds)) + int wr = n != dc->RS.ckn || memcmp(dc->RS.ck, d, n); + int cn = dc->RS.ncnt != dc->RS.pncnt || + memcmp(dc->RS.cnt, dc->RS.pcnt, sizeof dc->RS.cnt[0] * (size_t)dc->RS.ncnt); + if (same && was_valid && (wr || cn || dc->RS.cmds)) rt_fatal("vk_exec: descriptor build skip check: a draw of program %llu with the inputs of the previous one " "gets %s", (unsigned long long)ex->prog->serial, - RS.cmds ? "descriptor commands" : wr ? "other writes" : "other census counts"); + dc->RS.cmds ? "descriptor commands" : wr ? "other writes" : "other census counts"); if (same && was_valid) BS.bind_skip++; // the draws a skip would have spared, compared instead - RS.ckn = n; - memcpy(RS.ck, d, n); + dc->RS.ckn = n; + memcpy(dc->RS.ck, d, n); } - B.res_valid = valid; + dc->B.res_valid = valid; } vk_exec_frame.bind_push_ns += (uint64_t)(bt_now() - push_t0); ex->recorded++; @@ -3449,11 +3490,11 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { static void bind_pipeline(VkExec *ex, VkPipelineBindPoint bp, VkPipeline p) { int k = bp == VK_PIPELINE_BIND_POINT_COMPUTE; - if (B.pipe[k] == p) return; + if (dc->B.pipe[k] == p) return; vkCmdBindPipeline(ex->cb, bp, p); ex->recorded++; - B.pipe[k] = p; - if (!k) B.st_valid = 0; + dc->B.pipe[k] = p; + if (!k) dc->B.st_valid = 0; } static VkViewport draw_viewport(VkExec *ex) { @@ -3511,6 +3552,7 @@ static void record_fixed(VkExec *ex) { } static void bind_vertex_buffers(VkExec *ex) { + CbState *held = &dc->B; VkBuffer vb[17]; VkDeviceSize vo[17], vsz[17], vstr[17]; int vcls[17] = {0}; @@ -3519,20 +3561,21 @@ static void bind_vertex_buffers(VkExec *ex) { // A stream whose range is the one its bound buffer was resolved from, with no pool or mirror change since // (vk_mem_gen, read before any resolve), resolves to that buffer again: most draws change one stream or none. uint64_t gen = vk_mem_gen(); - int same_gen = B.vb_valid && B.vgen == gen; + int same_gen = held->vb_valid && held->vgen == gen; // Most draws change no stream at all: with every range as it was bound and no pool or mirror change, the loop // below would take each buffer from B again and return at the compare after it. - if (same_gen && !memcmp(B.vin, ex->vbuf, sizeof B.vin)) { + if (same_gen && !memcmp(held->vin, ex->vbuf, sizeof held->vin)) { int same = 1; - for (int b = 0; !dyn && b < 16 && same; b++) same = B.vstr[b] == ex->pipe.vstream[b].stride; + for (int b = 0; !dyn && b < 16 && same; b++) same = held->vstr[b] == ex->pipe.vstream[b].stride; if (same) return; } for (int b = 0; b < 17; b++) { vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0; if (b == VK_PIPE_ZERO_BINDING) continue; const NVNbufferRange *r = &ex->vbuf[b]; - if (same_gen && B.vin[b].address == r->address && B.vin[b].size == r->size) { - vb[b] = B.vb[b], vo[b] = B.vo[b], vsz[b] = B.vsz[b], vcls[b] = B.vcls[b], vpool[b] = B.vpool[b]; + if (same_gen && held->vin[b].address == r->address && held->vin[b].size == r->size) { + vb[b] = held->vb[b], vo[b] = held->vo[b], vsz[b] = held->vsz[b]; + vcls[b] = held->vcls[b], vpool[b] = held->vpool[b]; continue; } VkMemSpan sp; @@ -3541,11 +3584,12 @@ static void bind_vertex_buffers(VkExec *ex) { uint64_t n = r->size && r->size < sp.avail ? r->size : sp.avail; vb[b] = sp.buffer, vo[b] = sp.offset, vsz[b] = n, vcls[b] = sp.cls, vpool[b] = sp.pool; } - memcpy(B.vcls, vcls, sizeof vcls), memcpy(B.vpool, vpool, sizeof vpool); - memcpy(B.vin, ex->vbuf, sizeof B.vin); - B.vgen = gen; - if (B.vb_valid && !memcmp(B.vb, vb, sizeof vb) && !memcmp(B.vo, vo, sizeof vo) && !memcmp(B.vsz, vsz, sizeof vsz) && - (dyn || !memcmp(B.vstr, vstr, sizeof vstr))) + memcpy(held->vcls, vcls, sizeof vcls), memcpy(held->vpool, vpool, sizeof vpool); + memcpy(held->vin, ex->vbuf, sizeof held->vin); + held->vgen = gen; + if (held->vb_valid && !memcmp(held->vb, vb, sizeof vb) && !memcmp(held->vo, vo, sizeof vo) && + !memcmp(held->vsz, vsz, sizeof vsz) && + (dyn || !memcmp(held->vstr, vstr, sizeof vstr))) return; vkCmdBindVertexBuffers2(ex->cb, 0, 17, vb, vo, vsz, dyn ? NULL : vstr); ex->recorded++; @@ -3554,8 +3598,9 @@ static void bind_vertex_buffers(VkExec *ex) { if (rt_trace & RT_TRACE_PERF) for (int b = 0; b < 17; b++) if (vpool[b]) vk_mem_bound_note(vpool[b]); - memcpy(B.vb, vb, sizeof vb), memcpy(B.vo, vo, sizeof vo), memcpy(B.vsz, vsz, sizeof vsz), memcpy(B.vstr, vstr, sizeof vstr); - B.vb_valid = 1; + memcpy(held->vb, vb, sizeof vb), memcpy(held->vo, vo, sizeof vo), memcpy(held->vsz, vsz, sizeof vsz); + memcpy(held->vstr, vstr, sizeof vstr); + held->vb_valid = 1; } // NX2NAVI_FOLIAGE_SHADOW_CULL=[f] (0 to 3, default 1; 3 turns it off): the foliage's mesh draws into the sun's shadow cascades @@ -3717,14 +3762,8 @@ static uint64_t *link_budget(void) { // texture type in answers_for, and stops on a difference. // tex_serial (bumped by every texture and image bind) equal to the key's means the handles were not rebound since // they were compared, so the 1 KiB compare is skipped; a rebind to the same handles still matches by the compare. -typedef struct { - const prog_t *prog; - uint64_t serial; - uint32_t gen, tex_serial; - uint64_t tex[2][32], img[2][32]; - VkPipeState pipe; -} MemoKey; -static struct { int on, valid, has_fs; MemoKey k; const GfxShader *vs, *fs; } SM = {-1}; +// The memo and its key (MemoKey) are DrawCtx.SM. +static int sm_on = -1; static void memo_key(const VkExec *ex, uint32_t gen, MemoKey *k) { k->prog = ex->prog, k->serial = ex->prog->serial, k->gen = gen, k->tex_serial = ex->tex_serial; @@ -3745,18 +3784,18 @@ static int memo_match(const VkExec *ex, uint32_t gen, MemoKey *k) { } static int shader_memo_on(void) { - if (SM.on < 0) { + if (sm_on < 0) { const char *e = cfg_value("NX2NAVI_SHADER_MEMO"); - SM.on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; + sm_on = !strcmp(e, "check") ? 2 : strcmp(e, "0") != 0; } - return SM.on; + return sm_on; } -static struct { int on, valid; VkPipeState s; const VkPipeInfo *info; } PM = {-1}; +static int pm_on = -1; // the memo is DrawCtx.PM static int pipe_memo_on(void) { - if (PM.on < 0) PM.on = cfg_bool("NX2NAVI_PIPE_MEMO"); - return PM.on; + if (pm_on < 0) pm_on = cfg_bool("NX2NAVI_PIPE_MEMO"); + return pm_on; } // Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped. @@ -3896,8 +3935,7 @@ static void prog_lead(const prog_t *p, int mesh) { // mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing // vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. 2: a mesh draw of -// the mesh fallback (mfb_draw): the vertex stage is mfb_cur_vs, in an ordinary pipeline without vertex input. -static const GfxShader *mfb_cur_vs; +// the mesh fallback (mfb_draw): the vertex stage is DrawCtx.mfb_cur_vs, in an ordinary pipeline without vertex input. static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) { int fb = mesh == 2; // from here on mesh says what the draw is, and !fb that its pipeline is a mesh pipeline @@ -3922,8 +3960,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int int has_ts = 0, ts_wait = 0, vst = mesh ? NVN_STAGE_MESH : NVN_STAGE_VS; const GfxShader *vs, *fs, *ts = NULL; uint32_t gen = memo ? vk_texture_slots_gen() : 0; // before shader_for, whose lookups may bump it - if (memo && SM.valid && memo_match(ex, gen, &SM.k)) { - vs = SM.vs, fs = SM.fs, has_vs = 1, has_fs = SM.has_fs; + if (memo && dc->SM.valid && memo_match(ex, gen, &dc->SM.k)) { + vs = dc->SM.vs, fs = dc->SM.fs, has_vs = 1, has_fs = dc->SM.has_fs; if (memo == 2) { int hv, hf, wv, wf; const GfxShader *v2 = shader_for(ex, NVN_STAGE_VS, &hv, &wv, 0), *f2 = shader_for(ex, NVN_STAGE_FS, &hf, &wf, 0); @@ -3931,13 +3969,13 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int rt_fatal("vk_exec: shader memo of program %llu differs from the lookup", (unsigned long long)ex->prog->serial); } } else { - SM.valid = 0; - if (fb) vs = mfb_cur_vs, has_vs = 1; + dc->SM.valid = 0; + if (fb) vs = dc->mfb_cur_vs, has_vs = 1; else vs = shader_for(ex, vst, &has_vs, &vs_wait, 0); if (mesh && !fb) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0); fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, 0); if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs)) - memo_key(ex, gen, &SM.k), SM.vs = vs, SM.fs = fs, SM.has_fs = has_fs, SM.valid = 1; + memo_key(ex, gen, &dc->SM.k), dc->SM.vs = vs, dc->SM.fs = fs, dc->SM.has_fs = has_fs, dc->SM.valid = 1; } if (!has_vs) return draw_skipped(ex, mesh, SK_NO_VERTEX_STAGE); // A draw into a target the previous frame did not render (or into none) waits for a pending shader or pipeline: @@ -4010,8 +4048,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int const char *why = NULL; uint64_t w0 = 0, m0 = 0; // m0: when the lookup missed (wait_note) int kind = WK_LINK; - if (pipe_memo_on() && PM.valid && !memcmp(&PM.s, &s, sizeof s)) { - info = PM.info, pending = 0; + if (pipe_memo_on() && dc->PM.valid && !memcmp(&dc->PM.s, &s, sizeof s)) { + info = dc->PM.info, pending = 0; p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); } else { // Nearly every draw finds its pipeline, so the first lookup waits for nothing and the clock is read only for a @@ -4060,8 +4098,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int pool_charge(last, b0); } if (m0) wait_note(kind, m0, pending), draw_stats_tick(); - PM.valid = !pending && p && info; - if (PM.valid) PM.s = s, PM.info = info; + dc->PM.valid = !pending && p && info; + if (dc->PM.valid) dc->PM.s = s, dc->PM.info = info; } uint64_t waited = w0 ? kern_now_ns() - w0 : 0; static int wait_logs; @@ -4079,7 +4117,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int if (!p || !info) return draw_skipped(ex, mesh, SK_PIPE_FAILED); if (info->pool && !pool_on()) return draw_skipped(ex, mesh, SK_POOL_OFF); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p); - if (!B.st_valid || memcmp(&B.st, &s, sizeof s)) { + if (!dc->B.st_valid || memcmp(&dc->B.st, &s, sizeof s)) { // Only the dynamic states whose values changed (vk_pipeline_set_dynamic_delta); NX2NAVI_DYN_DELTA=0 records // them all after every pipeline bind, as before. static int delta = -1; @@ -4087,11 +4125,11 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int delta = cfg_bool("NX2NAVI_DYN_DELTA"); if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)"); } - if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &B.dyn, mesh && !fb, + if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &dc->B.dyn, mesh && !fb, &vk_exec_frame.dyn_left); else vk_pipeline_set_dynamic(ex->cb, &s); ex->recorded++; - B.st = s, B.st_valid = 1; + dc->B.st = s, dc->B.st_valid = 1; if (ex->mark && vk_exec_frame.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; } if (!mesh) { @@ -4509,7 +4547,7 @@ static void mo_emit(VkCommandBuffer cb) { static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0); static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off) { - if (MO.failed || !vk_mvprev_on() || !B.st_valid) return; + if (MO.failed || !vk_mvprev_on() || !dc->B.st_valid) return; static int timed = -1; if (timed < 0) timed = rt_knob("MVOBJ_TIMES", "0"); // NX2NAVI_MVOBJ_TIMES=1: the summary's times (the clock reads cost) if (!timed) return mvobj_draw_(ex, d, ib, ib_off, 0); @@ -4523,13 +4561,13 @@ static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSiz static void mvobj_banks(VkExec *ex) { static int on = -1; if (on < 0) on = rt_knob("MVOBJ_BANKS", "0"); - if (!on || !B.st_valid) return; + if (!on || !dc->B.st_valid) return; uint64_t frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (frame < 2999 || frame > 3000) return; static struct { const GfxShader *vs; int bank; uint64_t frame; float w[64]; int n; } tab[4096]; static int logged; const GfxShaderRes *ur; - int nu = gfx_shader_resources(B.st.vs, GFX_RES_UBO, &ur); + int nu = gfx_shader_resources(dc->B.st.vs, GFX_RES_UBO, &ur); for (int i = 0; i < nu; i++) { int bank = ur[i].slot; if (bank < 3 || bank == 4 || bank == 5 || bank == 7 || bank > 16) continue; @@ -4537,16 +4575,17 @@ static void mvobj_banks(VkExec *ex) { uint32_t n = u->size < 256 ? (uint32_t)u->size / 4 : 64; const float *now = n ? (const float *)vk_usnap_bytes(u->addr, n * 4) : NULL; if (!now) continue; - uint32_t k = (uint32_t)(((uintptr_t)B.st.vs ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull >> 52); + uint32_t k = + (uint32_t)(((uintptr_t)dc->B.st.vs ^ (uint64_t)bank * 0x9E3779B97F4A7C15ull) * 0xff51afd7ed558ccdull >> 52); if (frame == 2999) { - tab[k].vs = B.st.vs, tab[k].bank = bank, tab[k].frame = frame, tab[k].n = (int)n; + tab[k].vs = dc->B.st.vs, tab[k].bank = bank, tab[k].frame = frame, tab[k].n = (int)n; memcpy(tab[k].w, now, n * 4); - } else if (tab[k].vs == B.st.vs && tab[k].bank == bank && tab[k].frame == 2999 && logged < 80) { + } else if (tab[k].vs == dc->B.st.vs && tab[k].bank == bank && tab[k].frame == 2999 && logged < 80) { char line[700]; int o = 0, nd = 0; for (int j = 0; j < tab[k].n && j < (int)n && o < 600; j++) if (memcmp(&tab[k].w[j], &now[j], 4)) nd++, o += snprintf(line + o, sizeof line - o, " [%d] %g>%g", j, tab[k].w[j], now[j]); - if (nd) logged++, rt_log("mvbanks: vs %p c%d size %#llx: %d words%s", (const void *)B.st.vs, bank, + if (nd) logged++, rt_log("mvbanks: vs %p c%d size %#llx: %d words%s", (const void *)dc->B.st.vs, bank, (unsigned long long)u->size, nd, line); tab[k].frame = 0; } @@ -4641,14 +4680,15 @@ static int mo_prev_cached(int index, const p_bind_buf *u, VkDescriptorBufferInfo static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSize ib_off, uint64_t t0) { mvobj_banks(ex); const VkExecTarget *z = &ex->depth; - if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || B.st.samples > 1 || B.st.raster_discard) + if (!z->image || z->width < DLAA_SCENE_MIN_W || z->format != VK_FORMAT_D32_SFLOAT || dc->B.st.samples > 1 || + dc->B.st.raster_discard) return; // solid geometry only, which tests and writes the scene depth: effects, particles and screen quads write none, and // drawn again they would put their motion on the surfaces behind them (DLSS takes those from the surface) - if (!B.st.ds.dt || !B.st.ds.dw) return; + if (!dc->B.st.ds.dt || !dc->B.st.ds.dw) return; // a vertex shader that reads a user bank besides the camera's (c4) may move on its own const GfxShaderRes *ur; - int nu = gfx_shader_resources(B.st.vs, GFX_RES_UBO, &ur), reads = 0; + int nu = gfx_shader_resources(dc->B.st.vs, GFX_RES_UBO, &ur), reads = 0; for (int i = 0; i < nu; i++) reads |= ur[i].slot >= 3 && ur[i].slot <= 16 && ur[i].slot != 4; if (!reads) return; MO.cand++; @@ -4660,20 +4700,20 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi // the variant: the translation of the same code and answers, kept per shader once found; it names the banks the // position depends on (0: none, the shader is skipped from then on) static struct { const GfxShader *of, *vs; uint32_t banks; } vmemo[1024]; - uint32_t vi = (uint32_t)((uintptr_t)B.st.vs * 0x9E3779B97F4A7C15ull >> 54); - if (vmemo[vi].of != B.st.vs) { + uint32_t vi = (uint32_t)((uintptr_t)dc->B.st.vs * 0x9E3779B97F4A7C15ull >> 54); + if (vmemo[vi].of != dc->B.st.vs) { int has, pending = 0; - sf_mvprev = 1; + dc->sf_mvprev = 1; const GfxShader *v = shader_for(ex, NVN_STAGE_VS, &has, &pending, 0); - sf_mvprev = 0; + dc->sf_mvprev = 0; if (!v || pending) { MO.building++; return; } GfxShaderIo io[32]; - uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(B.st.vs, io); + uint32_t extra = gfx_shader_outputs(v, io) & ~gfx_shader_outputs(dc->B.st.vs, io); uint32_t banks = extra ? gfx_shader_mvprev_banks(v) & 0x1fff8u : 0; - vmemo[vi].of = B.st.vs, vmemo[vi].vs = v, vmemo[vi].banks = banks; + vmemo[vi].of = dc->B.st.vs, vmemo[vi].vs = v, vmemo[vi].banks = banks; vk_mvprev_want((banks & ~(1u << 4)) >> 3); // binding index = bank - 3 } const GfxShader *vs = vmemo[vi].vs; @@ -4714,7 +4754,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi exact = e && *e ? strtol(e, NULL, 16) : -1; } if (exact >= 0 && banks != (uint32_t)exact) return; - mo_mask_note(banks, B.st.vs); + mo_mask_note(banks, dc->B.st.vs); // last frame's camera and blocks; held still when none of the position's blocks changed (most of the scenery) VkDescriptorBufferInfo cam, prevb[17]; float jit[2]; @@ -4753,7 +4793,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi } if (t0) MO.ns_part[1] += kern_now_ns() - tp, MO.n_part[1]++, tp = kern_now_ns(); GfxShaderIo io[32]; - uint32_t extra = gfx_shader_outputs(vs, io) & ~gfx_shader_outputs(B.st.vs, io); + uint32_t extra = gfx_shader_outputs(vs, io) & ~gfx_shader_outputs(dc->B.st.vs, io); const GfxShaderRes *r; if (gfx_shader_resources(vs, GFX_RES_IMG, &r)) { MO.storage++; @@ -4764,7 +4804,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi MO.no_fs++; return; } - VkPipeState ms = B.st; + VkPipeState ms = dc->B.st; ms.vs = vs, ms.fs = fs, ms.feedback = 0; memset(ms.color_fmt, 0, sizeof ms.color_fmt); ms.color_fmt[0] = NVN_FORMAT_RG16F; @@ -4776,7 +4816,7 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi // scene depth passes, anything nearer hides it (NX2NAVI_MVOBJ_EQUAL=1: EQUAL, for comparison) static int eq = -1; if (eq < 0) eq = rt_knob("MVOBJ_EQUAL", "0"); - int df = B.st.ds.df; + int df = dc->B.st.ds.df; df = eq ? NVN_DEPTH_FUNC_EQUAL : df == NVN_DEPTH_FUNC_LESS ? NVN_DEPTH_FUNC_LEQUAL : df == NVN_DEPTH_FUNC_GREATER ? NVN_DEPTH_FUNC_GEQUAL : df == NVN_DEPTH_FUNC_LEQUAL || df == NVN_DEPTH_FUNC_GEQUAL ? df : NVN_DEPTH_FUNC_EQUAL; @@ -4826,13 +4866,13 @@ static void mvobj_draw_(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSi MQ.par = (VkDescriptorBufferInfo){pbuf, poff, 16}; memcpy(MQ.par_in, par_in, sizeof par_in); } - memset(mo_ovr, 0, sizeof mo_ovr); + memset(dc->mo_ovr, 0, sizeof dc->mo_ovr); for (int bk = 3; bk <= 16; bk++) // last frame's block, or this frame's where none was found - mo_ovr[NVN_STAGE_VS][bk + 15] = bk == 4 ? cam : have >> bk & 1 ? prevb[bk] : ubo_info(ex, NVN_STAGE_VS, bk); - mo_ovr[NVN_STAGE_FS][3] = MQ.par; - mo_ovr_on = 1; + dc->mo_ovr[NVN_STAGE_VS][bk + 15] = bk == 4 ? cam : have >> bk & 1 ? prevb[bk] : ubo_info(ex, NVN_STAGE_VS, bk); + dc->mo_ovr[NVN_STAGE_FS][3] = MQ.par; + dc->mo_ovr_on = 1; int bound = mo_desc(ex, info, q); - mo_ovr_on = 0; + dc->mo_ovr_on = 0; if (!bound) { MO.storage++; return; @@ -5165,9 +5205,9 @@ static int mfb_dispatch(VkExec *ex, const GfxShader *cs, int stage, VkBuffer rec VkPipeline p = vk_pipeline_compute(cs, &info); if (!p || !info || (info->pool && !pool_on())) return 0; bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); - mfb_stage = stage; + dc->mfb_stage = stage; bind_resources(ex, info); - mfb_stage = 0; + dc->mfb_stage = 0; VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0); vkCmdDispatchIndirect(ex->cb, rec, at); vk_pass_mark_end(ex->cb, m); @@ -5223,7 +5263,7 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { if (!mfb_dispatch(ex, c->cs_task, NVN_STAGE_TASK, rec, at)) { draw_skipped(ex, 1, SK_PIPE_FAILED); return; } vk_exec_barrier(ex); vk_meshfb_scan(ex->cb, at, c->ti.payload_bytes / 4, c->ti.threads, slot_words, verts); - B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own + dc->B.pipe[1] = VK_NULL_HANDLE; // the scan bound its own ex->recorded += 4; vk_exec_barrier(ex); } @@ -5232,8 +5272,8 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { vk_meshfb_note(ex->cb, at); ex->recorded++; vk_exec_barrier(ex); - c0_binst = 0; - mfb_cur_vs = vs; + dc->c0_binst = 0; + dc->mfb_cur_vs = vs; int prim = c->mi.ipp == 1 ? NVN_DRAW_PRIMITIVE_POINTS : c->mi.ipp == 2 ? NVN_DRAW_PRIMITIVE_LINES : NVN_DRAW_PRIMITIVE_TRIANGLES; if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 2)) return; vk_exec_frame.draws++; @@ -5252,7 +5292,7 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { } PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); - c0_binst = 0; + dc->c0_binst = 0; if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first); vk_exec_frame.draws++; @@ -5417,7 +5457,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { if (!vk_mem_resolve(d->ind, 12, &ip) || ip.offset % 4) return warn_once(W_INDIRECT, "indirect mesh draw arguments at %#llx are not backed or not aligned: skipped", (unsigned long long)d->ind); - c0_binst = 0; + dc->c0_binst = 0; VK_MEM_USE(ip); if (fallback) { mfb_draw(ex, 0, &ip); @@ -5453,7 +5493,7 @@ static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType } static void op_draw(VkExec *ex, const p_draw *d) { - c0_binst = (uint32_t)d->base_instance; + dc->c0_binst = (uint32_t)d->base_instance; if (d->count <= 0 || !prepare_draw(ex, d->prim, d->count, 0)) return; vk_exec_frame.draws++; uint32_t inst = d->instances > 0 ? (uint32_t)d->instances : 1, first = (uint32_t)d->first, n = (uint32_t)d->count; @@ -5502,7 +5542,7 @@ static int restart_for(const VkExec *ex, int prim, int itype) { } static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { - c0_binst = (uint32_t)d->base_instance; + dc->c0_binst = (uint32_t)d->base_instance; static const uint32_t isize[3] = {1, 2, 4}; if (d->itype < 0 || d->itype > 2) return warn_once(W_INDEX, "index type %d is unknown: draw skipped", d->itype); int restart = restart_for(ex, d->prim, d->itype); @@ -5542,7 +5582,7 @@ static void op_draw_idx(VkExec *ex, const p_draw_idx *d) { // dispatch). The base instance stays out of c0 (0x644): c0 cannot be patched by a transfer // inside a pass; gl_BaseInstance of the Vulkan draw is the argument's. Byte indices need the count to be widened: skipped. static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) { - c0_binst = 0; + dc->c0_binst = 0; if (d->itype != NVN_INDEX_TYPE_UNSIGNED_SHORT && d->itype != NVN_INDEX_TYPE_UNSIGNED_INT) return warn_once(W_INDEX, "indirect draw with index type %d: skipped", d->itype); VkMemSpan ib, ip; @@ -5803,7 +5843,7 @@ static void untranslated_dispatch_fallback(VkExec *ex) { // push did not go through the recorder (vk_rec.h rec_push; PORT-HANDOFF 7.12 item 11), which it now does. VkImage tm = e5 && dst_view ? vk_tonemap_record(ex->cb, e5, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, dw, dh) : VK_NULL_HANDLE; if (tm) { // the tonemapped picture, already at the target's size, then the same blit as below - B.pipe[1] = VK_NULL_HANDLE; // the replay must bind its compute pipeline again + dc->B.pipe[1] = VK_NULL_HANDLE; // the replay must bind its compute pipeline again si = tm, sw = dw, sh = dh; bl.srcOffsets[1] = (VkOffset3D){(int32_t)sw, (int32_t)sh, 1}; } @@ -5891,7 +5931,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS vk_exec_barrier_dispatch(ex); // ends the pass; earlier writes are visible to the shader bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); - c0_dim[0] = (uint32_t)d->x, c0_dim[1] = (uint32_t)d->y, c0_dim[2] = (uint32_t)d->z; + dc->c0_dim[0] = (uint32_t)d->x, dc->c0_dim[1] = (uint32_t)d->y, dc->c0_dim[2] = (uint32_t)d->z; twin_dispatch = 1; { // the replay's binding path is the CPU cost the frame profile needs to separate from recording uint64_t bt = bt_now(); @@ -5927,8 +5967,8 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vk_exec_barrier_dispatch(ex); VK_MEM_USE(ip); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); - c0_dim[0] = c0_dim[1] = c0_dim[2] = 0; - B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads + dc->c0_dim[0] = dc->c0_dim[1] = dc->c0_dim[2] = 0; + dc->B.c0_valid[2] = 0; // a fresh c0 range: the copy below must not change one an earlier dispatch reads twin_dispatch = 1; { // the replay's binding path is the CPU cost the frame profile needs to separate from recording uint64_t bt = bt_now(); @@ -5937,17 +5977,17 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { vk_exec_frame.bind_calls++; } twin_dispatch = 0; - if (B.c0_valid[2]) { // the shader binds c0 + if (dc->B.c0_valid[2]) { // the shader binds c0 // NVN2 programs read the group counts from c0 byte 0 (c0_info); at 0x200 their bank holds pushed constants // or the driver's tables, which the copy would overwrite const uint8_t *pctl, *pcode; uint32_t plen; int nvn2 = find_stage(ex->prog, NVN_STAGE_CS, &pctl, &pcode, &plen) && ctl_nvn2(pctl); - VkBufferCopy cp = {ip.offset, B.c0_off[2] + (nvn2 ? 0 : 0x200), sizeof(VkDispatchIndirectCommand)}; - vkCmdCopyBuffer(ex->cb, ip.buffer, B.c0_buf[2], 1, &cp); + VkBufferCopy cp = {ip.offset, dc->B.c0_off[2] + (nvn2 ? 0 : 0x200), sizeof(VkDispatchIndirectCommand)}; + vkCmdCopyBuffer(ex->cb, ip.buffer, dc->B.c0_buf[2], 1, &cp); ex->recorded++; vk_exec_barrier(ex); - B.c0_valid[2] = 0; // the range no longer holds B.c0[2] + dc->B.c0_valid[2] = 0; // the range no longer holds B.c0[2] } VkPassMark *m = dispatch_mark(ex, cs, 0, 0, 0); vkCmdDispatchIndirect(ex->cb, ip.buffer, ip.offset); @@ -5995,7 +6035,7 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { case OP_BIND_PROGRAM: { const p_bind_program *b = payload; ex->prog = b->prog, ex->prog_stages = b->stages; - bind_gen++; + dc->bind_gen++; break; } case OP_BIND_STATE: { @@ -6044,7 +6084,7 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { rt_log("vk: uniform buffer bind to index %d (bank %d), stage %d, %llu bytes: beyond Maxwell's banks, dropped", b->index, b->index + 3, b->stage, (unsigned long long)b->size); } - bind_gen++; + dc->bind_gen++; break; } case OP_BIND_SSBO: { @@ -6057,14 +6097,14 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { rt_log("vk: storage buffer bind, stage %d slot %d: %#llx, %llu bytes", b->stage, b->index, (unsigned long long)b->addr, (unsigned long long)b->size); } - bind_gen++; + dc->bind_gen++; break; } case OP_BIND_TEX: { const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 8 ? ex->tex[b->stage] : NULL, 128, b); ex->tex_serial++; - bind_gen++; + dc->bind_gen++; break; } case OP_BIND_SAMP: { @@ -6076,15 +6116,14 @@ static void op_draw_impl(VkExec *ex, uint32_t op, const void *payload) { const p_bind_handles *b = payload; bind_handles(b->stage >= 0 && b->stage < 8 ? ex->img[b->stage] : NULL, 32, b); ex->tex_serial++; // the shader memo's handle check (feedback_scan reads only textures: a rescan at most) - bind_gen++; + dc->bind_gen++; break; } case OP_UPDATE_UB: op_update_ub(ex, payload); break; case OP_PUSH_CC: { const p_push_cc *p = payload; if (p->offset + (uint64_t)p->size <= sizeof ex->cs_push) memcpy(ex->cs_push + p->offset, p->data, p->size); - cs_push_bytes = p->offset + p->size; - bind_gen++; // storage buffers may come from these words (push_ptr) + dc->bind_gen++; // storage buffers may come from these words (push_ptr) break; } case OP_DRAW: op_draw(ex, payload); break;