mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 07:09:57 -04:00
Let the submitting thread go on past the replay workers' groups, with up to four groups in flight.
With NX2NAVI_REPLAY_PIPE=1 (off by default) the submitting thread hands a group's chunks to the workers, takes chunks of it itself until none is left to take, and goes on with the ops after the group in a new command buffer after the chunks' places in the part (vk_sync_append_later), also into the next group's front pass. Up to four groups are handed out and not in their part yet; their chunks wait in one queue in the order of the stream. The submitting thread waits for them only where the order needs it: for room, before a part goes to the GPU, before copies and texture or buffer clears, where DLAA's run or the contact shadows' hook acts, and where a lookup would make a texture entry or take back a retired one (the op goes again afterwards, as a chunk's does in turn). What the chunks read stays as the stream had it while the submitting thread runs ahead: uniform snapshot drops count from their place in the stream, a chunk reads only the inline uniform updates before its group, the HDR present's source goes by stream position, the texture pool set syncs under the chunks' lock, each group keeps its own render conditions, occlusion segments and closed query slots, and the queries after a group's last report go into the next segment made after it. A group's first chunk runs its pass hooks after the group before began its pass; turns wait for every earlier chunk, of earlier groups too. Locks the chunks take, the submitting thread takes while it is behind. Draws keep the sampled-texture descriptors they resolved per draw context and reuse them without the texture lock while their slot is unchanged; the workers waited for that lock at nearly every draw. Texture lookups that change nothing no longer wait for their turn. The perf line says where the submitting thread waited, what the chunks took, and which texture lookups waited; NX2NAVI_GPU_TIMELINE also shows the chunks and the waits. Off by default because it does not pay yet: in Hyrule Field with 3 workers it shortens the replay (11.5 against 12.1 ms a frame) but not the frame (49.2 against 49.6 fps in the same batch). The GPU gets a group's commands only once its last chunk is recorded, so the main thread's wait for the GPU after the frame's first submission grew by 0.8 ms. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -242,6 +242,21 @@ typedef struct {
|
||||
} MemoKey;
|
||||
|
||||
// ---- the recording context
|
||||
// bind_resources: the sampled-texture descriptors this context resolved, each with the generations and slot token it
|
||||
// was resolved under: while its slot, the submission and the samplers are as they were (slot_still, as the texture
|
||||
// descriptor memo TD checks, without the lock) it holds what TD would give, and a hit takes no texture lock. The
|
||||
// replay workers' chunks waited for that lock at nearly every draw: 6.1 s of waits a minute in bind_resources alone
|
||||
// (Hyrule Field, 3 workers, NX2NAVI_LOCK_TIME, 2026-11-12).
|
||||
#define TM_N 512
|
||||
typedef struct {
|
||||
uint64_t h, tok;
|
||||
uint32_t gen, ep, sg;
|
||||
uint16_t type;
|
||||
uint8_t separate, valid;
|
||||
VkImage image;
|
||||
VkDescriptorImageInfo out;
|
||||
} TexMemo;
|
||||
|
||||
// What the draws, binds and dispatches keep from one call to the next while they record a command buffer: the state
|
||||
// the command buffer holds (B), the signatures that let a draw skip its descriptor build or push (DS, RS, PB), the
|
||||
// previous draw's shaders and pipeline (SM, PM), the shader lookup's caches (AC, stagem, slotm) and the submission's
|
||||
@@ -280,6 +295,7 @@ typedef struct DrawCtx {
|
||||
struct { const void *d; uint64_t ind; VkDeviceSize at; } ma[64]; // mesh_ring_entry: entries filled ahead
|
||||
int ma_n, ma_i; // ... how many, and the next one a draw takes
|
||||
uint32_t ma_brk; // ... vk_exec_brk_now when they were filled
|
||||
TexMemo TM[TM_N]; // bind_resources: sampled-texture descriptors (tm_take)
|
||||
VkWriteDescriptorSet bind_w[MAX_BINDINGS]; // bind_resources: write structs, packed descriptors, texel views
|
||||
VkDescSlot bind_d[MAX_DESC];
|
||||
VkBufferView bind_bv[MAX_DESC];
|
||||
@@ -1083,7 +1099,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
if (serial) {
|
||||
uint64_t h = (serial * 0x9E3779B97F4A7C15ull ^ (uint64_t)st) ^ nvn_hm_hash(&a, sizeof a); // a is zero-padded
|
||||
set = shader_tab[(h ^ h >> 29) & (SHADER_SETS - 1)];
|
||||
int lk = vk_exec_in_chunk(); // only the chunks of a parallel group look up at once
|
||||
int lk = vk_exec_shared(); // only while a parallel group records do several threads look up at once
|
||||
if (lk) plat_mutex_lock(&shader_lock);
|
||||
for (int w = 0; w < SHADER_WAYS && set[w].used; w++) {
|
||||
ShaderEnt *e = &set[w];
|
||||
@@ -1111,7 +1127,7 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin
|
||||
if (!s) warn_once(W_SHADER, "a stage %d shader of program %llu failed to translate: draws with it are skipped", st,
|
||||
(unsigned long long)serial);
|
||||
else if (set) { // in front, the least recently used way drops out
|
||||
int lk = vk_exec_in_chunk();
|
||||
int lk = vk_exec_shared();
|
||||
if (lk) plat_mutex_lock(&shader_lock);
|
||||
memmove(&set[1], &set[0], (SHADER_WAYS - 1) * sizeof *set);
|
||||
set[0] = (ShaderEnt){.serial = serial, .stage = st, .used = 1, .a = a, .s = s};
|
||||
@@ -1377,7 +1393,8 @@ static void upd_note(const VkExec *ex, const p_update_ub *u) {
|
||||
// 1 when an update of this submission supplied [a, a+n).
|
||||
static int upd_read(const VkExec *ex, uint64_t a, void *out, size_t n) {
|
||||
if (UPD.fence != ex->fence) return 0;
|
||||
for (uint32_t i = UPD.n, end = UPD.n > UPD_N ? UPD.n - UPD_N : 0; i-- > end;) {
|
||||
uint32_t top = vk_exec_upd_top(UPD.n); // a replay worker's chunk: the updates before its group
|
||||
for (uint32_t i = top, end = top > UPD_N ? top - UPD_N : 0; i-- > end;) {
|
||||
const typeof(UPD.u[0]) *e = &UPD.u[i % UPD_N];
|
||||
if (a >= e->hi || a + n <= e->lo) continue;
|
||||
if (a < e->lo || a + n > e->hi) return 0; // partly covered: memory
|
||||
@@ -1787,10 +1804,14 @@ static struct {
|
||||
} TD[TD_N];
|
||||
|
||||
static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate,
|
||||
VkDescriptorImageInfo *out) {
|
||||
VkDescriptorImageInfo *out, VkImage *img_out) {
|
||||
int td_on = td_mode();
|
||||
VkImage image;
|
||||
if (!h || storage || !td_on) return (void)tex_info_full(ex, r, h, storage, separate, out, &image);
|
||||
VkImage image = VK_NULL_HANDLE;
|
||||
if (!h || storage || !td_on) {
|
||||
tex_info_full(ex, r, h, storage, separate, out, &image);
|
||||
if (img_out) *img_out = image;
|
||||
return;
|
||||
}
|
||||
uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); // before any lookup
|
||||
typeof(TD[0]) *m = &TD[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 7 ^ (uint64_t)separate) >> 51];
|
||||
if (m->valid && m->h == h && m->type == r->type && m->separate == separate && m->sgen == sg &&
|
||||
@@ -1802,6 +1823,7 @@ static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int s
|
||||
*out = m->out;
|
||||
out->imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image) ? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT
|
||||
: VK_IMAGE_LAYOUT_GENERAL;
|
||||
if (img_out) *img_out = m->image;
|
||||
if (td_on == 2) {
|
||||
VkDescriptorImageInfo full;
|
||||
tex_info_full(ex, r, h, storage, separate, &full, &image);
|
||||
@@ -1817,6 +1839,7 @@ static void tex_info_lookup(VkExec *ex, const GfxShaderRes *r, uint64_t h, int s
|
||||
// entry never matches, and the next binding fills it
|
||||
*m = (typeof(*m)){.h = h, .tok = tok, .gen = gen, .epoch = ep, .sgen = sg, .type = r->type,
|
||||
.separate = (uint8_t)separate, .valid = 1, .reduced = (uint8_t)reduced, .image = image, .out = *out};
|
||||
if (img_out) *img_out = image;
|
||||
}
|
||||
|
||||
// Returns whether the image bound was made without its top mip levels (texture detail, VkTexImage.skip).
|
||||
@@ -2470,9 +2493,16 @@ static void pool_sync(VkExec *ex);
|
||||
|
||||
// A chunk of a parallel group (vk_exec.c "replay workers") binds the set as vk_exec_draw_group_begin left it, up to date
|
||||
// when the group began; the chunk that takes this submission's first set takes it in turn, under pool_mt. Elsewhere the
|
||||
// replay syncs the pools at every draw that binds them, as before.
|
||||
// replay syncs the pools at every draw that binds them, as before, under pool_mt past a group whose chunks still record
|
||||
// (NX2NAVI_REPLAY_PIPE): the set is the submission's, read by the GPU when the part runs.
|
||||
static void pool_sync_draw(VkExec *ex) {
|
||||
if (!vk_exec_in_chunk()) return pool_sync(ex);
|
||||
if (!vk_exec_shared()) return pool_sync(ex);
|
||||
if (!vk_exec_in_chunk()) {
|
||||
plat_mutex_lock(&pool_mt);
|
||||
pool_sync(ex);
|
||||
plat_mutex_unlock(&pool_mt);
|
||||
return;
|
||||
}
|
||||
if (__atomic_load_n(&PL.cur_fence, __ATOMIC_ACQUIRE) == ex->fence) return;
|
||||
vk_exec_turn();
|
||||
plat_mutex_lock(&pool_mt);
|
||||
@@ -2482,11 +2512,17 @@ static void pool_sync_draw(VkExec *ex) {
|
||||
|
||||
static int mesh_ring_make(void);
|
||||
|
||||
void vk_exec_draw_group_begin(VkExec *ex) {
|
||||
uint32_t vk_exec_draw_group_begin(VkExec *ex) {
|
||||
dummies_init();
|
||||
if (vk_ctx()->has_mesh_shader && !vk_ctx()->mesh_fallback) mesh_ring_make(); // the chunks' indirect mesh draws
|
||||
if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
|
||||
if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) pool_sync(ex); // the set this submission took
|
||||
if (pool_on() && PL.cur_fence == ex->fence && !PL.sealed) { // the set this submission took
|
||||
int lk = vk_exec_shared(); // earlier groups may still record (NX2NAVI_REPLAY_PIPE)
|
||||
if (lk) plat_mutex_lock(&pool_mt);
|
||||
pool_sync(ex);
|
||||
if (lk) plat_mutex_unlock(&pool_mt);
|
||||
}
|
||||
return UPD.fence == ex->fence ? UPD.n : 0;
|
||||
}
|
||||
|
||||
static void pool_sync(VkExec *ex) {
|
||||
@@ -2749,7 +2785,8 @@ static void pool_sync_entries(VkExec *ex) {
|
||||
typedef struct { VkImage src, img; VmaAllocation mem; VkImageView view[2]; uint32_t w, h, layers; VkFormat fmt; } Twin;
|
||||
static Twin twins[TWINS];
|
||||
static unsigned twin_next;
|
||||
static int twin_dispatch; // op_dispatch*: the resources being bound are a dispatch's
|
||||
// op_dispatch*: the resources being bound are a dispatch's, on this thread (the replay workers record draws meanwhile)
|
||||
static _Thread_local int twin_dispatch;
|
||||
static VkBuffer twin_buf;
|
||||
static VmaAllocation twin_buf_mem;
|
||||
static VkDeviceSize twin_buf_cap;
|
||||
@@ -2899,8 +2936,9 @@ static Twin *twin_update(VkExec *ex, const VkTexImage *img) {
|
||||
// The image descriptor of slot h; in a dispatch, a compare-sampled colour image the driver cannot compare is swapped
|
||||
// for its depth twin; in a pass larger than it, the scene's light target, once DLSS scaled it this frame, for DLSS's
|
||||
// output (vk_dlaa.c).
|
||||
static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out) {
|
||||
tex_info_lookup(ex, r, h, storage, separate, out);
|
||||
static void tex_info(VkExec *ex, const GfxShaderRes *r, uint64_t h, int storage, int separate, VkDescriptorImageInfo *out,
|
||||
VkImage *img_out) {
|
||||
tex_info_lookup(ex, r, h, storage, separate, out, img_out);
|
||||
if (h && !storage && vk_dlaa_swap_img && ex->in_pass && ex->pass_w > vk_dlaa_swap_img->width + 1 &&
|
||||
vk_texture_by_id_locked((int)(h & 0xfffff), NULL, NULL) == vk_dlaa_swap_img)
|
||||
out->imageView = vk_dlaa_swap_view, out->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
|
||||
@@ -3116,7 +3154,7 @@ static void rs_add(RsRange *r, uint32_t *n, const VkExec *ex, const void *at, si
|
||||
static const RsList *rs_list(const VkExec *ex, const VkPipeInfo *info) {
|
||||
DrawCtx *const cx = ex->dc;
|
||||
if (info == cx->rs_last) return cx->rs_last_l;
|
||||
int lk = vk_exec_in_chunk();
|
||||
int lk = vk_exec_shared();
|
||||
if (lk) plat_mutex_lock(&rs_lock);
|
||||
if (!rs_lists.klen) nvn_hm_init(&rs_lists, sizeof info);
|
||||
RsList *l = nvn_hm_get(&rs_lists, &info);
|
||||
@@ -3458,15 +3496,36 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
|
||||
} else {
|
||||
kind = 1;
|
||||
int locked = 0; // tex_info looks up with the texture lock held; kept descriptors (pb_take) need none
|
||||
// the context's memo (TexMemo) for sampled textures while other threads record (the lock is free
|
||||
// otherwise), outside a dispatch's depth twins and DLSS's stand-in image, with the generations read
|
||||
// before any lookup below
|
||||
int tm = !img && !r->exact && td_mode() == 1 && !vk_dlaa_swap_img && !twin_dispatch && vk_exec_shared();
|
||||
uint32_t tg = tm ? vk_texture_slots_gen() : 0, te = tm ? vk_texture_epoch() : 0;
|
||||
uint32_t ts = tm ? vk_sampler_gen() : 0;
|
||||
for (uint32_t k = 0; k < cnt; k++) {
|
||||
if (pb_take(ex, b, nd + k, k, &o[k])) continue;
|
||||
if (!locked) vk_texture_lock(), locked = 1;
|
||||
int slot = r->slot + (int)k;
|
||||
uint64_t h = 0;
|
||||
if (img) h = slot >= 0 && slot < 32 ? ex->img[st][slot] : 0;
|
||||
else h = slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0;
|
||||
uint64_t tt = bt_now();
|
||||
tex_info(ex, r, h, img, r->separate, &o[k].image);
|
||||
TexMemo *m = tm && h ? &cx->TM[(h * 0x9E3779B97F4A7C15ull ^ (uint64_t)r->type << 9 ^
|
||||
(uint64_t)r->separate) >> 55 & (TM_N - 1)] : NULL;
|
||||
if (m && m->valid && m->h == h && m->type == r->type && m->separate == (uint8_t)r->separate &&
|
||||
m->ep == te && m->sg == ts && slot_still(&m->gen, m->tok, h, tg)) {
|
||||
o[k].image = m->out;
|
||||
o[k].image.imageLayout = m->image && ex->nfbl && in_fbl(ex, m->image)
|
||||
? VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT
|
||||
: VK_IMAGE_LAYOUT_GENERAL;
|
||||
} else {
|
||||
if (!locked) vk_texture_lock(), locked = 1;
|
||||
VkImage ti = VK_NULL_HANDLE;
|
||||
uint64_t tok = m ? token_of(h) : 0; // before the lookup
|
||||
tex_info(ex, r, h, img, r->separate, &o[k].image, &ti);
|
||||
if (m) *m = (TexMemo){.h = h, .tok = tok, .gen = tg, .ep = te, .sg = ts, .type = r->type,
|
||||
.separate = (uint8_t)r->separate, .valid = 1, .image = ti,
|
||||
.out = o[k].image};
|
||||
}
|
||||
vk_exec_frame.bind_tex_ns += (uint64_t)(bt_now() - tt);
|
||||
o[k].w[2] = (uint32_t)o[k].image.imageLayout; // zero the tail padding
|
||||
pb_keep(cx, nd + k, &o[k]);
|
||||
@@ -4526,7 +4585,7 @@ static int mo_desc(VkExec *ex, const VkPipeInfo *info, MoDraw *q) {
|
||||
vk_texture_lock();
|
||||
for (uint32_t k = 0; k < cnt; k++) {
|
||||
int slot = r->slot + (int)k;
|
||||
tex_info(ex, r, slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0, 0, r->separate, &o[k].image);
|
||||
tex_info(ex, r, slot >= 0 && slot < 128 ? ex->tex[st][slot] : 0, 0, r->separate, &o[k].image, NULL);
|
||||
}
|
||||
vk_texture_unlock();
|
||||
} else {
|
||||
@@ -4667,7 +4726,7 @@ static void mvobj_draw(VkExec *ex, const p_draw_idx *d, VkBuffer ib, VkDeviceSiz
|
||||
const GfxShader *vs;
|
||||
uint32_t banks;
|
||||
int listed = mvobj_pick(ex, &vs, &banks, t0);
|
||||
int lk = listed && vk_exec_in_chunk();
|
||||
int lk = listed && vk_exec_shared();
|
||||
if (lk) plat_mutex_lock(&mo_lock);
|
||||
if (listed) mvobj_draw_(ex, d, ib, ib_off, t0, vs, banks);
|
||||
if (t0) __atomic_add_fetch(&MO.ns_check, kern_now_ns() - t0, __ATOMIC_RELAXED);
|
||||
@@ -6240,6 +6299,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) {
|
||||
vk_exec_frame.bind_calls++;
|
||||
}
|
||||
twin_dispatch = 0;
|
||||
if (vk_texture_deferred()) return; // past a parallel group: the op goes again once it is done (vk_exec.c op_again)
|
||||
VkPassMark *m = dispatch_mark(ex, cs, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z);
|
||||
vkCmdDispatch(ex->cb, (uint32_t)d->x, (uint32_t)d->y, (uint32_t)d->z);
|
||||
vk_pass_mark_end(ex->cb, m);
|
||||
@@ -6279,6 +6339,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) {
|
||||
vk_exec_frame.bind_calls++;
|
||||
}
|
||||
twin_dispatch = 0;
|
||||
if (vk_texture_deferred()) return; // as in op_dispatch
|
||||
if (cx->B.c0_valid[2]) { // the shader binds c0
|
||||
// NVN2 programs read the group counts from c0 byte 0 (c0_info); at 0x200 their bank holds pushed constants
|
||||
// or the driver's tables, which the copy would overwrite
|
||||
|
||||
Reference in New Issue
Block a user