From 83faa1daca494046c92a3c79994fcf7bcb8d9d0e Mon Sep 17 00:00:00 2001 From: chrg6 Date: Wed, 11 Nov 2026 07:37:38 -0500 Subject: [PATCH] Let a pool's finalize wait for the submissions issued so far instead of the idle queue. The finalize of a used pool flushed and waited for every upload batch and then for the idle queue. During an area load the queue also holds the load's texture uploads, which name no pool's storage: entering Kokiri Forest, one finalize waited 418 ms and the frame took 462 ms. The game finalizes a pool once its own work with it is done, so the finalize now waits for the submissions issued so far and returns; the pool's buffer and memory go once the upload batches handed out by then and the submission being recorded are through. NX2NAVI_POOL_FINI_IDLE=1 keeps the old wait. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 6 ++-- runtime/vk_memory.c | 86 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index db7da005..77b3946f 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -73,7 +73,7 @@ static const char *const known[] = { "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", - "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VA_REPORT", + "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_FINI_IDLE", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VA_REPORT", "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "COOP", "COOP_NAME", "COOP_SERVER", "COOP_PORT", "COOP_ROOM", "COOP_AUTO", "COOP_DROP", // co-op (coop.c); the last three have no menu row "COOP_PROGRESS", "COOP_ITEMS", "COOP_SONGS", "COOP_QUEST", "COOP_KEYS", "COOP_RUPEES", "COOP_AMMO", // the room's rules (coop_game.c) @@ -128,7 +128,7 @@ static const struct { const char *key, *section; } sections[] = { {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, {"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, {"MEM_NEED_MB", "diagnostics"}, {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, - {"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, + {"POOL_CREATE_MS", "performance"}, {"POOL_FINI_IDLE", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"PIPE_FRAME_WAIT_MS", "performance"}, {"PIPE_SKIP_STEADY", "performance"}, {"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"}, @@ -258,7 +258,7 @@ static const CfgMeta metas[] = { {"DYN_RES", "0", "0|1", LIVE}, {"DYN_RES_TARGET", "16.7", "1..100", LIVE}, {"DYN_RES_STEP", "45", "8..600", LIVE}, // the recording thread: a core of its own (auto: with 12 or more logical processors) and the highest priority // (vk_exec.c rec_claim) - {"RECORDER_CORE", "auto", "auto|0|1", RESTART}, {"RECORDER_PRIO", "1", "0|1", RESTART}, {"REPLAY_TIME", "0", "0|1", RESTART}, {"LOCK_TIME", "0", "0|1", RESTART}, {"POOL_CREATE_BUDGET", "16", "1..4096", RESTART}, {"POOL_CREATE_MS", "6", "0..100", RESTART}, {"POOL_LOAD_BACKLOG", "512", "0..65536", RESTART}, {"POOL_LOAD_MS", "50", "1..1000", RESTART}, + {"RECORDER_CORE", "auto", "auto|0|1", RESTART}, {"RECORDER_PRIO", "1", "0|1", RESTART}, {"REPLAY_TIME", "0", "0|1", RESTART}, {"LOCK_TIME", "0", "0|1", RESTART}, {"POOL_CREATE_BUDGET", "16", "1..4096", RESTART}, {"POOL_CREATE_MS", "6", "0..100", RESTART}, {"POOL_FINI_IDLE", "0", "0|1", RESTART}, {"POOL_LOAD_BACKLOG", "512", "0..65536", RESTART}, {"POOL_LOAD_MS", "50", "1..1000", RESTART}, // held submissions handed to the queue together (vk_sync.c, submit batching) {"SUBMIT_BATCH", "4", "1..16", RESTART}, {"CACHE_DIE", "1", "0|1|ab", RESTART}, {"ONE_PER_CORE", "0", "0|1|ab", RESTART}, // perf.c cpu_placement diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 9018acd6..eb7d643e 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -115,6 +115,9 @@ static void shadow_forget(PoolRec *r); static void pool_retiring(PoolRec *r); // pool_fini: kept until pool_retire (or vk_mem_shutdown) static void shadow_table_add(PoolRec *r); static void pool_retire(void *arg); // completion thread: destroys a finalized pool's record +static int pool_fini_idle(void); +static void pool_fini_wait(void); +static void pool_wait_add(PoolRec *r); static void grow(void **arr, int *cap, size_t elem); static PlatMutex pool_lock = PLAT_MUTEX_INIT; @@ -395,13 +398,24 @@ static void pool_fini(pool_t *p) { // Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the // guest gets the storage back when this returns. vk_texture_range_gone(r->lo, r->size, r->alloc); + int narrow = 0; if (r->used && r->kind != POOL_VIRT) { - vk_upload_flush(1); - vk_queue_lock(); - vkQueueWaitIdle(c->gfx_queue); - vk_queue_unlock(); + if (pool_fini_idle()) { + vk_upload_flush(1); + vk_queue_lock(); + vkQueueWaitIdle(c->gfx_queue); + vk_queue_unlock(); + } else { + narrow = 1; + pool_fini_wait(); + } } vk_mem_collect(); + if (narrow) { // the Vulkan objects outlive the upload batches and the submission being recorded (pool_wait_add) + pool_retiring(r); + pool_wait_add(r); + return; + } // A record that waits for the next submission is on the retiring list until its callback runs: when no // submission follows (a pool freed at the end), vk_mem_shutdown destroys it. Left to the callback alone, a // shadow copy's buffer was still allocated when the allocator went, which stops the process there. @@ -3963,10 +3977,73 @@ static void destroy_dead(VkCtx *c, const Dead *d) { // The objects the GPU is done with: destroyed here (out != NULL: moved to *out, for the caller to destroy without // cmd_lock; *nout of them). +// ---- The finalize of a used pool (pool_fini) +// The finalize returned only once the queue was idle and every upload batch was through (vkQueueWaitIdle), since the +// guest gets the storage back when it returns. During an area load the queue also holds the load's texture uploads, +// which name no pool's storage: entering Kokiri Forest, one finalize waited 418 ms for them, the frame took 462 ms. +// The game finalizes a pool once its own work with it is done, as on the console, so the finalize now waits for the +// submissions issued so far (the game's work, the mirror copies and write-backs in it) and returns. The pool's buffer +// and memory, which images cut from it (vk_texture_range_gone) and the upload batches open or in flight may still +// name, go later: once those batches and the submission being recorded are through (pool_wait_collect, from the +// upload collector). NX2NAVI_POOL_FINI_IDLE=1 waits for the idle queue as before. +typedef struct { PoolRec *r; uint64_t seq, tl; } PoolWait; +static PoolWait *pwait; // retire_lock +static int npwait, cpwait; +static uint64_t pf_n, pf_ns, pf_max_ns, pf_logged; // narrowed finalizes, their waits (any thread; statistics only) + +static int pool_fini_idle(void) { + static int on = -1; + if (on < 0) { + on = rt_knob("POOL_FINI_IDLE", "0") == 1; + rt_log("vk: a pool's finalize waits for %s (NX2NAVI_POOL_FINI_IDLE=%d)", + on ? "the idle queue, the upload batches included" : "the submissions issued so far", on); + } + return on; +} + +static void pool_fini_wait(void) { + uint64_t t0 = kern_now_ns(), v = vk_sync_issued(); + vk_sync_fence_wait(v, UINT64_MAX); + uint64_t d = kern_now_ns() - t0; + uint64_t n = __atomic_add_fetch(&pf_n, 1, __ATOMIC_RELAXED); + __atomic_add_fetch(&pf_ns, d, __ATOMIC_RELAXED); + if (d > __atomic_load_n(&pf_max_ns, __ATOMIC_RELAXED)) __atomic_store_n(&pf_max_ns, d, __ATOMIC_RELAXED); + if (d > 20000000ull && __atomic_add_fetch(&pf_logged, 1, __ATOMIC_RELAXED) <= 20) + rt_log("vk: a pool's finalize waited %.1f ms for the submissions up to %llu (finalizes so far %llu, %.1f ms in all)", + d / 1e6, (unsigned long long)v, (unsigned long long)n, __atomic_load_n(&pf_ns, __ATOMIC_RELAXED) / 1e6); + else if (n % 256 == 0) + rt_log("vk: pool finalizes that waited for the GPU: %llu, %.1f ms in all, the longest %.1f ms", + (unsigned long long)n, __atomic_load_n(&pf_ns, __ATOMIC_RELAXED) / 1e6, + __atomic_load_n(&pf_max_ns, __ATOMIC_RELAXED) / 1e6); +} + +static void pool_wait_add(PoolRec *r) { + // every upload batch handed out so far, and the submission another thread may be recording (vk_sync_submit sets + // issued after it), as defer stamps a deferred object + PoolWait w = {r, __atomic_load_n(&next_seq, __ATOMIC_ACQUIRE), vk_sync_issued() + 1}; + plat_mutex_lock(&retire_lock); + if (npwait == cpwait) grow((void **)&pwait, &cpwait, sizeof *pwait); + pwait[npwait++] = w; + plat_mutex_unlock(&retire_lock); +} + +// cmd_lock held (collect_locked_out): the waiting pools whose batches and submission are through go now. +static void pool_wait_collect(uint64_t tl) { + PoolRec *go[64]; + int n = 0; + plat_mutex_lock(&retire_lock); + for (int i = 0; i < npwait && n < 64;) + if (pwait[i].seq <= done_seq && pwait[i].tl <= tl) go[n++] = pwait[i].r, pwait[i] = pwait[--npwait]; + else i++; + plat_mutex_unlock(&retire_lock); + for (int i = 0; i < n; i++) pool_retire(go[i]); // off the retiring list, then destroyed +} + static void collect_locked_out(VkCtx *c, Dead **out, int *nout, int *cout) { retire(c, 0); uint64_t tl = 0; if (c->timeline) vkGetSemaphoreCounterValue(c->device, c->timeline, &tl); + if (npwait) pool_wait_collect(tl); int k = 0; for (int i = 0; i < ndead; i++) { Dead *d = &dead[i]; @@ -4304,6 +4381,7 @@ void vk_mem_shutdown(void) { plat_mutex_unlock(&shadow_lock); for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran nretiring = 0; + npwait = 0; // those waiting for pool_wait_collect were on the retiring list too __atomic_add_fetch(&pool_gen, 1, __ATOMIC_RELEASE); nphys = 0; plat_mutex_unlock(&pool_lock);