diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 1c77e8f0..fc2685dd 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -162,9 +162,9 @@ # made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay. # 2 (diagnosis): the same marks, and such textures give way to stand-ins in the texture pool, but none is freed. # tex_disuse = 1 -# 1: a texture the game releases and takes straight back, with nothing else changed at its address, leaves the texture -# table's slots there as they were, instead of having them all looked up again (large towns release and take back -# thousands a second). 0: every release and take-back counts as a change, as before. +# 1: a texture the game releases or takes back, with nothing else changed at its address, puts the texture table's +# slots there back in a state they were in before, and they are not all looked up again (large towns release and take +# back thousands a second). 0: every release and take-back counts as a new state. # tex_gen_undo = 1 # Host zstd decoder in place of the game's (0 = the game's), and a check that decodes every call both ways and logs # the differences (1 = on; slow). Ocarina of Time 1.1.1 keeps the game's decoder while zstd_native is unset: its host @@ -598,6 +598,10 @@ # are made inaccessible until their first read or write, which is logged with its instruction ("mem: write-back # probe"). The bytes are there all along; the probe only watches, at the cost of the faults it takes. # wb_probe = 0 +# 1: what the GPU wrote goes back to the game's memory in a GPU submission of its own, and the wait after each frame's +# first submission ends when that submission's passes are through, without waiting for the copy back (3 to 4 ms of +# every frame). 0: the copy back is part of the submission, and the wait covers it. +# wb_split = 1 # 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every # page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in # Kokiri Forest. diff --git a/runtime/cfg.c b/runtime/cfg.c index c2f7b5ad..34ce4563 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -62,7 +62,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", @@ -119,7 +119,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; @@ -232,7 +232,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/gfx.h b/runtime/gfx.h index 2a1b7778..762404de 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -145,6 +145,9 @@ typedef struct { uint64_t (*submit)(const uint64_t *handles, int n); // replays; returns this submission's fence value // (gfx_submit_queue names the game's queue) int (*fence_wait)(uint64_t value, uint64_t timeout_ns); // 1 reached; caller has given its core up + // As fence_wait, but only until the GPU has executed the submission's own commands: its write-backs to guest + // memory and its fence's side effects may follow (vk_sync.c exec_tl). The same as fence_wait where there is no split. + int (*exec_wait)(uint64_t value, uint64_t timeout_ns); uint64_t (*fence_done)(void); // highest value whose side effects are written // Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that // has passed or at_ns is 0. diff --git a/runtime/gfx_null.c b/runtime/gfx_null.c index dd01f281..b73010a4 100644 --- a/runtime/gfx_null.c +++ b/runtime/gfx_null.c @@ -146,6 +146,7 @@ const GfxBackend gfx_null_backend = { .program_release = null_program_release, .submit = null_submit, .fence_wait = null_fence_wait, + .exec_wait = null_fence_wait, .fence_done = null_fence_done, .present = null_present, .finish = null_finish, diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 28aeeb1b..73f5d213 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -660,6 +660,9 @@ NVN_API(nvnQueueSubmitCommands) { // point). Copies of the data at submit made it worse, a mix of two frames' versions. // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits // after every submission, 0 never. + // The wait is for the submission's passes, which is what the rewrite races with: exec_wait leaves out the + // write-backs of what they wrote (vk_sync.c exec_tl), which nothing here needs before the game goes on. The fences + // the game itself waits on cover them as before. static long long finish_mask = -2; if (finish_mask == -2) { const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH"); @@ -667,7 +670,7 @@ NVN_API(nvnQueueSubmitCommands) { if (e && *e && finish_mask == 1 && !(e[0] == '0' && (e[1] == 'x' || e[1] == 'X'))) finish_mask = -1; // "1": all } int idx = nvn_submits_in_frame++; - if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->fence_wait(f, UINT64_MAX); + if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->exec_wait(f, UINT64_MAX); } // The producer (1.2.1 main f_70008c6460) releases the read lock on obj+0x538 at lr 0x70008c70c8; when diff --git a/runtime/vk.h b/runtime/vk.h index 4ca82456..a124ee7f 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -570,6 +570,11 @@ void vk_sync_start(void); // vk_init (fatal without a device) a uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted +// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet. +int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns); +int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own +VkCommandBuffer vk_sync_split_exec(void); // vk_sync_split for the part before a submission's write-backs +int vk_mem_mirror_wb_pending(void); // vk_memory.c: vk_mem_mirror_end has write-backs to record uint64_t vk_sync_fence_done(void); // highest done value uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1) void vk_sync_finish(void); // every submitted value done diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index f5389dfd..a2e715c8 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -183,6 +183,11 @@ int vk_replay_async(void) { return on; } +// With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's. +static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { + return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns); +} + // The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end. int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); } @@ -606,6 +611,7 @@ const GfxBackend gfx_vulkan_backend = { .program_release = vk_be_program_release, .submit = vk_be_submit, .fence_wait = vk_sync_fence_wait, + .exec_wait = vk_be_exec_wait, .fence_done = vk_sync_fence_done, .present = vk_be_present, .frame_start = vk_dlssg_frame_start, diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index bd891bb2..c2f056ae 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1325,6 +1325,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_pass_end(ex); x_here = 0; if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes + // The write-backs in a part of their own, after one that signals the executed timeline (vk_sync.c exec_tl): the + // wait after a frame's first submission then ends when the submission's passes are through, not its write-backs. + if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) { + ex->cb = vk_sync_split_exec(); + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + } if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory if (ex->nviews) { ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ced07ef4..38cfaf51 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2331,6 +2331,8 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) { return j->n + 2; } +int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); } + int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { if (!vk_mirror_on) return 0; int rec = 0; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index cae29b16..6d9e1176 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -62,7 +62,7 @@ static struct { int n; VkSubmitInfo2 si[HOLD_CAP]; VkCommandBufferSubmitInfo cbi[HOLD_CAP][2]; - VkSemaphoreSubmitInfo sig[HOLD_CAP]; + VkSemaphoreSubmitInfo sig[HOLD_CAP][2]; uint64_t last; // the newest timeline value a held submission signals, 0 none uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried } H; @@ -71,6 +71,26 @@ static volatile uint64_t held_t0; // when the oldest held submission was hel static volatile uint64_t queued_v; // the newest timeline value handed to the queue static int hold_max = -1; +// ---- the executed timeline +// A submission's write-backs (vk_memory.c vk_mem_mirror_end: what its shaders wrote into VRAM mirrors goes back to guest +// memory) are the last thing in it, 3 to 4 ms of GPU time a frame in Ocarina, 60 MiB merged. The wait after a frame's +// first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run that submission's passes +// before the game rewrites what they read; it does not need the write-backs, and waited for them too: 5 ms of every +// frame with the render thread idle. +// exec_tl is a second timeline that reaches a submission's value when the GPU has executed its own commands: the +// replay ends the part before the write-backs (vk_exec.c, vk_sync_split_exec), that part signals exec_tl, and the +// write-backs follow in a part of their own, which signals the timeline as before. A submission without that split +// signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among +// them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's. +static VkSemaphore exec_tl; +static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl +static int wb_split = -1; + +int vk_sync_wb_split(void) { + if (wb_split < 0) wb_split = cfg_bool("NX2NAVI_WB_SPLIT"); + return wb_split && exec_tl != VK_NULL_HANDLE; +} + static void hold_flush_locked(VkCtx *c) { if (!H.n) return; VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); @@ -111,7 +131,10 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) { if (si->commandBufferInfoCount) memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount); H.si[i].pCommandBufferInfos = H.cbi[i]; - if (si->signalSemaphoreInfoCount) H.sig[i] = si->pSignalSemaphoreInfos[0], H.si[i].pSignalSemaphoreInfos = &H.sig[i]; + if (si->signalSemaphoreInfoCount) { // at most two: the timeline and the executed timeline (exec_tl) + memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount); + H.si[i].pSignalSemaphoreInfos = H.sig[i]; + } if (value) H.last = value; uint64_t now = kern_now_ns(); if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE); @@ -430,6 +453,12 @@ void vk_sync_start(void) { if (reserved < now) reserved = now; stop = 0; plat_mutex_unlock(&st_lock); + // the executed timeline starts where the timeline is; without it every exec wait is a fence wait + VkSemaphoreTypeCreateInfo et = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO, + .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = now}; + VkSemaphoreCreateInfo ei = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &et}; + if (vkCreateSemaphore(c->device, &ei, NULL, &exec_tl) != VK_SUCCESS) exec_tl = VK_NULL_HANDLE; + exec_sig = now; vk_mem_note_submit(now); completion_thread = plat_thread_new(completion_main, c, "vk completion"); if (!completion_thread) rt_fatal("vk: cannot start the completion thread"); @@ -484,7 +513,12 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { return cb; } -VkCommandBuffer vk_sync_split(void) { +static VkCommandBuffer split_part(int exec); +VkCommandBuffer vk_sync_split(void) { return split_part(0); } +// The part that ends here is the last with the submission's own commands: it signals the executed timeline. +VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); } + +static VkCommandBuffer split_part(int exec) { VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; vk_gpu_passes_part(cb, 1); @@ -496,6 +530,10 @@ VkCommandBuffer vk_sync_split(void) { {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}}; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1, .pCommandBufferInfos = pro ? cbi : cbi + 1}; + VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, + .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; + if (exec && exec_tl && exec_sig < cur_fence) + si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence; ts_submitted(); queue_submit(c, &si, 0); // held or handed to the queue (submit batching) vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted @@ -556,12 +594,17 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { s = &slots[cur_slot]; VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}}; - VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, - .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; + // the timeline, and the executed timeline unless a part of this submission signalled it already (split_part) + VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, + .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}, + {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, + .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}}; + int both = exec_tl && exec_sig < fence; + if (both) exec_sig = fence; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u), - .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1, - .pSignalSemaphoreInfos = &sig}; + .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = both ? 2u : 1u, + .pSignalSemaphoreInfos = sig}; ts_submitted(); queue_submit(c, &si, fence); // held or handed to the queue (submit batching) s->fence = recorded ? fence : 0; @@ -635,6 +678,27 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { return ok; } +// Waits until the GPU has executed submission value's own commands (the executed timeline); its write-backs and the +// CPU side effects of its fence may still be to come. 1 when reached. +int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { + vk_sync_start(); + if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns); + if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; + uint64_t p0 = perf_begin(); + hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now + VkCtx *c = vk_ctx(); + VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, + .pSemaphores = &exec_tl, .pValues = &value}; + VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns); + if (r != VK_SUCCESS && r != VK_TIMEOUT) { + vk_device_fault_report(r); + rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r); + } + perf_end(PERF_FENCE, p0); + perf_gpu_wait(p0); + return r == VK_SUCCESS; +} + uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); } uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); } @@ -661,6 +725,8 @@ void vk_sync_shutdown(void) { plat_mutex_lock(&sub_lock); VkCtx *c = vk_ctx(); if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL); + if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL); + exec_tl = VK_NULL_HANDLE; perf_gpu_stats = NULL; if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL); ts_pool = VK_NULL_HANDLE; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 52007d66..e931b411 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -57,7 +57,7 @@ typedef struct Tex { uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it int uploaded, released; uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve) - uint32_t rel_agen, rel_pre; // its address generation after and before the release's bump (tex_gen_unrelease) + uint64_t uid; // names the entry in its bucket's released set (tex_gen_flip); 0 until its first release uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale) uint32_t back_gap; // most presents between a release and its taking back so far (stale_after) uint32_t n_up; // uploads of this entry (vk_texture_debug) @@ -203,13 +203,35 @@ static uint16_t *slot_bucket; static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; } -// tex_gen_bump calls, by site: destroy, create, take-back, release; and the take-backs that undid their release's bump -// instead (tex_gen_unrelease), for vk_texture_bucket_stats +// tex_gen_bump calls, by site: destroy, create, take-back, release; and the releases and take-backs that went back to a +// generation their bucket's state had before (tex_gen_flip), for vk_texture_bucket_stats static uint64_t n_bumps, n_site[4], n_undone; static uint32_t bump_count[AGEN_N]; -// The highest generation each bucket ever had. A bump takes the next one after it, not the one after the current -// value, so a value a bucket went back from (tex_gen_unrelease) is never given to another state. +// The highest generation each bucket ever had. A new state takes the next one after it, not the one after the current +// value, so a value a bucket went back from (tex_gen_flip) is never given to another state. static uint32_t agen_top[AGEN_N]; +// What a slot at an address resolves to depends on the entries there and on which of them are released +// (slot_resolve). Per bucket: epoch counts every change other than a release or a take-back (an entry made, destroyed, +// retired, held or taken back with more than its flag changing), sig is the released entries' uids XORed, which a +// release and a take-back each flip. (bucket, epoch, sig) names the state; gs remembers the generation each state +// got, so a bucket that comes back to a state gets that state's generation back. +static uint32_t bepoch[AGEN_N]; +static uint64_t bsig[AGEN_N]; +#define GS_N 8192 +static struct { uint64_t sig; uint32_t b, epoch, gen, valid; } gs[GS_N]; +// A slot lookup is resolving (vk_texture_by_id_locked around slot_resolve). The lookup's callers stamp what they keep +// with the generation they read before it: the slot's own cache, the pool's tokens, the descriptor, type and feedback +// memos (vk_exec_draw.c). A lookup that itself changes the state (it takes a released entry back for a slot registered +// since the release) leaves such a stamp naming the state it left, with a result that state would not give: the next +// release of that entry is later than the slot's registration, the slot is stale then (slot_resolve), and the kept +// result would hand out the released entry, whose storage the game may be unmapping. So the state such a lookup +// leaves loses its remembered generation (tex_gen_flip): it gets a new one when the bucket is next in it, and the +// stamp never matches again. +static int in_lookup; + +static uint32_t gs_at(uint32_t b) { + return (uint32_t)(((bsig[b] ^ ((uint64_t)b << 32 | bepoch[b])) * 0x9E3779B97F4A7C15ull) >> 51); +} static void tex_gen_bump(uint64_t addr) { n_bumps++; @@ -217,35 +239,59 @@ static void tex_gen_bump(uint64_t addr) { if (!++tex_gen) tex_gen = 1; slots_gen++; uint32_t b = agen_idx(addr); + bepoch[b]++; __atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE); + gs[gs_at(b)] = (typeof(gs[0])){bsig[b], b, bepoch[b], agen[b], 1}; dirty_push(DIRTY_BUCKET | b); } -// NX2NAVI_TEX_GEN_UNDO (default 1; 0: a take-back bumps like any change). Ocarina finalizes and initializes transient -// textures all through a frame: in a large town about 40,000 releases and as many take-backs per 3,600 submissions, -// each a bump of its address bucket, and the buckets with the most slots (1,300 registered over one address) were the -// ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it and spent its 6 ms of a -// frame in half the frames there, which is also when new textures arrive late. -// A release followed by the take-back of the same entry, with no other change at its address between them, leaves every -// slot there resolving as before the release: slot_resolve reads the entries at the address and their released flags, -// and those are as they were. So the take-back puts the bucket's generation back to its value from before the -// release, and whatever was resolved then counts as current again: the pool's tokens, the descriptor memo and the -// slots' own cached entries. What was resolved while the entry was released carries the generation in between, which -// no state gets again (agen_top). The global generations still move, so their readers look, and find nothing changed. -// Not for an entry that was superseded or overwritten while released: its take-back changes more than the flag. -static void tex_gen_unrelease(Tex *e) { +// NX2NAVI_TEX_GEN_UNDO (default 1; 0: every release and take-back is a new generation). Ocarina finalizes and +// initializes transient textures all through a frame: in a large town about 40,000 releases and as many take-backs +// per 3,600 submissions, each a bump of its address bucket, and the buckets with the most slots (1,300 registered over +// one address) were the ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it. +// A release or a take-back changes one entry's released flag and nothing else, so it moves the bucket between states +// it has usually been in before: the textures the game keeps released and takes back for a moment, and those it +// releases and takes straight back. The bucket then gets the generation that state had, and whatever was resolved in +// that state counts as current again: the pool's tokens, the descriptor memo and the slots' own cached entries. A +// state seen for the first time gets a new generation (agen_top), and any other change starts a new epoch, so a +// generation never stands for two states. The global generations still move, so their readers look, and find the +// tokens equal. plain: the take-back of an entry that was superseded or overwritten while released changes more than +// the flag, and is a change like any other. +// One input of slot_resolve is not in the state: the stale rule compares a slot's registration with the entry's +// release (rel_serial), which is later at every release. A slot is stale at every later release once it is at one, so +// what it resolves to in a released state stays right; the one that is not stale takes the entry back, which is the +// lookup-made change in_lookup deals with. +static void tex_gen_flip(Tex *e, int plain) { static int on = -1; + static uint64_t uids; if (on < 0) on = rt_knob("TEX_GEN_UNDO", "1") == 1; - uint32_t b = agen_idx(e->key.addr), at = e->rel_agen; - e->rel_agen = 0; - if (!on || !at || agen[b] != at || e->superseded || e->overwritten) { + if (!e->uid) { + uint64_t z = ++uids * 0x9E3779B97F4A7C15ull; + z = (z ^ z >> 30) * 0xBF58476D1CE4E5B9ull, z = (z ^ z >> 27) * 0x94D049BB133111EBull; + e->uid = (z ^ z >> 31) | 1; + } + uint32_t b = agen_idx(e->key.addr); + if (in_lookup) { // see in_lookup: the state this lookup leaves is not to get its generation again + typeof(gs[0]) *m0 = &gs[gs_at(b)]; + if (m0->valid && m0->b == b && m0->epoch == bepoch[b] && m0->sig == bsig[b]) m0->valid = 0; + } + bsig[b] ^= e->uid; + if (!on || plain) { tex_gen_bump(e->key.addr); return; } - n_undone++; if (!++tex_gen) tex_gen = 1; slots_gen++; - __atomic_store_n(&agen[b], e->rel_pre, __ATOMIC_RELEASE); + typeof(gs[0]) *m = &gs[gs_at(b)]; + if (m->valid && m->b == b && m->epoch == bepoch[b] && m->sig == bsig[b]) { + n_undone++; + __atomic_store_n(&agen[b], m->gen, __ATOMIC_RELEASE); + } else { + n_bumps++; + bump_count[b]++; + __atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE); + *m = (typeof(*m)){bsig[b], b, bepoch[b], agen[b], 1}; + } dirty_push(DIRTY_BUCKET | b); } @@ -2134,7 +2180,7 @@ static Tex *entry_of(const tex_t *t) { e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one uint64_t gap = e->samp_frame - e->rel_frame; if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; - e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_unrelease(e); + e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); e->overwritten = 0; if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work e->superseded = 0, e->reused = 1; @@ -2201,9 +2247,7 @@ static void mark_released(Tex *e) { // tex_lock held; e is not released nreleased++; rel_append(e); n_site[3]++; - e->rel_pre = agen[agen_idx(e->key.addr)]; - tex_gen_bump(e->key.addr); - e->rel_agen = agen[agen_idx(e->key.addr)]; + tex_gen_flip(e, 0); } // e, in the GPU-only address window, is written by the GPU in frame fr: the entries its storage runs over are @@ -2473,8 +2517,8 @@ void vk_texture_bucket_stats(char *out, size_t n) { } uint64_t hot = 0; // bumps weighted by the slots they invalidate for (int b = 0; b < AGEN_N; b++) hot += (uint64_t)bump_count[b] * occ[b]; - snprintf(out, n, "%llu bumps (destroy %llu, create %llu, take-back %llu, release %llu; %llu take-backs put their " - "release's generation back instead), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u", + snprintf(out, n, "%llu new generations (destroy %llu, create %llu, take-back %llu, release %llu; %llu releases and " + "take-backs went back to a generation their state had), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u", (unsigned long long)n_bumps, (unsigned long long)n_site[0], (unsigned long long)n_site[1], (unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone, (unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]], @@ -2513,7 +2557,9 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { view_t v; int hv; resolve_held = 0; + in_lookup = 1; s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it + in_lookup = 0; s->cview = v, s->chas_view = (uint8_t)hv; s->cgen = g; s->cepoch = 0;