diff --git a/nx2navi.ini.example b/nx2navi.ini.example index b6a1a00e..9da59324 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -590,6 +590,10 @@ # are made inaccessible until their first read or write, which is logged with its instruction ("mem: write-back # probe"). The bytes are there all along; the probe only watches, at the cost of the faults it takes. # wb_probe = 0 +# 1: what the GPU wrote goes back to the game's memory in a GPU submission of its own, and the wait after each frame's +# first submission ends when that submission's passes are through, without waiting for the copy back (3 to 4 ms of +# every frame). 0: the copy back is part of the submission, and the wait covers it. +# wb_split = 1 # 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every # page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in # Kokiri Forest. diff --git a/runtime/cfg.c b/runtime/cfg.c index c2b024b2..8bcb63f5 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -62,7 +62,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", @@ -118,7 +118,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; @@ -231,7 +231,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/gfx.h b/runtime/gfx.h index 2a1b7778..762404de 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -145,6 +145,9 @@ typedef struct { uint64_t (*submit)(const uint64_t *handles, int n); // replays; returns this submission's fence value // (gfx_submit_queue names the game's queue) int (*fence_wait)(uint64_t value, uint64_t timeout_ns); // 1 reached; caller has given its core up + // As fence_wait, but only until the GPU has executed the submission's own commands: its write-backs to guest + // memory and its fence's side effects may follow (vk_sync.c exec_tl). The same as fence_wait where there is no split. + int (*exec_wait)(uint64_t value, uint64_t timeout_ns); uint64_t (*fence_done)(void); // highest value whose side effects are written // Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that // has passed or at_ns is 0. diff --git a/runtime/gfx_null.c b/runtime/gfx_null.c index dd01f281..b73010a4 100644 --- a/runtime/gfx_null.c +++ b/runtime/gfx_null.c @@ -146,6 +146,7 @@ const GfxBackend gfx_null_backend = { .program_release = null_program_release, .submit = null_submit, .fence_wait = null_fence_wait, + .exec_wait = null_fence_wait, .fence_done = null_fence_done, .present = null_present, .finish = null_finish, diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 28aeeb1b..73f5d213 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -660,6 +660,9 @@ NVN_API(nvnQueueSubmitCommands) { // point). Copies of the data at submit made it worse, a mix of two frames' versions. // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits // after every submission, 0 never. + // The wait is for the submission's passes, which is what the rewrite races with: exec_wait leaves out the + // write-backs of what they wrote (vk_sync.c exec_tl), which nothing here needs before the game goes on. The fences + // the game itself waits on cover them as before. static long long finish_mask = -2; if (finish_mask == -2) { const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH"); @@ -667,7 +670,7 @@ NVN_API(nvnQueueSubmitCommands) { if (e && *e && finish_mask == 1 && !(e[0] == '0' && (e[1] == 'x' || e[1] == 'X'))) finish_mask = -1; // "1": all } int idx = nvn_submits_in_frame++; - if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->fence_wait(f, UINT64_MAX); + if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->exec_wait(f, UINT64_MAX); } // The producer (1.2.1 main f_70008c6460) releases the read lock on obj+0x538 at lr 0x70008c70c8; when diff --git a/runtime/vk.h b/runtime/vk.h index 4ca82456..a124ee7f 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -570,6 +570,11 @@ void vk_sync_start(void); // vk_init (fatal without a device) a uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted +// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet. +int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns); +int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own +VkCommandBuffer vk_sync_split_exec(void); // vk_sync_split for the part before a submission's write-backs +int vk_mem_mirror_wb_pending(void); // vk_memory.c: vk_mem_mirror_end has write-backs to record uint64_t vk_sync_fence_done(void); // highest done value uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1) void vk_sync_finish(void); // every submitted value done diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index f5389dfd..a2e715c8 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -183,6 +183,11 @@ int vk_replay_async(void) { return on; } +// With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's. +static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { + return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns); +} + // The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end. int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); } @@ -606,6 +611,7 @@ const GfxBackend gfx_vulkan_backend = { .program_release = vk_be_program_release, .submit = vk_be_submit, .fence_wait = vk_sync_fence_wait, + .exec_wait = vk_be_exec_wait, .fence_done = vk_sync_fence_done, .present = vk_be_present, .frame_start = vk_dlssg_frame_start, diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index bd891bb2..c2f056ae 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1325,6 +1325,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_pass_end(ex); x_here = 0; if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes + // The write-backs in a part of their own, after one that signals the executed timeline (vk_sync.c exec_tl): the + // wait after a frame's first submission then ends when the submission's passes are through, not its write-backs. + if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) { + ex->cb = vk_sync_split_exec(); + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + } if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory if (ex->nviews) { ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ced07ef4..38cfaf51 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2331,6 +2331,8 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) { return j->n + 2; } +int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); } + int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { if (!vk_mirror_on) return 0; int rec = 0; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index cae29b16..6d9e1176 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -62,7 +62,7 @@ static struct { int n; VkSubmitInfo2 si[HOLD_CAP]; VkCommandBufferSubmitInfo cbi[HOLD_CAP][2]; - VkSemaphoreSubmitInfo sig[HOLD_CAP]; + VkSemaphoreSubmitInfo sig[HOLD_CAP][2]; uint64_t last; // the newest timeline value a held submission signals, 0 none uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried } H; @@ -71,6 +71,26 @@ static volatile uint64_t held_t0; // when the oldest held submission was hel static volatile uint64_t queued_v; // the newest timeline value handed to the queue static int hold_max = -1; +// ---- the executed timeline +// A submission's write-backs (vk_memory.c vk_mem_mirror_end: what its shaders wrote into VRAM mirrors goes back to guest +// memory) are the last thing in it, 3 to 4 ms of GPU time a frame in Ocarina, 60 MiB merged. The wait after a frame's +// first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run that submission's passes +// before the game rewrites what they read; it does not need the write-backs, and waited for them too: 5 ms of every +// frame with the render thread idle. +// exec_tl is a second timeline that reaches a submission's value when the GPU has executed its own commands: the +// replay ends the part before the write-backs (vk_exec.c, vk_sync_split_exec), that part signals exec_tl, and the +// write-backs follow in a part of their own, which signals the timeline as before. A submission without that split +// signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among +// them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's. +static VkSemaphore exec_tl; +static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl +static int wb_split = -1; + +int vk_sync_wb_split(void) { + if (wb_split < 0) wb_split = cfg_bool("NX2NAVI_WB_SPLIT"); + return wb_split && exec_tl != VK_NULL_HANDLE; +} + static void hold_flush_locked(VkCtx *c) { if (!H.n) return; VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); @@ -111,7 +131,10 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) { if (si->commandBufferInfoCount) memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount); H.si[i].pCommandBufferInfos = H.cbi[i]; - if (si->signalSemaphoreInfoCount) H.sig[i] = si->pSignalSemaphoreInfos[0], H.si[i].pSignalSemaphoreInfos = &H.sig[i]; + if (si->signalSemaphoreInfoCount) { // at most two: the timeline and the executed timeline (exec_tl) + memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount); + H.si[i].pSignalSemaphoreInfos = H.sig[i]; + } if (value) H.last = value; uint64_t now = kern_now_ns(); if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE); @@ -430,6 +453,12 @@ void vk_sync_start(void) { if (reserved < now) reserved = now; stop = 0; plat_mutex_unlock(&st_lock); + // the executed timeline starts where the timeline is; without it every exec wait is a fence wait + VkSemaphoreTypeCreateInfo et = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO, + .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = now}; + VkSemaphoreCreateInfo ei = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &et}; + if (vkCreateSemaphore(c->device, &ei, NULL, &exec_tl) != VK_SUCCESS) exec_tl = VK_NULL_HANDLE; + exec_sig = now; vk_mem_note_submit(now); completion_thread = plat_thread_new(completion_main, c, "vk completion"); if (!completion_thread) rt_fatal("vk: cannot start the completion thread"); @@ -484,7 +513,12 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { return cb; } -VkCommandBuffer vk_sync_split(void) { +static VkCommandBuffer split_part(int exec); +VkCommandBuffer vk_sync_split(void) { return split_part(0); } +// The part that ends here is the last with the submission's own commands: it signals the executed timeline. +VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); } + +static VkCommandBuffer split_part(int exec) { VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; vk_gpu_passes_part(cb, 1); @@ -496,6 +530,10 @@ VkCommandBuffer vk_sync_split(void) { {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}}; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1, .pCommandBufferInfos = pro ? cbi : cbi + 1}; + VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, + .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; + if (exec && exec_tl && exec_sig < cur_fence) + si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence; ts_submitted(); queue_submit(c, &si, 0); // held or handed to the queue (submit batching) vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted @@ -556,12 +594,17 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { s = &slots[cur_slot]; VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}}; - VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, - .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; + // the timeline, and the executed timeline unless a part of this submission signalled it already (split_part) + VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, + .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}, + {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, + .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}}; + int both = exec_tl && exec_sig < fence; + if (both) exec_sig = fence; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u), - .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1, - .pSignalSemaphoreInfos = &sig}; + .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = both ? 2u : 1u, + .pSignalSemaphoreInfos = sig}; ts_submitted(); queue_submit(c, &si, fence); // held or handed to the queue (submit batching) s->fence = recorded ? fence : 0; @@ -635,6 +678,27 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { return ok; } +// Waits until the GPU has executed submission value's own commands (the executed timeline); its write-backs and the +// CPU side effects of its fence may still be to come. 1 when reached. +int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { + vk_sync_start(); + if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns); + if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; + uint64_t p0 = perf_begin(); + hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now + VkCtx *c = vk_ctx(); + VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, + .pSemaphores = &exec_tl, .pValues = &value}; + VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns); + if (r != VK_SUCCESS && r != VK_TIMEOUT) { + vk_device_fault_report(r); + rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r); + } + perf_end(PERF_FENCE, p0); + perf_gpu_wait(p0); + return r == VK_SUCCESS; +} + uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); } uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); } @@ -661,6 +725,8 @@ void vk_sync_shutdown(void) { plat_mutex_lock(&sub_lock); VkCtx *c = vk_ctx(); if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL); + if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL); + exec_tl = VK_NULL_HANDLE; perf_gpu_stats = NULL; if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL); ts_pool = VK_NULL_HANDLE;