End the wait after a frame's first submission when its passes are through, not its write-backs.

Every frame the render thread waited about 5 ms for the GPU after the frame's first submission, with the game
CPU-bound in every area. The wait is there so that the GPU has run that submission's passes before the game
rewrites what they read. But the submission also ends with the write-backs of what its shaders wrote, 60 MiB merged
into guest memory in 3 to 4 ms of GPU time, and the wait covered those too.
The write-backs now go into a part of their own. The part before them signals a second timeline, the executed
timeline, and the first-submission wait waits on that; the write-back part signals the timeline as before, so every
fence the game asks for still covers them, in the same queue order. wb_split = 0 keeps one part and the old wait.
Not measured in the game yet.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-09 23:42:12 -05:00
parent be0a3eda67
commit 9728424aec
10 changed files with 108 additions and 11 deletions

View File

@@ -590,6 +590,10 @@
# are made inaccessible until their first read or write, which is logged with its instruction ("mem: write-back
# probe"). The bytes are there all along; the probe only watches, at the cost of the faults it takes.
# wb_probe = 0
# 1: what the GPU wrote goes back to the game's memory in a GPU submission of its own, and the wait after each frame's
# first submission ends when that submission's passes are through, without waiting for the copy back (3 to 4 ms of
# every frame). 0: the copy back is part of the submission, and the wait covers it.
# wb_split = 1
# 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every
# page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in
# Kokiri Forest.

View File

@@ -62,7 +62,7 @@ static const char *const known[] = {
"SWKBD_ANSWER",
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"ZSTD_CHECK", "ZSTD_NATIVE",
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP",
@@ -118,7 +118,7 @@ static const struct { const char *key, *section; } sections[] = {
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
{"VK_NAN_PRESERVE", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"},
};
@@ -231,7 +231,7 @@ static const CfgMeta metas[] = {
{"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag
// diagnostics
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
{"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha

View File

@@ -145,6 +145,9 @@ typedef struct {
uint64_t (*submit)(const uint64_t *handles, int n); // replays; returns this submission's fence value
// (gfx_submit_queue names the game's queue)
int (*fence_wait)(uint64_t value, uint64_t timeout_ns); // 1 reached; caller has given its core up
// As fence_wait, but only until the GPU has executed the submission's own commands: its write-backs to guest
// memory and its fence's side effects may follow (vk_sync.c exec_tl). The same as fence_wait where there is no split.
int (*exec_wait)(uint64_t value, uint64_t timeout_ns);
uint64_t (*fence_done)(void); // highest value whose side effects are written
// Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that
// has passed or at_ns is 0.

View File

@@ -146,6 +146,7 @@ const GfxBackend gfx_null_backend = {
.program_release = null_program_release,
.submit = null_submit,
.fence_wait = null_fence_wait,
.exec_wait = null_fence_wait,
.fence_done = null_fence_done,
.present = null_present,
.finish = null_finish,

View File

@@ -660,6 +660,9 @@ NVN_API(nvnQueueSubmitCommands) {
// point). Copies of the data at submit made it worse, a mix of two frames' versions.
// NX2NAVI_SUBMIT_FINISH=<mask>: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits
// after every submission, 0 never.
// The wait is for the submission's passes, which is what the rewrite races with: exec_wait leaves out the
// write-backs of what they wrote (vk_sync.c exec_tl), which nothing here needs before the game goes on. The fences
// the game itself waits on cover them as before.
static long long finish_mask = -2;
if (finish_mask == -2) {
const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH");
@@ -667,7 +670,7 @@ NVN_API(nvnQueueSubmitCommands) {
if (e && *e && finish_mask == 1 && !(e[0] == '0' && (e[1] == 'x' || e[1] == 'X'))) finish_mask = -1; // "1": all
}
int idx = nvn_submits_in_frame++;
if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->fence_wait(f, UINT64_MAX);
if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->exec_wait(f, UINT64_MAX);
}
// The producer (1.2.1 main f_70008c6460) releases the read lock on obj+0x538 at lr 0x70008c70c8; when

View File

@@ -570,6 +570,11 @@ void vk_sync_start(void); // vk_init (fatal without a device) a
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet.
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns);
int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own
VkCommandBuffer vk_sync_split_exec(void); // vk_sync_split for the part before a submission's write-backs
int vk_mem_mirror_wb_pending(void); // vk_memory.c: vk_mem_mirror_end has write-backs to record
uint64_t vk_sync_fence_done(void); // highest done value
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
void vk_sync_finish(void); // every submitted value done

View File

@@ -183,6 +183,11 @@ int vk_replay_async(void) {
return on;
}
// With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's.
static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) {
return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns);
}
// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end.
int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); }
@@ -606,6 +611,7 @@ const GfxBackend gfx_vulkan_backend = {
.program_release = vk_be_program_release,
.submit = vk_be_submit,
.fence_wait = vk_sync_fence_wait,
.exec_wait = vk_be_exec_wait,
.fence_done = vk_sync_fence_done,
.present = vk_be_present,
.frame_start = vk_dlssg_frame_start,

View File

@@ -1325,6 +1325,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
vk_exec_pass_end(ex);
x_here = 0;
if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes
// The write-backs in a part of their own, after one that signals the executed timeline (vk_sync.c exec_tl): the
// wait after a frame's first submission then ends when the submission's passes are through, not its write-backs.
if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) {
ex->cb = vk_sync_split_exec();
ex->dirty = VK_EXEC_DIRTY_ALL;
vk_exec_draw_new_cb();
}
if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory
if (ex->nviews) {
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);

View File

@@ -2331,6 +2331,8 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) {
return j->n + 2;
}
int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); }
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
if (!vk_mirror_on) return 0;
int rec = 0;

View File

@@ -62,7 +62,7 @@ static struct {
int n;
VkSubmitInfo2 si[HOLD_CAP];
VkCommandBufferSubmitInfo cbi[HOLD_CAP][2];
VkSemaphoreSubmitInfo sig[HOLD_CAP];
VkSemaphoreSubmitInfo sig[HOLD_CAP][2];
uint64_t last; // the newest timeline value a held submission signals, 0 none
uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried
} H;
@@ -71,6 +71,26 @@ static volatile uint64_t held_t0; // when the oldest held submission was hel
static volatile uint64_t queued_v; // the newest timeline value handed to the queue
static int hold_max = -1;
// ---- the executed timeline
// A submission's write-backs (vk_memory.c vk_mem_mirror_end: what its shaders wrote into VRAM mirrors goes back to guest
// memory) are the last thing in it, 3 to 4 ms of GPU time a frame in Ocarina, 60 MiB merged. The wait after a frame's
// first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run that submission's passes
// before the game rewrites what they read; it does not need the write-backs, and waited for them too: 5 ms of every
// frame with the render thread idle.
// exec_tl is a second timeline that reaches a submission's value when the GPU has executed its own commands: the
// replay ends the part before the write-backs (vk_exec.c, vk_sync_split_exec), that part signals exec_tl, and the
// write-backs follow in a part of their own, which signals the timeline as before. A submission without that split
// signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among
// them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's.
static VkSemaphore exec_tl;
static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl
static int wb_split = -1;
int vk_sync_wb_split(void) {
if (wb_split < 0) wb_split = cfg_bool("NX2NAVI_WB_SPLIT");
return wb_split && exec_tl != VK_NULL_HANDLE;
}
static void hold_flush_locked(VkCtx *c) {
if (!H.n) return;
VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
@@ -111,7 +131,10 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) {
if (si->commandBufferInfoCount)
memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount);
H.si[i].pCommandBufferInfos = H.cbi[i];
if (si->signalSemaphoreInfoCount) H.sig[i] = si->pSignalSemaphoreInfos[0], H.si[i].pSignalSemaphoreInfos = &H.sig[i];
if (si->signalSemaphoreInfoCount) { // at most two: the timeline and the executed timeline (exec_tl)
memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount);
H.si[i].pSignalSemaphoreInfos = H.sig[i];
}
if (value) H.last = value;
uint64_t now = kern_now_ns();
if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE);
@@ -430,6 +453,12 @@ void vk_sync_start(void) {
if (reserved < now) reserved = now;
stop = 0;
plat_mutex_unlock(&st_lock);
// the executed timeline starts where the timeline is; without it every exec wait is a fence wait
VkSemaphoreTypeCreateInfo et = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = now};
VkSemaphoreCreateInfo ei = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &et};
if (vkCreateSemaphore(c->device, &ei, NULL, &exec_tl) != VK_SUCCESS) exec_tl = VK_NULL_HANDLE;
exec_sig = now;
vk_mem_note_submit(now);
completion_thread = plat_thread_new(completion_main, c, "vk completion");
if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
@@ -484,7 +513,12 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
return cb;
}
VkCommandBuffer vk_sync_split(void) {
static VkCommandBuffer split_part(int exec);
VkCommandBuffer vk_sync_split(void) { return split_part(0); }
// The part that ends here is the last with the submission's own commands: it signals the executed timeline.
VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); }
static VkCommandBuffer split_part(int exec) {
VkCtx *c = vk_ctx();
VkCommandBuffer cb = slots[cur_slot].cb;
vk_gpu_passes_part(cb, 1);
@@ -496,6 +530,10 @@ VkCommandBuffer vk_sync_split(void) {
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
.pCommandBufferInfos = pro ? cbi : cbi + 1};
VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
.value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
if (exec && exec_tl && exec_sig < cur_fence)
si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence;
ts_submitted();
queue_submit(c, &si, 0); // held or handed to the queue (submit batching)
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
@@ -556,12 +594,17 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
s = &slots[cur_slot];
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}};
VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
// the timeline, and the executed timeline unless a part of this submission signalled it already (split_part)
VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT},
{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}};
int both = exec_tl && exec_sig < fence;
if (both) exec_sig = fence;
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u),
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1,
.pSignalSemaphoreInfos = &sig};
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = both ? 2u : 1u,
.pSignalSemaphoreInfos = sig};
ts_submitted();
queue_submit(c, &si, fence); // held or handed to the queue (submit batching)
s->fence = recorded ? fence : 0;
@@ -635,6 +678,27 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
return ok;
}
// Waits until the GPU has executed submission value's own commands (the executed timeline); its write-backs and the
// CPU side effects of its fence may still be to come. 1 when reached.
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
vk_sync_start();
if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns);
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1;
uint64_t p0 = perf_begin();
hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now
VkCtx *c = vk_ctx();
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
.pSemaphores = &exec_tl, .pValues = &value};
VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns);
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
vk_device_fault_report(r);
rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r);
}
perf_end(PERF_FENCE, p0);
perf_gpu_wait(p0);
return r == VK_SUCCESS;
}
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
@@ -661,6 +725,8 @@ void vk_sync_shutdown(void) {
plat_mutex_lock(&sub_lock);
VkCtx *c = vk_ctx();
if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL);
exec_tl = VK_NULL_HANDLE;
perf_gpu_stats = NULL;
if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL);
ts_pool = VK_NULL_HANDLE;