diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 773fb476..fe8d0d85 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -864,12 +864,13 @@ # are made inaccessible until their first read or write, which is logged with its instruction ("mem: write-back # probe"). The bytes are there all along; the probe only watches, at the cost of the faults it takes. # wb_probe = 0 -# 0: after each frame's first GPU submission the game waits until the video card has run it and everything before it -# has reached the game's memory, the last frame's visibility results among them. 1: the wait ends as soon as the card -# has run the submission, and what the card wrote goes back to the game's memory in a submission of its own. That is -# about 0.7 ms less of every frame in the open (6 % at 85 frames a second), but the game then reads visibility -# results that may be a frame old, and things near the camera can come and go from frame to frame. With shadow -# storage (Linux, Wine) 1 changes nothing; 2 forces it there as well, for tests. +# The wait after each frame's first GPU submission. 1 (default): it ends when the video card has run the submission +# and the results the game reads next are in its memory (the last frame's visibility counts above all); what the card +# wrote besides goes back to the game's memory in a submission of its own, behind the wait. 0: the game waits until +# everything before it has reached its memory, about 0.7 ms more of every frame in the open. 3: the wait ends as soon +# as the card has run the submission, without the visibility counts: the game then reads counts that may be a frame +# old, and lit things near the camera can lose their light for single frames. With shadow storage (Linux, Wine) 1 +# and 3 change nothing; 2 forces 1 there as well, for tests. # wb_split = 1 # 1: what the game's shaders write into its memory is copied back from the video card once, and after that only when # the game's CPU code touches it (most of it never is: 66 MiB a frame in Hyrule Field), for about a third more frames diff --git a/runtime/cfg.c b/runtime/cfg.c index ea93310f..34a284d1 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -317,7 +317,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2|3", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 12badf39..83000367 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -699,10 +699,10 @@ NVN_API(nvnQueueSubmitCommands) { // point). Copies of the data at submit made it worse, a mix of two frames' versions. // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits // after every submission, 0 never. - // The wait is the fence's (exec_wait, vk_sync.c vk_sync_exec_wait): besides the submission's passes it covers the - // completion thread's side effects for it and for the submissions before it, the last frame's occlusion counts - // among them, which the game reads once it goes on. NX2NAVI_WB_SPLIT=1 ends it when the GPU has run the passes - // (vk_sync.c "the executed timeline" has what that left out and what it cost). + // The wait (exec_wait, vk_sync.c vk_sync_exec_wait) ends when the GPU has run the submission's passes and the + // results the game reads once it goes on are in its memory, the last frame's occlusion counts among them; the + // write-backs and the completion thread's other side effects are not waited for. NX2NAVI_WB_SPLIT=0 makes it + // the fence's wait, which covers those too (vk_sync.c "the executed timeline" has what each leaves out and costs). static long long finish_mask = -2; if (finish_mask == -2) { const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH"); diff --git a/runtime/vk.h b/runtime/vk.h index 8eb1f851..c76a5850 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -700,6 +700,9 @@ extern int vk_sub_in_frame; // replay: the submission being recorded is the fr // Runs fn(arg) on the completion thread once fence is done on the GPU (event writes, counter reports, sync state). // Callbacks run in the order they were added, after those of earlier values; a value already done runs at once there. void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg); +// The same for a side effect the game's CPU reads (an occlusion count, a counter report, an event value): run before +// the others, and by the wait after a frame's first submission itself (vk_sync.c "the executed timeline"). +void vk_completion_add_result(uint64_t fence, void (*fn)(void *), void *arg); // gfx_vulkan_backend (vk_backend.c): presents go to a present thread and run after the submissions made before them. void rdoc_init(void); // runtime/rdoc.c: NX2NAVI_RDOC debug capture; before vkCreateInstance int rdoc_active(void); diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index e5b9a09e..40dc3d6a 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -396,7 +396,7 @@ static void oq_segment(VkExec *ex, uint64_t addr, int reset) { s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg; memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg); oq_nseg = 0, oq_lost = 0; - vk_completion_add(ex->fence, oq_cb, s); + vk_completion_add_result(ex->fence, oq_cb, s); } static void reset_counter(VkExec *ex, const p_reset_counter *p) { @@ -601,7 +601,7 @@ static void signal_event(VkExec *ex, const p_signal_event *p) { EventWrite *e = malloc(sizeof *e); if (!e) rt_fatal("vk: out of memory"); *e = (EventWrite){p->val, p->mode, p->value}; - vk_completion_add(ex->fence, event_cb, e); + vk_completion_add_result(ex->fence, event_cb, e); } static void report_counter(VkExec *ex, const p_report_counter *p) { @@ -620,7 +620,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { CounterWrite *a = malloc(sizeof *a); if (!a) rt_fatal("vk: out of memory"); *a = (CounterWrite){p->addr, report_seq}; - vk_completion_add(ex->fence, counter_cb, a); + vk_completion_add_result(ex->fence, counter_cb, a); } Pending *e = pending_slot(p->addr, 1); if (e) e->seq = report_seq, e->frame = cond_frame, e->unknown = samples, e->type = (int)p->type; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 43da1869..44dc998d 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -45,6 +45,13 @@ static uint64_t reserved; // asynchronous replay: newest value handed ou static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; +// The side effects the game's CPU reads (vk_completion_add_result: occlusion counts, counter reports, event values), +// in a queue of their own: the wait after a frame's first submission runs those that are due itself (results_run, +// "the executed timeline" below). One runner at a time, in the order they were added. +static Pending *rqueue; +static int rqhead, rqn, rqcap; +static PlatMutex res_lock = PLAT_MUTEX_INIT; // the runner; taken before st_lock +static void results_run(uint64_t upto); // ---- submit batching // Every game submission used to be its own vkQueueSubmit2, about 150 a frame, each a trip into the driver on the render @@ -75,8 +82,8 @@ static int hold_max = -1; // ---- the executed timeline // The wait after a frame's first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run // that submission's passes before the game rewrites what they read. With NX2NAVI_WB_SPLIT=0 it is the fence's wait. -// The default is 1 again: the fence's wait cost about 4.5 % of the frames a second and the flicker it was put in -// against was still reported with it. NX2NAVI_WB_SPLIT=1 makes it a wait on exec_tl, a second timeline that reaches a submission's value when +// With 1, the default, it is a wait on exec_tl, after which the waiting thread itself runs the results the game +// reads (results_run, the last paragraph but one below). exec_tl is a second timeline that reaches a submission's value when // the GPU has executed its own commands: the replay ends the part before the write-backs (vk_exec.c, // vk_sync_split_exec; vk_memory.c vk_mem_mirror_end: what the shaders wrote into VRAM mirrors goes back to guest // memory), that part signals exec_tl, and the write-backs follow in a part of their own, which signals the timeline @@ -103,6 +110,23 @@ static int hold_max = -1; // fence's wait; no scripted run showed such a frame with either wait, so that the late counts are what the player // saw is not shown. The fence's wait takes 0.65 to 0.7 ms more of a frame there: 85.5 against 86 and 92 frames a // second in the field, 80.7 against 85.8, and 98 against 105 in a village, in single runs beside other work. +// Later a player's capture showed it plainly: in Hyrule Field, the camera nearly still, trees, ground and walls lost +// their direct light for single frames about every 36 frames (the game's own cycle there), 57 of 1,800 frames, with +// the wait on exec_tl alone and 28,000 to 33,000 occlusion counts per 60 frames written after the game had gone on. +// The fence's wait covers them but waits for everything else the completion thread does for the last frame's +// submissions as well: of its 0.49 ms a frame, 0.11 ms are the occlusion counts and 0.23 to 0.40 ms one loop that +// stamps the pages those submissions' shaders wrote (vk_exec_draw.c cb_done), which the game does not read. +// So the results the game reads have a queue of their own (vk_completion_add_result: occlusion counts, counter +// reports, event values), and the waiter runs those of its submission and of the ones before it as soon as exec_tl is +// reached (results_run; the GPU ran those submissions first, so their queries are complete). The completion thread +// runs them the same way when it reaches a value, before its other callbacks, and one runner at a time keeps them in +// the order they were queued. What stays behind the wait: the write-backs and the page stamps, as with the bare wait. +// A player's hand test of this wait at the place of that capture showed none of those frames (2026-11-11). No +// scripted run has measured it: the one run made for it, with the wait on exec_tl alone, standing in the same field, +// had 8,147 of 8,160 waits back early and 1,170 occlusion counts a frame written after them, and showed no such +// frame either, so the scripted scene does not reproduce what the player saw. What the wait costs against the +// bare one is not measured; the results it runs take 0.11 ms a frame on the completion thread there. +// NX2NAVI_WB_SPLIT=3 is the wait on exec_tl alone, as it was, to measure against. // With shadow storage (vk_memory.c) NX2NAVI_WB_SPLIT=1 keeps the fence's wait as well: there the write-backs // themselves reach guest memory in a completion callback. With an import a // write-back stores into guest memory on the GPU, right behind the passes. With a shadow copy it stores into the copy, @@ -123,16 +147,20 @@ static int hold_max = -1; static VkSemaphore exec_tl; static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl static int wb_split = -1; +static int wb_results; // the wait on exec_tl also runs the results the game reads (not with 3) int vk_sync_wb_split(void) { if (wb_split < 0) { const char *e = cfg_get("NX2NAVI_WB_SPLIT"); - int forced = e && !strcmp(e, "2"), shadow = vk_mem_shadow_storage(); + int forced = e && !strcmp(e, "2"), bare = e && !strcmp(e, "3"), shadow = vk_mem_shadow_storage(); + wb_results = !bare; wb_split = cfg_bool("NX2NAVI_WB_SPLIT") && (forced || !shadow); rt_log("vk: the wait after a frame's first submission %s (NX2NAVI_WB_SPLIT%s)", - wb_split ? "ends when the GPU has run the submission's own commands: its write-backs and the completion " - "thread's side effects are left out" - : "is the fence's: it covers the write-backs and the completion thread's side effects", + !wb_split ? "is the fence's: it covers the write-backs and the completion thread's side effects" + : wb_results ? "ends when the GPU has run the submission's own commands and the results the game reads " + "are in its memory: the write-backs and the other side effects are left out" + : "ends when the GPU has run the submission's own commands: its write-backs and every side " + "effect of the completion thread are left out", shadow && !forced ? "; shadow storage keeps the fence's wait unless it is 2" : ""); } return wb_split && exec_tl != VK_NULL_HANDLE; @@ -494,7 +522,7 @@ static void completion_main(void *arg) { __atomic_store_n(&cb_tid, plat_thread_id(), __ATOMIC_RELEASE); plat_mutex_lock(&st_lock); for (;;) { - while (!stop && done == issued && !(qn && queue[qhead].fence <= done)) + while (!stop && done == issued && !(qn && queue[qhead].fence <= done) && !(rqn && rqueue[rqhead].fence <= done)) plat_cond_wait(&cv_work, &st_lock); if (stop) break; uint64_t reached = done; @@ -532,6 +560,10 @@ static void completion_main(void *arg) { // work that never finished: frees of what the GPU may still hold, records handed out again, counter reports. // done stays where it is, the waiters stay asleep, and the thread that saw the loss ends the process. if (vk_device_is_lost()) break; + plat_mutex_unlock(&st_lock); + results_run(reached); // what the game reads first; a waiter after its submit may have run them already + plat_mutex_lock(&st_lock); + if (vk_device_is_lost()) break; while (qn && queue[qhead].fence <= reached) { Pending p = queue[qhead++]; qn--; @@ -794,26 +826,58 @@ uint64_t vk_sync_submit_own(void (*record)(VkCommandBuffer, void *), void *arg) return submit(NULL, 0, &own); } -void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) { - plat_mutex_lock(&st_lock); - if (qhead + qn == qcap) { - if (qhead) { - memmove(queue, queue + qhead, sizeof *queue * (size_t)qn); - qhead = 0; +// st_lock held: p at the end of the queue *q. +static void pending_push(Pending **q, int *head, int *n, int *cap, Pending p) { + if (*head + *n == *cap) { + if (*head) { + memmove(*q, *q + *head, sizeof **q * (size_t)*n); + *head = 0; } else { - int cap = qcap ? qcap * 2 : 256; - Pending *q = realloc(queue, sizeof *q * (size_t)cap); - if (!q) rt_fatal("vk: out of memory (completion queue)"); - queue = q, qcap = cap; + int c = *cap ? *cap * 2 : 256; + Pending *m = realloc(*q, sizeof *m * (size_t)c); + if (!m) rt_fatal("vk: out of memory (completion queue)"); + *q = m, *cap = c; } } - queue[qhead + qn++] = (Pending){fence, fn, arg}; + (*q)[*head + (*n)++] = p; +} + +void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) { + plat_mutex_lock(&st_lock); + pending_push(&queue, &qhead, &qn, &qcap, (Pending){fence, fn, arg}); // The completion thread sleeps only while nothing is issued beyond done: a callback for a later value (the replay's, // for the submission it records: counter reports, freed resources) becomes due through the submit, which wakes it. if (fence <= done) plat_cond_broadcast(&cv_work); plat_mutex_unlock(&st_lock); } +// A side effect the game's CPU reads once the GPU has run submission fence's own commands: it runs before the other +// side effects of the values reached, and a wait after a submit runs it itself (vk_sync_exec_wait). +void vk_completion_add_result(uint64_t fence, void (*fn)(void *), void *arg) { + plat_mutex_lock(&st_lock); + pending_push(&rqueue, &rqhead, &rqn, &rqcap, (Pending){fence, fn, arg}); + if (fence <= done) plat_cond_broadcast(&cv_work); + plat_mutex_unlock(&st_lock); +} + +// Runs the results queued for the values up to upto, which the GPU has passed: on the completion thread for each +// value it reaches, and on a guest thread whose wait on exec_tl has just returned. st_lock not held. +static void results_run(uint64_t upto) { + plat_mutex_lock(&res_lock); + plat_mutex_lock(&st_lock); + while (rqn && rqueue[rqhead].fence <= upto && !vk_device_is_lost()) { + Pending p = rqueue[rqhead++]; + rqn--; + plat_mutex_unlock(&st_lock); + __atomic_store_n(&cb_late, p.fence <= __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE), __ATOMIC_RELAXED); + p.fn(p.arg); + __atomic_store_n(&cb_late, 0, __ATOMIC_RELAXED); + plat_mutex_lock(&st_lock); + } + plat_mutex_unlock(&st_lock); + plat_mutex_unlock(&res_lock); +} + static int fence_wait_impl(uint64_t value, uint64_t timeout_ns); int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) { @@ -849,8 +913,9 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { } // The wait after a submit: the fence's (vk_sync_fence_wait), or with NX2NAVI_WB_SPLIT until the GPU has executed -// submission value's own commands (the executed timeline), its write-backs and the completion thread's side effects -// for it and for the submissions before it still to come. 1 when reached. +// submission value's own commands (the executed timeline) and the results the game reads are in its memory +// (results_run), its write-backs and the completion thread's other side effects for it and for the submissions +// before it still to come. 1 when reached. int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { vk_sync_start(); // The fence's wait unless the split is asked for ("the executed timeline" above says what the game reads that a @@ -873,6 +938,9 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { vk_device_lost_check(r, "the wait for a submission's commands"); rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r); } + // The results the game reads, of this submission and of those before it, which the GPU ran first: here, not when + // the completion thread gets to them behind the other side effects of the last frame's submissions. + if (r == VK_SUCCESS && wb_results) results_run(value); perf_end(PERF_FENCE, p0); perf_gpu_wait(p0); if (r == VK_SUCCESS) { @@ -907,7 +975,7 @@ void vk_sync_shutdown(void) { plat_thread_join(completion_thread); completion_thread = NULL; plat_mutex_lock(&st_lock); - qhead = qn = 0; // callbacks of values never submitted + qhead = qn = rqhead = rqn = 0; // callbacks of values never submitted plat_mutex_unlock(&st_lock); plat_mutex_lock(&sub_lock); VkCtx *c = vk_ctx();