diff --git a/docs/releases/v0.7.2.md b/docs/releases/v0.7.2.md index 3ed45a4d..6aecd747 100644 --- a/docs/releases/v0.7.2.md +++ b/docs/releases/v0.7.2.md @@ -1,5 +1,21 @@ Draft notes for the release after 0.7.1. The version number is not decided yet. +## Faster + +- **AMD cards: about a fifth more frames in open areas, and more even frame times.** The game asks the video card + which objects are visible, about a hundred questions a frame in Hyrule Field, and waits for a frame to be finished + on the card before it lets the next one start. The answers were fetched one call at a time, on the thread that also + reports "finished", and on AMD's Windows driver each of those calls goes into the kernel: every frame was reported + finished about 4 ms late, with every thread of the game waiting. They are now fetched a few hundred at a time. + Standing in Hyrule Field on an RX 7900 XT (driver 26.10.1, 3440x1440, FSR): 53 fps instead of 45, and the longest + frame of a minute 46 ms instead of 55 (`oq_batch = 0` in `nx2navi.ini` brings the old readback back). +- **No more frames drawn one present late.** The game signals from the CPU when a frame's data is written, and its + render thread submits the frame's commands without waiting for that signal. When the signal came a millisecond or + two after the submit call, in about every other frame in Hyrule Field, the frame was set aside and drawn at its + present, and frames of 23 and 15 ms took turns. The submit call now waits for the signal, 10 ms at most + (`gate_wait_ms`): 54.5 fps instead of 53 in the same place and the longest frame 33 ms instead of 46; with + `vk_recorder = 1`, which AMD cards do not have on by default, 58.6 fps. + ## Fixed - Black and white blocks on AMD RX 5000 and RX 6000 cards: on Navi and on Link next to her, on bright floors near her, diff --git a/nx2navi.ini.example b/nx2navi.ini.example index af589f65..b3344468 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -74,6 +74,13 @@ # recorder_core = auto # Submissions handed to the GPU queue together in one call, 1 to 16 (1 = each one at once). # submit_batch = 4 +# Occlusion query results are read back from the video card in batches, 0 or 1. Default: on. 0 reads each query with a +# call of its own, as before, which on AMD's Windows driver held every frame's "GPU done" back by about 4 ms. +# oq_batch = 1 +# How many milliseconds a submit call waits for the game's own signal that the frame's data is written, when the +# frame's commands begin with a wait for that signal (0 to 100; 0: the frame is set aside at once and drawn at its +# present, as before). +# gate_wait_ms = 10 # Dynamic resolution, 0 or 1, applied at once: with a DLSS upscaling mode, the scene resolution follows the GPU time # per frame toward dyn_res_target milliseconds (1 to 100), one DLSS step at a time, each held for dyn_res_step frames # (8 to 600), from the DLSS mode set down to performance. Default: off. diff --git a/runtime/cfg.c b/runtime/cfg.c index 6f6d6d5a..65a1d8c6 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -47,7 +47,7 @@ static const char *const known[] = { "HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE", "HDR_LIVE", "HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", - "BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST", + "BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "OQ_BATCH", "GATE_WAIT_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", @@ -115,7 +115,7 @@ static const struct { const char *key, *section; } sections[] = { {"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"}, - {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"}, + {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"COMPUTE_KEEP", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, @@ -234,6 +234,8 @@ static const CfgMeta metas[] = { {"PIPE_FAST_WORKERS", "1", "0|1", RESTART}, // vk_pipeline.c lib_worker, fast_take {"FRAME_HOLD_MS", "120", "0..500", RESTART}, // vk_backend.c frame_hold {"PACE_SMOOTH", "1", "0|1", RESTART}, // vk_backend.c pace_smooth + {"OQ_BATCH", "1", "0|1", RESTART}, // vk_exec_xfer.c oq_take + {"GATE_WAIT_MS", "10", "0..100", RESTART}, // vk_exec.c vk_exec_gate_wait {"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART}, {"LIB_WARM", "1", "0|1|force", RESTART}, // vk_pipeline.c lw_start: the driver's cache of stage libraries, warmed when cold {"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART}, diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index cbd08768..87576bd3 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -386,6 +386,7 @@ static uint64_t submit_sync(const uint64_t *handles, int n) { if (!vk_exec_held()) ph_flush(&PH.after); } vk_present_drain(); + vk_exec_gate_wait(handles, n); // a submission that begins with a wait for the CPU's signal: before its copies vk_texture_new_submit(); uint64_t f = vk_sync_submit(handles, n); // The submission stopped at a wait whose signal the CPU gave while it was being replayed (vk_exec_xfer.c "events diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 6ac73408..37ece725 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1124,6 +1124,72 @@ static int cont_ready(const Cont *c) { return !c->at || vk_exec_event_met(nvn_op_payload(c->at)) || vk_exec_submit_seq - c->since > CONT_MAX; } +// ---- the gate: a submission that begins with a wait for the CPU's signal (NX2NAVI_GATE_WAIT_MS, default 10; 0 off) +// Ocarina's main queue begins with a wait for an event the game signals from the CPU once it has written what the +// frame reads (vk_exec_xfer.c "events the CPU signals"). The render thread submits that queue without waiting for the +// signal, and polls the event afterwards, a millisecond at a time, before it presents. In Hyrule Field the signal +// came 1 to 2 ms after the submit call in about every other frame: the main thread first waits for the previous +// frame's fence and reads back what it needs. The submission then took its copies (the mirror sync and the uniform +// snapshot, 1.1 to 1.7 ms), met the wait unmet and was held; the render thread polled, presented, and the present +// replayed the frame (vk_backend.c held_resume) under a value the game's fence does not stand for, so the game's next +// wait for that fence returned with the GPU still at work and the frame after was replayed inside its submit call +// again: frames of 23 and 15 ms in turn where each takes 17. +// So when a submission would stop at such a wait before it has recorded anything, the submit call waits for the +// signal here, before the copies are taken, as the console's GPU waits at the wait itself: the copies are then +// taken after the game has finished writing, and the frame is replayed in the call the game's fence stands for. +// Only state may precede the wait in the command buffer, and the event must be one the CPU has signalled before; +// the wait ends after NX2NAVI_GATE_WAIT_MS at the latest, and the submission is then held as before. +static struct { uint64_t n, ns, late; } GW; // since the last perf line: waits, their time, waits that ran out + +static const void *gate_first_wait(const op_hdr *h) { + for (int k = 0; k < 256; k++) { + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) return NULL; + const void *a = nvn_op_payload(h); + switch (h->op) { + case OP_WAIT_EVENT: return a; + case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; + // state the walk only stores: nothing is recorded and nothing reaches the CPU before the wait + case OP_BEGIN: case OP_SET_RT: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_BIND_PROGRAM: + case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO: + case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_STENCIL: case OP_POLY_OFFSET: + case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_COPY_STRIDE: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP: + case OP_RASTER_DISCARD: case OP_PRIM_RESTART: case OP_PUSH_CC: + break; + default: return NULL; // a draw, a copy, a signal, a call: the walk decides, as before + } + h = nvn_op_next(h); + } + return NULL; +} + +void vk_exec_gate_wait(const uint64_t *handles, int n) { + static int64_t cap_ns = -1; + if (cap_ns < 0) { + int ms = cfg_int("NX2NAVI_GATE_WAIT_MS"); + cap_ns = ms > 0 ? (int64_t)ms * 1000000 : 0; + } + if (!cap_ns || n < 1 || !handles[0] || !gfx_submit_queue || !evo_enabled()) return; + const void *w = gate_first_wait(nvn_op_first(handles[0])); + if (!w || vk_exec_event_holds_now(w) || !vk_exec_event_cpu_known((const void *)((const p_wait_event *)w)->val)) return; + uint64_t t0 = kern_now_ns(), now = t0; + for (unsigned i = 0; !vk_exec_event_holds_now(w); i++) { + if ((i & 15) == 15 && (now = kern_now_ns()) - t0 >= (uint64_t)cap_ns) { + GW.late++; + break; + } + if (i < 20000) plat_pause(); + else plat_yield(); + } + GW.n++, GW.ns += kern_now_ns() - t0; + if (GW.n >= 600) { + if (rt_trace & RT_TRACE_PERF) + rt_log("perf: gate: %llu submissions waited for the CPU's signal before their copies, %.2f ms on average; " + "%llu waited %lld ms without it and were held", (unsigned long long)GW.n, (double)GW.ns / (double)GW.n / 1e6, + (unsigned long long)GW.late, (long long)(cap_ns / 1000000)); + GW.n = GW.ns = GW.late = 0; + } +} + int vk_exec_held_ready(void) { for (int s = 0; evo_pending && s < EVO_QUEUES; s++) if (evq[s].head && cont_ready(evq[s].head)) return 1; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 6ce192c6..2f65f91c 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -242,6 +242,11 @@ extern int vk_exec_signalled; // set by every replayed SignalEve // Events the CPU signals (vk_exec_xfer.c "events the CPU signals"). void vk_exec_event_cpu(const void *event); // any thread: nvnEventSignal is about to write the event's value void vk_exec_event_sync(int begin); // vk_exec_record: its mirror sync and uniform copy are taken now / it ends +int vk_exec_event_cpu_known(const void *event); // the CPU has signalled this event before +int vk_exec_event_holds_now(const void *payload); // an OP_WAIT_EVENT's condition on the value in memory now +// vk_backend.c submit_sync, before a submission is recorded: waits for the CPU's signal when the submission begins +// with a wait for it (vk_exec.c "the gate"). +void vk_exec_gate_wait(const uint64_t *handles, int n); int vk_exec_event_at_sync(void); // NX2NAVI_EVENT_AT_SYNC uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after the copies would have passed, since the last call // Since the last call, which resets them: waits, waits reached before their signal, those of them a later replayed diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index f5cd3a6c..a0e83134 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -265,38 +265,103 @@ void vk_exec_oq_draw(VkExec *ex) { void vk_exec_oq_pass_end(VkExec *ex) { oq_close(ex); } +static void oq_batch_drop(void); // the results kept by the completion thread (oq_take) + void vk_exec_xfer_shutdown(void) { if (oq_state > 0) vkDestroyQueryPool(vk_ctx()->device, oq_pool, NULL); oq_pool = VK_NULL_HANDLE, oq_state = 0, oq_next = 0, oq_counting = oq_reported = oq_keep = 0, oq_open = -1; oq_nseg = 0, oq_lost = 0, oq_count = 0; memset((void *)oq_busy, 0, sizeof oq_busy); + oq_batch_drop(); +} + +// The completion thread reads the results in batches (NX2NAVI_OQ_BATCH, default 1; 0: every segment by itself, as +// before). The game tests one proxy per report, so a segment holds one query, and a frame of Hyrule Field has about a +// hundred of them: one vkGetQueryPoolResults and one vkResetQueryPool each. On an RX 7900 XT (26.10.1) each of those +// calls goes into the kernel, and the two hundred took 3.9 ms of every frame on the completion thread, which runs a +// submission's callbacks before it lets the submission's waiters go: the game's main thread waits for the previous +// frame's fence before it lets the next frame's GPU work start, so the frame after one replayed inside its submit +// call began 4 to 6 ms late with every thread idle. Slots come from the ring in order, so the slots of the segments +// that follow lie behind the one asked for: one call reads OQ_BATCH slots from there on with their availability, the +// available ones are kept (the results stay in the pool until the reset), and the segments after it take theirs from +// what is kept. A slot not available in the batch (its part had not run yet) is read again when its segment comes. +// The slots taken are reset in runs, one vkResetQueryPool for each: a run ends when a slot that does not continue it +// is taken or a new batch is read. Until then its slots stay busy for the replay, at most OQ_BATCH of the ring's +// OQ_N, also while the game reports nothing. +#define OQ_BATCH 256 +static struct { + uint32_t first, n; // the slots kept: [first, first + n) + uint64_t v[OQ_BATCH * 2]; // per slot: samples, available + uint8_t taken[OQ_BATCH]; + uint32_t r0, rn; // the run of taken slots still to be reset + uint64_t calls, slots, segs; // since the last perf line: result calls, slots they read, segments served +} OQB; +static int oq_batch_mode = -1; + +static void oq_batch_drop(void) { OQB.n = OQB.rn = 0; } + +static void oq_reset_flush(VkDevice dev) { + if (!OQB.rn) return; + vkResetQueryPool(dev, oq_pool, OQB.r0, OQB.rn); + for (uint32_t k = 0; k < OQB.rn; k++) __atomic_store_n(&oq_busy[OQB.r0 + k], 0, __ATOMIC_RELEASE); + OQB.rn = 0; +} + +static void oq_read(VkDevice dev, uint32_t first, uint32_t n) { + VkResult r = vkGetQueryPoolResults(dev, oq_pool, first, n, sizeof OQB.v[0] * 2 * n, OQB.v, 2 * sizeof OQB.v[0], + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); + if (r != VK_SUCCESS && r != VK_NOT_READY) { + vk_device_fault_report(r); + rt_fatal("vk: occlusion query results failed (%d)", (int)r); + } + OQB.first = first, OQB.n = n; + memset(OQB.taken, 0, n); + OQB.calls++, OQB.slots += n; +} + +// The samples of slot q, whose submission has completed; *avail 0 when the query has no result. +static uint64_t oq_take(VkDevice dev, uint32_t q, int *avail) { + uint32_t k = q - OQB.first; + if (!(q >= OQB.first && k < OQB.n && !OQB.taken[k] && OQB.v[2 * k + 1])) { + // not kept, or kept as unavailable (read before its part ran): from this slot on, up to the replay's next slot + oq_reset_flush(dev); // before the kept results go: the run's slots are among them + uint32_t next = __atomic_load_n(&oq_next, __ATOMIC_RELAXED) % OQ_N, n = OQ_N - q; + if (next > q && next - q < n) n = next - q; + if (n > OQ_BATCH) n = OQ_BATCH; + if (oq_batch_mode == 0) n = 1; + oq_read(dev, q, n); + k = 0; + } + OQB.taken[k] = 1; + if (OQB.rn && q != OQB.r0 + OQB.rn) oq_reset_flush(dev); + if (!OQB.rn) OQB.r0 = q; + OQB.rn++; + *avail = OQB.v[2 * k + 1] != 0; + return OQB.v[2 * k]; } static void oq_cb(void *arg) { OqSegment *s = arg; VkDevice dev = vk_ctx()->device; - uint64_t sum = 0, v[64 * 2]; - for (uint32_t i = 0; i < s->n;) { - uint32_t j = i + 1; - while (j < s->n && j - i < 64 && s->q[j] == s->q[j - 1] + 1) j++; + if (oq_batch_mode < 0) oq_batch_mode = cfg_bool("NX2NAVI_OQ_BATCH"); + uint64_t sum = 0; + for (uint32_t i = 0; i < s->n; i++) { // The fence has passed, so a query that is still unavailable will not become available: waiting would block the // completion thread, and with it every later callback, for good. It counts as visible (one sample). - VkResult r = vkGetQueryPoolResults(dev, oq_pool, s->q[i], j - i, sizeof v, v, 2 * sizeof v[0], - VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); - if (r != VK_SUCCESS && r != VK_NOT_READY) { - vk_device_fault_report(r); - rt_fatal("vk: occlusion query results failed (%d)", (int)r); + int avail; + uint64_t v = oq_take(dev, s->q[i], &avail); + if (avail) sum += v; + else { + sum += 1; + LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[i]); } - vkResetQueryPool(dev, oq_pool, s->q[i], j - i); - for (uint32_t k = i; k < j; k++) { - if (v[2 * (k - i) + 1]) sum += v[2 * (k - i)]; - else { - sum += 1; - LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[k]); - } - __atomic_store_n(&oq_busy[s->q[k]], 0, __ATOMIC_RELEASE); - } - i = j; + } + if (s->n && ++OQB.segs >= 12000) { + if (rt_trace & RT_TRACE_PERF) + rt_log("perf: occlusion queries: %llu segments read with %llu result calls of %.1f slots on average", + (unsigned long long)OQB.segs, (unsigned long long)OQB.calls, + OQB.calls ? (double)OQB.slots / (double)OQB.calls : 0.0); + OQB.calls = OQB.slots = OQB.segs = 0; } if (s->reset) oq_count = 0; else { @@ -412,6 +477,22 @@ void vk_exec_event_cpu(const void *event) { plat_mutex_unlock(&cpu_ev_lock); } +// 1 when the CPU has signalled this event before (nvnEventSignal): its waits are gates the game opens from the CPU. +int vk_exec_event_cpu_known(const void *event) { + plat_mutex_lock(&cpu_ev_lock); + int i = 0; + while (i < cpu_ev_n && cpu_ev[i].val != event) i++; + int known = i < cpu_ev_n; + plat_mutex_unlock(&cpu_ev_lock); + return known; +} + +// The wait's condition on the value in memory now, whatever the replay has seen. Any thread. +int vk_exec_event_holds_now(const void *payload) { + const p_wait_event *p = payload; + return ev_holds(__atomic_load_n(p->val, __ATOMIC_ACQUIRE), p->mode, p->value); +} + void vk_exec_event_sync(int begin) { plat_mutex_lock(&cpu_ev_lock); ev_sync_seq = begin ? cpu_ev_seq : UINT64_MAX;