diff --git a/docs/releases/v0.9.3.md b/docs/releases/v0.9.3.md new file mode 100644 index 00000000..3d967b56 --- /dev/null +++ b/docs/releases/v0.9.3.md @@ -0,0 +1,19 @@ +## Faster + +- **Experimental: the next frame no longer waits for the end of the last one's graphics work** (`tail_early = 1` in + nx2navi.ini, off by default; contributed by epicmeme546). The game waits each frame until the graphics card has + finished the frame before. The last of that work is the upscaler (FSR 4, FSR 3.1 or DLSS) and the interface drawn + after it, and nothing the game reads comes out of it. With the setting on, the copies of what the card wrote for the + game go before that last stretch and the game goes on once they are there, so the card finishes the picture while + the next frame is already being prepared. The card runs the same work as before. On an RX 7900 XT with a Ryzen 7 + 5800X3D at 3440x1440, FSR 4 Native AA, Hyrule Field: 50.7 instead of 42.5 fps standing and 56.0 instead of 45.9 along + a route; with `replay_workers = 3` as well, 63.5 instead of 49.4 standing. With FSR 3.1, whose pass is shorter, 51.8 + instead of 47.8 standing. Its pictures matched standing, along the route and through warps to five areas, but it is + new: leave it off to play, and report what you see if you try it. It does nothing without one of those upscalers, + and nothing yet on NVIDIA and Intel cards, where the lazy write-back is on. + +## Changed + +- **The draw replay on several CPU cores is off by default again** (`replay_workers = 0`). With three workers on by + default in 0.9.1 and 0.9.2 some players' games ended with a lost graphics device; the cause is not found yet. The + settings menu has it under Game, Testing, as *Replay workers (experimental)*: 0 to 8, after a restart. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 12c200f5..cef8e1e7 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -73,6 +73,13 @@ # With 1 a worker records each long run of draws while the submitting thread waits for it: no faster, it checks that # the pictures stay the same. # replay_workers = 0 +# EXPERIMENTAL. 1 lets the game start on its next frame while the graphics card still runs the end of the last one: the +# upscaler (FSR 4, FSR 3.1 or DLSS) and the interface drawn after it. The graphics card runs the same work, with the +# copies of what it wrote for the game moved before that end, and the game's waits end once those copies and the +# results it reads are there. A frame whose end hands the game anything else runs as before. Default: 0. Standing in +# Hyrule Field on an RX 7900 XT with FSR 4: 50.7 instead of 42.5 fps, and 63.5 instead of 49.4 with replay_workers = 3. +# Does nothing without one of those upscalers, nor with the lazy write-back (the default on NVIDIA and Intel cards). +# tail_early = 0 # EXPERIMENTAL, THE GAME CRASHES WITH 1. 1 lifts the virtual core gate: guest threads run without the console's limit # of one thread per core, and the game keeps per-core data without locks. With 1 the game crashes or hangs, # usually at start. Keep 0 to play. diff --git a/runtime/cfg.c b/runtime/cfg.c index 01261137..4b19652b 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -64,7 +64,7 @@ static const char *const known[] = { "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "POOL_PRESNAP_RUNS", "SUBMIT_FINISH_GPU", "SVC_WAITS", "SNAP_WIDEN", "SUBMIT_FINISH_RESULTS", "SUBMIT_FINISH_KINDS", "FOLIAGE", "MIRROR_STAGE", "FLUSH_SKIP_CLEAN", "MIRROR_HOT", "MESH_EMULATION", "MESH_EMU_MB", "MESH_EMU_TASKS", "MESH_EMU_BATCH", "GUEST_OPT", "LOG_TEX_NEW", "DUMP_KEY", "VEC_INDEX_FIX", "PASS_TIMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", - "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", + "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "TAIL_EARLY", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", @@ -130,7 +130,7 @@ static const struct { const char *key, *section; } sections[] = { {"BIND_RIGHT", "input"}, {"BIND_DOWN", "input"}, {"BIND_LSTICK_LEFT", "input"}, {"BIND_LSTICK_UP", "input"}, {"BIND_LSTICK_RIGHT", "input"}, {"BIND_LSTICK_DOWN", "input"}, {"BIND_RSTICK_LEFT", "input"}, {"BIND_RSTICK_UP", "input"}, {"BIND_RSTICK_RIGHT", "input"}, {"BIND_RSTICK_DOWN", "input"}, {"BIND_SHAKE", "input"}, - {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, {"REPLAY_PIPE", "performance"}, + {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, {"REPLAY_PIPE", "performance"}, {"TAIL_EARLY", "performance"}, {"FREE_CORES", "performance"}, {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, @@ -260,6 +260,7 @@ static const CfgMeta metas[] = { // Advanced: scheduling {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, {"REPLAY_WORKERS", "0", "0..16", RESTART}, {"REPLAY_PIPE", "0", "0..4", RESTART}, + {"TAIL_EARLY", "0", "0|1", RESTART}, // vk_exec.c "the early tail" {"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART}, // dynamic resolution (dynres.c): the scene follows the GPU time toward DYN_RES_TARGET milliseconds {"DYN_RES", "0", "0|1", LIVE}, {"DYN_RES_TARGET", "16.7", "1..100", LIVE}, {"DYN_RES_STEP", "45", "8..600", LIVE}, diff --git a/runtime/gfx.h b/runtime/gfx.h index a07d0e7a..8a71bf3a 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -160,6 +160,10 @@ typedef struct { // As fence_wait, but only until the GPU has executed the submission's own commands: its write-backs to guest // memory and its fence's side effects may follow (vk_sync.c exec_tl). The same as fence_wait where there is no split. int (*exec_wait)(uint64_t value, uint64_t timeout_ns); + // The game's own wait for a fence (nvnSyncWait): fence_wait, or where the back end knows the rest of the value's + // GPU work reads nothing the game can change and hands it nothing, until its results are there (vk_sync.c + // vk_sync_game_wait). + int (*game_wait)(uint64_t value, uint64_t timeout_ns); uint64_t (*fence_done)(void); // highest value whose side effects are written // Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that // has passed or at_ns is 0. diff --git a/runtime/gfx_null.c b/runtime/gfx_null.c index b73010a4..bd4d1b61 100644 --- a/runtime/gfx_null.c +++ b/runtime/gfx_null.c @@ -147,6 +147,7 @@ const GfxBackend gfx_null_backend = { .submit = null_submit, .fence_wait = null_fence_wait, .exec_wait = null_fence_wait, + .game_wait = null_fence_wait, .fence_done = null_fence_done, .present = null_present, .finish = null_finish, diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 3945b419..e62c4cfb 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -564,7 +564,7 @@ NVN_API(nvnSyncWait) { int ok; { NVN_BLOCKING(); - ok = b->fence_wait(s->fence, timeout); + ok = b->game_wait(s->fence, timeout); } if (ok) b->cpu_synced(s->fence); ret_x(c, ok ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED); diff --git a/runtime/vk.h b/runtime/vk.h index f0b3cbbe..d85ac1c7 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -379,6 +379,23 @@ int vk_mem_take_live(void); // for the newest such submission up to the one it waits for. enum { VK_RES_EVENT = 1, VK_RES_COUNTER = 2, VK_RES_OQ = 4, VK_RES_WRITE = 8, VK_RES_WRITEBACK = 16, VK_RES_KINDS = 5 }; void vk_exec_note_results(int kind); // VK_RES_* +// The early tail (vk_exec.c "the early tail", NX2NAVI_TAIL_EARLY). vk_exec_tail: the replay is recording a submission's +// tail. vk_sync.c: the tail's command buffer set aside for the write-backs' one (hold), then the part with the +// write-backs handed over with the executed timeline's signal and the tail put behind it (release); whether a part can +// take that order now (ok); the game's wait for a fence (game_wait); for a value whose tail the GPU still runs, about +// how long that takes (tail_pending_ns, else 0). +struct VkExec; +void vk_exec_tail_begin(struct VkExec *ex); +int vk_exec_tail(void); +void vk_exec_tail_ub(uint64_t fence, uint64_t addr, const void *data, uint32_t size); // vk_exec_xfer.c +int vk_sync_tail_ok(void); +VkCommandBuffer vk_sync_tail_hold(void); +VkCommandBuffer vk_sync_tail_release(void); +int vk_sync_game_wait(uint64_t value, uint64_t timeout_ns); +uint64_t vk_sync_tail_pending_ns(uint64_t value); +int vk_mem_live_swap(int v); // vk_memory.c: the note of a late read of guest memory, exchanged +uint64_t vk_mem_gpu_wrote_count(void); // vk_memory.c: GPU writes of pool memory noted so far +int vk_mem_wb_lazy(void); // vk_memory.c: the lazy write-back runs int vk_exec_take_results(void); // the kinds noted since the last take void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over void vk_mem_lazy_handed(void); diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 167518af..d80d6cdb 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -919,6 +919,7 @@ const GfxBackend gfx_vulkan_backend = { .submit = vk_be_submit, .fence_wait = vk_sync_fence_wait, .exec_wait = vk_be_exec_wait, + .game_wait = vk_sync_game_wait, .fence_done = vk_sync_fence_done, .present = vk_be_present, .frame_start = vk_dlssg_frame_start, diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index 84c95176..0e892a3d 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -2033,6 +2033,7 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { const VkExecTarget *t = &D.color, *z = &D.depth; if (!dst) dst = t; uint32_t w = t->width, h = t->height, ow = dst->width, oh = dst->height; + vk_exec_tail_begin(ex); // NX2NAVI_TAIL_EARLY: from here on the frame's tail, in a command buffer of its own VkCommandBuffer cb = ex->cb; vk_exec_barrier(ex); // what drew the scene is done (no pass is open here) vk_gpu_gap_note(VK_GAP_UPSCALER, (uint64_t)ow * oh); diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 76404c49..cda73217 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1122,6 +1122,59 @@ static int split_n = -1, split_idle, split_in_pass, split_draws, split_at, part_ static uint64_t split_ns, part_t0; static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits, n_cuts; +// ---- the early tail (NX2NAVI_TAIL_EARLY, default 0) +// Standing in Hyrule Field with FSR 4 (RX 7900 XT, 2026-11-13) a frame's main submission ends with one part of 4.7 ms +// of GPU time, the temporal upscaler and the interface drawn after it, and one of 1.4 ms, the write-backs of what the +// shaders wrote into VRAM mirrors. The game waits for all of it before it writes the next frame's data (the wait after +// a frame's first submission, nvn_api_core.c, and its own fence waits), about 7 ms in which the replay thread has +// nothing to record. What the game reads of the submission is done before the upscaler runs: the occlusion and +// counter reports, the event values, and the write-backs' bytes, which only came last. After it the stream holds two +// timestamp reports, and inline uniform updates for the interface's draws (64 bytes into one mirrored block, some +// thirty times), and nothing else the CPU sees. +// So from the upscaler on (vk_dlaa.c evaluate) the replay records the tail into a command buffer of its own +// (vk_sync_cut), without splits or parallel groups. An inline uniform update of a mirrored block in it takes no +// write-back: the GPU copies the bytes into the mirror in stream order as ever, and guest memory gets them from the +// CPU with the submission's results (vk_exec_draw.c op_update_ub, vk_exec_xfer.c vk_exec_tail_ub). At the submission's +// end, when the tail turned out to hand the CPU nothing but timestamps (vk_exec_tail_results), to write no other pool +// memory (vk_mem_gpu_wrote_count) and to bind no memory the GPU reads from the game's own pages (vk_mem_live_swap: a +// pool without a mirror, a device address table), +// the write-backs are recorded into a command buffer that goes to the queue before the tail's, with the executed +// timeline's signal (vk_sync_tail_hold, vk_sync_tail_release): the GPU runs the same commands, the write-backs before +// the tail. The mirror copies the tail's first uses queued go with that part too, so they read the game's pages before +// the signal. The game's waits then end at that signal and the results the game reads (vk_sync.c vk_sync_exec_wait, +// vk_sync_game_wait), and the GPU runs the tail while the game and the replay are at the next frame. A timestamp +// reported in the tail is written then with the time the tail is expected to end at (vk_exec_xfer.c counter_cb), as +// the game measures the frame's GPU time with it. A tail that fails a test goes as before, the write-backs after it. +// Measured there, 150 s runs, each pair in one batch: 63.5 against 49.4 fps with three replay workers; without them +// 50.7 against 42.5 standing and 56.0 against 45.9 along the field route, and with FSR 3.1, whose tail the GPU runs +// for 1.2 ms where FSR 4's takes 3.5, 51.8 against 47.8 and 54.9 against 52.3. All but a few of the tails passed the +// tests there; in area loads and some places a tail hands the CPU results and goes as before. The pictures matched +// those without it standing, along the route and through warps to five areas. Not measured with the lazy write-back +// (NVIDIA's and Intel's default), where a tail's uniform updates are noted as writes and no tail goes early. +// Not with shadow storage, emulated mesh shaders, a frame dump or a pass capture, nor inside a chunk or a continuation. +static struct { + int on; // the submission's tail is being recorded + int live0; // vk_mem_live_swap's note from before it + uint64_t wrote0; // vk_mem_gpu_wrote_count when it began + // since the last perf line: tails; those that went early; those refused by results, writes, late reads, other + uint64_t n, early, why[4]; +} TL; +int vk_exec_tail_results(void); // vk_exec_xfer.c +void vk_sync_tail_stats(uint64_t *tails, uint64_t *waits, double *ms); +int vk_gpu_passes_capturing(void); +int vk_mesh_shader_native(void); // vk_device.c +int vk_exec_tail(void) { return TL.on; } + +static int tail_enabled(void) { + static int on = -1; + if (on < 0) { + on = cfg_bool("NX2NAVI_TAIL_EARLY"); + if (on) + rt_log("vk: the write-backs go before a frame's tail, and the game's waits end there (NX2NAVI_TAIL_EARLY)"); + } + return on; +} + int vk_exec_split_now(VkExec *ex) { int in_pass = ex->in_pass; if (in_pass) vk_exec_pass_end(ex); @@ -1153,7 +1206,7 @@ static void maybe_split(VkExec *ex) { split_idle ? "; a first part at once while the GPU is idle" : ""); split_at = split_n ? split_n : INT_MAX; } - if (!split_n) return; + if (!split_n || TL.on) return; // the tail stays one command buffer (the early tail) if (Q.drained < Q.out) { // groups the submitting thread went on past go into the part first, once done if (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < Q.out) return; // split_draws stays: a later op asks again group_drain(ex, PW_SPLIT); @@ -2318,7 +2371,7 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { int n = (int)(draws / CHUNK_DRAWS); if (n > RW.n + 1) n = RW.n + 1; if (n > GROUP_CHUNKS) n = GROUP_CHUNKS; - if (n < 2 || vk_dump_armed || !vk_sync_append_room(n)) { + if (n < 2 || vk_dump_armed || TL.on || !vk_sync_append_room(n)) { RW.skip_to[depth] = end, RW.serial_draws += draws, RW.small_end[why] += draws; return h; } @@ -2510,6 +2563,24 @@ static void rec_claim(void) { } } +// The tail begins here (the early tail): what was recorded so far waits in the chain, the tail gets a command buffer of +// its own. Not in a chunk or a continuation, nor with groups still out: the cut needs every piece of the part. +void vk_exec_tail_begin(VkExec *ex) { + if (!tail_enabled() || TL.on || ex != x_cur || rw_chunk || evo_in || Q.drained != Q.out) return; + if (!vk_sync_wb_split() || !vk_mesh_shader_native() || vk_dump_armed || vk_gpu_passes_capturing()) return; + if (!vk_sync_tail_ok()) return; + if (ex->in_pass) vk_exec_pass_end(ex); + vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL + VkCommandBuffer cb = vk_sync_cut(); + if (!cb) return; + ex->cb = cb; + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + n_cuts++; + TL.on = 1, TL.live0 = vk_mem_live_swap(0), TL.wrote0 = vk_mem_gpu_wrote_count(); + vk_exec_tail_results(); +} + int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): nothing is recorded uint64_t in_ns = kern_now_ns(); @@ -2527,6 +2598,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t nvn_pins_sweep(nvn_submit_count(), keep, 9); ex->cb = cb; ex->fence = fence; + TL.on = 0; ex->recorded = 0; ex->in_pass = 0; ex->mark = NULL; @@ -2593,12 +2665,47 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t // NX2NAVI_WB_SPLIT: the write-backs in a part of their own, after one that signals the executed timeline // (vk_sync.c exec_tl): the wait after a frame's first submission then ends when the submission's passes are // through, not its write-backs. Off by default: vk_sync.c says what else that wait then leaves out. - if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) { + int tail = 0; // the early tail: the write-backs go before the tail, with the executed timeline's signal + if (TL.on) { + int live = vk_mem_live_swap(0), res = vk_exec_tail_results(); + vk_mem_live_swap(live | TL.live0); // the submission's own note, as it would have been + int wrote = vk_mem_gpu_wrote_count() != TL.wrote0; + int other = !synced || !ex->recorded || vk_dump_armed || vk_gpu_passes_capturing() || !vk_sync_tail_ok(); + TL.on = 0, TL.n++; + if (res) TL.why[0]++; + else if (wrote) TL.why[1]++; + else if (live) TL.why[2]++; + else if (other || !vk_exec_pool_split(ex->fence)) TL.why[3]++; + else tail = 1, TL.early++; + if (tail) { + ex->cb = vk_sync_tail_hold(); + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + } + } + if (!tail && synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && + vk_exec_pool_split(ex->fence)) { ex->cb = vk_sync_split_exec(); ex->dirty = VK_EXEC_DIRTY_ALL; vk_exec_draw_new_cb(); } if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory + if (tail) { + ex->cb = vk_sync_tail_release(); + part_start = ex->recorded; // the last command buffer holds nothing: the tail is in the chain before it + } + if ((rt_trace & RT_TRACE_PERF) && TL.n >= 600) { + uint64_t tails, waits; + double ms; + vk_sync_tail_stats(&tails, &waits, &ms); + rt_log("perf: early tail: %llu tails, %llu with the write-backs before them (refused: %llu handed the CPU " + "results, %llu wrote pool memory, %llu read guest memory late, %llu other); %llu waits of the game " + "ended before their tail, which the GPU runs for about %.2f ms", + (unsigned long long)TL.n, (unsigned long long)TL.early, + (unsigned long long)TL.why[0], (unsigned long long)TL.why[1], (unsigned long long)TL.why[2], + (unsigned long long)TL.why[3], (unsigned long long)waits, ms); + TL.n = TL.early = 0, memset(TL.why, 0, sizeof TL.why); + } views_release(ex, fence); ex->cb = VK_NULL_HANDLE; if (n) n_subs++; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 1c743a08..f5819e42 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -7201,7 +7201,14 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { VK_MEM_PICK(sp); VkBufferCopy cp = {o, sp.offset, u->size}; vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp); - vk_mem_gpu_wrote(&sp, u->size); + // In a submission's tail (vk_exec.c "the early tail") an update of a mirrored block reaches guest memory with the + // results the game reads, from the bytes here, and takes no write-back: one after the tail would land when the + // game is already writing the next frame's data. Not with the lazy write-back, where the block's pages may be + // armed and a store into them would wait for the GPU (memory.c lazy_fault): the update is noted as ever there, + // and the tail goes after the write-backs as before. + if (vk_exec_tail() && !vk_mem_wb_lazy() && vk_mem_mirrored(u->dst, u->size)) + vk_exec_tail_ub(ex->fence, u->dst, u->data, u->size); + else vk_mem_gpu_wrote(&sp, u->size); vk_usnap_drop(u->dst, u->size); ex->recorded++; vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size); diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index ef379279..4b6695c9 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -67,12 +67,32 @@ typedef struct { uint64_t addr, size; int back; } Range; // back: written into // for the newest such submission). Write-backs of the mirrors note it in vk_memory.c; the lazily written ones fetch on // the CPU's first access. static volatile int32_t ex_results; // VK_RES_* bits +static int tail_results; // those noted while the submission's tail records (vk_exec.c "the early tail") +int vk_exec_tail_results(void) { int r = tail_results; tail_results = 0; return r; } void vk_exec_note_results(int kind) { + if (vk_exec_tail()) tail_results |= kind; __atomic_fetch_or(&ex_results, kind, __ATOMIC_RELAXED); vk_gpu_passes_result(kind); // NX2NAVI_GPU_PASSES: where in the submission it is done } int vk_exec_take_results(void) { return __atomic_exchange_n(&ex_results, 0, __ATOMIC_RELAXED); } +// An inline uniform update in a submission's tail (vk_exec_draw.c op_update_ub, vk_exec.c "the early tail"): the GPU +// copies the bytes into the pool's mirror where the stream has the update, and guest memory gets them here, with the +// results the game reads, in place of a write-back. +typedef struct { uint64_t addr; uint32_t size; uint8_t data[]; } UbWrite; +static void ub_write_cb(void *arg) { + UbWrite *w = arg; + memcpy((void *)(uintptr_t)w->addr, w->data, w->size); + free(w); +} +void vk_exec_tail_ub(uint64_t fence, uint64_t addr, const void *data, uint32_t size) { + UbWrite *w = malloc(sizeof *w + size); + if (!w) rt_fatal("vk: out of memory"); + w->addr = addr, w->size = size; + memcpy(w->data, data, size); + vk_completion_add_result(fence, ub_write_cb, w); +} + static void guest_written_cb(void *arg) { Range *r = arg; if (r->back) vk_gpu_written_back(r->addr, r->size); @@ -123,12 +143,18 @@ static void write_counter(uint64_t addr, uint64_t counter) { // clears on it differing from a zero record (mode 1), clears that must not run. static void pending_written(uint64_t addr, uint64_t seq); // below, with the pending table -typedef struct { uint64_t addr, seq; } CounterWrite; +// tail: the submission's value for a timestamp reported in its tail (vk_exec.c "the early tail"), else 0 +typedef struct { uint64_t addr, seq, tail; } CounterWrite; static void counter_cb(void *arg) { CounterWrite *w = arg; if (vk_sync_effect_late()) __atomic_fetch_add(&late_counters, 1, __ATOMIC_RELAXED); - write_counter(w->addr, 0); + // A timestamp of the tail, written while the GPU still runs the tail: the time the tail is expected to end at + uint64_t ahead = w->tail ? vk_sync_tail_pending_ns(w->tail) : 0; + if (ahead) { + uint64_t rec[2] = {0, (uint64_t)((unsigned __int128)(kern_now_ns() + ahead) * 12 / 625)}; + memcpy((void *)(uintptr_t)w->addr, rec, sizeof rec); + } else write_counter(w->addr, 0); pending_written(w->addr, w->seq); free(w); } @@ -761,10 +787,13 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { vk_mem_count(m.cls, VK_MUSE_GPU_WRITE, sizeof(NVNcounterData)); if (samples) oq_report(ex, p->addr); else { + // a timestamp in the tail is the one result a tail may hand the CPU: its time is written ahead (counter_cb) + int ts_tail = vk_exec_tail() && p->type == NVN_COUNTER_TYPE_TIMESTAMP, keep = tail_results; vk_exec_note_results(VK_RES_COUNTER); + if (ts_tail) tail_results = keep; CounterWrite *a = malloc(sizeof *a); if (!a) rt_fatal("vk: out of memory"); - *a = (CounterWrite){p->addr, report_seq}; + *a = (CounterWrite){p->addr, report_seq, ts_tail ? ex->fence : 0}; vk_completion_add_result(ex->fence, counter_cb, a); } Pending *e = pending_slot(p->addr, 1); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 2d2f6c48..b3402816 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -3303,7 +3303,11 @@ static uint64_t wb_skip(void) { } static int64_t m_wb_skipped; // bytes wb_skip kept in mirrors since the last summary +static volatile uint64_t n_gpu_wrote; // vk_mem_gpu_wrote_count +uint64_t vk_mem_gpu_wrote_count(void) { return __atomic_load_n(&n_gpu_wrote, __ATOMIC_RELAXED); } + static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { + __atomic_fetch_add(&n_gpu_wrote, 1, __ATOMIC_RELAXED); PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size || !wb_lazy()) { uint64_t skip = r && r->mstate == MS_ON && sp->buffer == r->mbuf ? wb_skip() : 0; @@ -3678,6 +3682,9 @@ static uint64_t capture_cap(void) { } void vk_mem_note_live(void) { __atomic_store_n(&mirror_live, 1, __ATOMIC_RELAXED); } +int vk_mem_live_swap(int v) { return __atomic_exchange_n(&mirror_live, v, __ATOMIC_RELAXED); } +static int wb_lazy(void); +int vk_mem_wb_lazy(void) { return wb_lazy(); } // the lazy write-back runs (vk_exec_draw.c op_update_ub) int vk_mem_take_live(void) { int live = __atomic_exchange_n(&mirror_live, 0, __ATOMIC_RELAXED); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 56e80544..70dd113d 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -62,6 +62,16 @@ static uint64_t live_last_of[LIVE_RING], live_last; // per value (index & LIVE_RING - 1): the newest value up to it whose completion hands results to the CPU (vk_exec_take_results) static uint64_t res_of[LIVE_RING], res_last; static uint8_t res_kinds[LIVE_RING]; // per value: its VK_RES_* kinds +// The early tail (vk_exec.c "the early tail"), per value under st_lock: early_of holds the value when its part with +// the write-backs signalled the executed timeline before its tail; empty_of when the submission handed the GPU nothing +// and the CPU no results (a submission held at its first event wait: Ocarina's first of a frame in the field). +static uint64_t early_of[LIVE_RING], empty_of[LIVE_RING]; +static VkCommandBuffer tail_cb; // sub_lock: the tail's command buffer between hold and release +// sub_lock: this submission's tail went behind its write-backs; the parts of it handed over before the last +static int tail_early, sub_parts; +static volatile uint64_t tail_ns; // about how long the GPU takes for a tail (a moving average) +static volatile uint64_t tail_t0, tail_val; // the first early return for tail_val, for tail_ns +static volatile uint64_t tail_n, tail_waits; // since the last vk_sync_tail_stats: early tails; waits they ended static int res_mask = 0x1f; // NX2NAVI_SUBMIT_FINISH_KINDS: the kinds the wait counts static int stop; static Pending *queue; // side effects in the order they were added @@ -915,6 +925,12 @@ static void completion_main(void *arg) { // finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each // checked against the timeline itself) is nothing they read, and it takes cmd_lock, which the upload // path holds, so done waited behind uploads. + uint64_t tv = __atomic_load_n(&tail_val, __ATOMIC_ACQUIRE); + if (tv && reached >= tv) { // the tail a wait went ahead of is through: how long it took from there + uint64_t d = kern_now_ns() - __atomic_load_n(&tail_t0, __ATOMIC_RELAXED), a = tail_ns; + if (d < 50000000ull) __atomic_store_n(&tail_ns, a ? (a * 7 + d) / 8 : d, __ATOMIC_RELAXED); + __atomic_compare_exchange_n(&tail_val, &tv, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED); + } uint64_t was = done; __atomic_store_n(&done, reached, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_done); @@ -1192,6 +1208,63 @@ VkCommandBuffer vk_sync_worker_begin(int w) { void vk_sync_worker_end(VkCommandBuffer cb) { vkEndCommandBuffer(cb); } static VkCommandBuffer split_part(int exec); + +// ---- the early tail (vk_exec.c "the early tail"). The replay cut the part where the tail began (vk_sync_cut), so the +// chain holds what came before and the command buffer being recorded is the tail's. vk_sync_tail_hold sets that one +// aside, ended and timed as a part of its own, and begins the one the write-backs are recorded into; +// vk_sync_tail_release hands the chain and the write-backs to the queue with the executed timeline's signal +// (split_part), and puts the tail into the chain for the submission's last part, which signals the timeline. The queue +// runs them in that order, so the GPU's work is what it was with the write-backs moved before the tail. +int vk_sync_tail_ok(void) { return !pre_cb && !tail_cb && exec_tl && exec_sig < cur_fence && nchain + 2 <= CHAIN_MAX; } + +VkCommandBuffer vk_sync_tail_hold(void) { + VkCtx *c = vk_ctx(); + VkCommandBuffer cb = slots[cur_slot].cb; + vk_gpu_passes_part(cb, 1); + ts_end(cb, cur_fence); + if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); + tail_cb = cb; + CmdSlot *s = take_slot(c, cur_fence); // the tail keeps its slot until the value is reached + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(s->cb, &bi); + ts_begin(s->cb); + rec_begin(s->cb, cur_fence); + vk_gpu_passes_part(s->cb, 0); + return s->cb; +} + +VkCommandBuffer vk_sync_tail_release(void) { + VkCommandBuffer cb = split_part(1); + chain[nchain++] = tail_cb; + tail_cb = VK_NULL_HANDLE; + tail_early = 1; + return cb; +} + +// The newest value up to `value` whose GPU work a wait of the game is for: the empty submissions after it are passed +// over. *early: that one's write-backs went before its tail, so the executed timeline reaches it there. +static uint64_t wait_target(uint64_t value, int *early) { + plat_mutex_lock(&st_lock); + uint64_t t = value; + while (t > done && empty_of[t & (LIVE_RING - 1)] == t) t--; + *early = t > done && early_of[t & (LIVE_RING - 1)] == t; + plat_mutex_unlock(&st_lock); + return t; +} + +uint64_t vk_sync_tail_pending_ns(uint64_t value) { + plat_mutex_lock(&st_lock); + int pending = value > done && early_of[value & (LIVE_RING - 1)] == value; + plat_mutex_unlock(&st_lock); + return pending ? __atomic_load_n(&tail_ns, __ATOMIC_RELAXED) : 0; +} + +void vk_sync_tail_stats(uint64_t *tails, uint64_t *waits, double *ms) { + *tails = __atomic_exchange_n(&tail_n, 0, __ATOMIC_RELAXED); + *waits = __atomic_exchange_n(&tail_waits, 0, __ATOMIC_RELAXED); + *ms = (double)__atomic_load_n(&tail_ns, __ATOMIC_RELAXED) / 1e6; +} VkCommandBuffer vk_sync_split(void) { return split_part(0); } // The part that ends here is the last with the submission's own commands: it signals the executed timeline. VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); } @@ -1213,6 +1286,7 @@ static VkCommandBuffer split_part(int exec) { si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence; ts_submitted(cbi, ncb); queue_submit(c, &si, 0); // held or handed to the queue (submit batching) + sub_parts++; vk_mem_lazy_handed(); vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached @@ -1254,6 +1328,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { vk_mem_dirty_submit(); // NX2NAVI_TRACE=perf: what a mirror would upload before this submit uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1; cur_fence = fence; + tail_early = sub_parts = 0; CmdSlot *s = take_slot(c, fence); VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; @@ -1313,6 +1388,9 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { if (live) live_last = fence; live_last_of[fence & (LIVE_RING - 1)] = live_last; res_kinds[fence & (LIVE_RING - 1)] = (uint8_t)res; + early_of[fence & (LIVE_RING - 1)] = tail_early ? fence : 0; + empty_of[fence & (LIVE_RING - 1)] = !own && !sent && !sub_parts && !res ? fence : 0; + if (tail_early) __atomic_fetch_add(&tail_n, 1, __ATOMIC_RELAXED); if (res & res_mask) res_last = fence; res_of[fence & (LIVE_RING - 1)] = res_last; __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); @@ -1525,8 +1603,13 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { if (vk_device_is_lost()) return __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value; if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns); if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; + // The early tail: a value whose GPU work is that of an earlier submission with its write-backs before its tail (the + // submissions since handed the GPU nothing) is reached where that one's executed signal is, its tail still to run. + int early = 0; + uint64_t asked = value, target = wb_results ? wait_target(value, &early) : value; + if (early) value = target; uint64_t p0 = perf_begin(); - hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now + hold_release(asked, 0, 0); // a held submission the waiter needs goes to the queue now VkCtx *c = vk_ctx(); VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, .pSemaphores = &exec_tl, .pValues = &value}; @@ -1540,6 +1623,13 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { if (r == VK_SUCCESS && wb_results) results_run(value); perf_end(PERF_FENCE, p0); perf_gpu_wait(p0); + if (r == VK_SUCCESS && early && __atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) { + __atomic_fetch_add(&tail_waits, 1, __ATOMIC_RELAXED); + uint64_t none = 0; // the first wait back for this tail starts its clock (the completion thread ends it) + if (__atomic_load_n(&tail_val, __ATOMIC_ACQUIRE) != value && + __atomic_compare_exchange_n(&tail_val, &none, value, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) + __atomic_store_n(&tail_t0, kern_now_ns(), __ATOMIC_RELAXED); + } if (r == VK_SUCCESS) { __atomic_fetch_add(&exw_n, 1, __ATOMIC_RELAXED); if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) { @@ -1551,6 +1641,19 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { return r == VK_SUCCESS; } +// The game's own wait for a fence (nvn_api_core.c nvnSyncWait): the fence's wait, or with the early tail the wait for +// the results of the submission the value's GPU work belongs to (vk_sync_exec_wait), when that one's tail is all the +// GPU still has of it: the tail reads nothing the game can change and hands it nothing. +int vk_sync_game_wait(uint64_t value, uint64_t timeout_ns) { + vk_sync_start(); + if (vk_sync_wb_split() && wb_results && __atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) { + int early = 0; + wait_target(value, &early); + if (early) return vk_sync_exec_wait(value, timeout_ns); + } + return vk_sync_fence_wait(value, timeout_ns); +} + uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); } uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }