diff --git a/docs/releases/v0.9.3.md b/docs/releases/v0.9.3.md new file mode 100644 index 00000000..88496e36 --- /dev/null +++ b/docs/releases/v0.9.3.md @@ -0,0 +1,29 @@ +## Faster + +- **Experimental: the next frame no longer waits for the end of the last one's graphics work** (`tail_early = 1` in + nx2navi.ini, off by default; contributed by epicmeme546). The game waits each frame until the graphics card has + finished the frame before. The last of that work is the upscaler (FSR 4, FSR 3.1 or DLSS) and the interface drawn + after it, and nothing the game reads comes out of it. With the setting on, the copies of what the card wrote for the + game go before that last stretch and the game goes on once they are there, so the card finishes the picture while + the next frame is already being prepared. The card runs the same work as before. On an RX 7900 XT with a Ryzen 7 + 5800X3D at 3440x1440, FSR 4 Native AA, Hyrule Field: 50.7 instead of 42.5 fps standing and 56.0 instead of 45.9 along + a route; with `replay_workers = 3` as well, 63.5 instead of 49.4 standing. With FSR 3.1, whose pass is shorter, 51.8 + instead of 47.8 standing. Its pictures matched standing, along the route and through warps to five areas, but it is + new: leave it off to play, and report what you see if you try it. It does nothing without one of those upscalers, + and nothing yet on NVIDIA and Intel cards, where the lazy write-back is on. +- **Experimental: the replay workers take more of a frame off the game's render thread** (`replay_pipe` and + `replay_small` in nx2navi.ini, both 0 by default, both with `replay_workers`; contributed by epicmeme546). With + `replay_pipe = N`, 1 to 4, the thread goes on past up to N groups of draws its workers still record. With + `replay_small = N` as well, a group of N draws or more that is too small to share out goes whole to one worker. On + the same PC standing in Kokiri Forest, with `tail_early = 1`: 60.5 fps with 3 workers alone, 62.2 with 5 workers and + `replay_pipe = 4`, 63.2 with `replay_small = 16` added, against 45.4 with all four settings off. The pictures matched + there and through warps to five areas. + +## Changed + +- **The draw replay on several CPU cores is off by default again** (`replay_workers = 0`). With three workers on by + default in 0.9.1 and 0.9.2 some players' games ended with a lost graphics device; the cause is not found yet. The + settings menu has it under Game, Testing, as *CPU helper threads (experimental)*: 0 to 8, after a restart. +- **The other experimental performance settings have rows there too**, each a switch: *Helpers work ahead* (on is + `replay_pipe = 4`), *Helpers take small jobs* (on is `replay_small = 16`) and *Start next frame early* + (`tail_early`). A row whose setting does nothing without another one is greyed out and says which it needs. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 6d7840e9..bf471353 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -71,11 +71,29 @@ # of latency, and can show a dark or flickering frame now and then: the other thread reads vertex and buffer memory # the game has sometimes rewritten by then. # replay_async = 0 -# Record the large groups of draws of a submission on this many replay worker threads beside the submitting one, 0 to -# 16. Default: 3 (Hyrule Field standing on an RX 7900 XT: 51.7 instead of 43.6 fps). 0: the replay records everything -# on one thread, as before; try it if a picture looks wrong. With 1 a worker records each long run of draws while the -# submitting thread waits for it: no faster, it checks that the pictures stay the same. -# replay_workers = 3 +# EXPERIMENTAL. Record the large groups of draws of a submission on this many replay worker threads beside the +# submitting one, 0 to 16. Default: 0, the replay records everything on one thread. 3 gave 51.7 instead of 43.6 fps +# standing in Hyrule Field on an RX 7900 XT, and more than 3 gave nothing more; some players' games ended with a lost +# graphics device with it on, so it is off unless you turn it on (also in the menu: Game, Testing, CPU helper threads). +# With 1 a worker records each long run of draws while the submitting thread waits for it: no faster, it checks that +# the pictures stay the same. +# replay_workers = 0 +# EXPERIMENTAL. 1 lets the game start on its next frame while the graphics card still runs the end of the last one: the +# upscaler (FSR 4, FSR 3.1 or DLSS) and the interface drawn after it. The graphics card runs the same work, with the +# copies of what it wrote for the game moved before that end, and the game's waits end once those copies and the +# results it reads are there. A frame whose end hands the game anything else runs as before. Default: 0. Standing in +# Hyrule Field on an RX 7900 XT with FSR 4: 50.7 instead of 42.5 fps, and 63.5 instead of 49.4 with replay_workers = 3. +# Does nothing without one of those upscalers, nor with the lazy write-back (the default on NVIDIA and Intel cards). +# Also in the menu: Game, Testing, Start next frame early. +# tail_early = 0 +# EXPERIMENTAL, with replay_workers. replay_pipe = N (1 to 4, default 0): the submitting thread goes on past up to N +# groups its workers still record. replay_small = N (default 0, needs replay_pipe): a group of N draws or more that is +# too small to share out goes whole to one worker. Kokiri Forest standing on an RX 7900 XT with FSR 4, tail_early = 1, +# replay_workers = 5, replay_pipe = 4 and replay_small = 16: 63 instead of 45 fps with all of them off. Also in the +# menu: Game, Testing, as switches: Helpers work ahead (on is replay_pipe = 4) and Helpers take small jobs (on is +# replay_small = 16). +# replay_pipe = 0 +# replay_small = 0 # EXPERIMENTAL, THE GAME CRASHES WITH 1. 1 lifts the virtual core gate: guest threads run without the console's limit # of one thread per core, and the game keeps per-core data without locks. With 1 the game crashes or hangs, # usually at start. Keep 0 to play. diff --git a/runtime/cfg.c b/runtime/cfg.c index f9499989..9f465b1b 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -65,7 +65,7 @@ static const char *const known[] = { "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "POOL_PRESNAP_RUNS", "SUBMIT_FINISH_GPU", "SVC_WAITS", "SNAP_WIDEN", "SUBMIT_FINISH_RESULTS", "SUBMIT_FINISH_KINDS", "FOLIAGE", "MIRROR_STAGE", "FLUSH_SKIP_CLEAN", "MIRROR_HOT", "MESH_EMULATION", "MESH_EMU_MB", "MESH_EMU_TASKS", "MESH_EMU_BATCH", "GUEST_OPT", "LOG_TEX_NEW", "DUMP_KEY", "VEC_INDEX_FIX", "PASS_TIMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", - "REFLEX", "LOW_LATENCY", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", + "REFLEX", "LOW_LATENCY", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_SMALL", "TAIL_EARLY", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_HD", "RUMBLE_HD_MS", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", @@ -135,7 +135,7 @@ static const struct { const char *key, *section; } sections[] = { {"BIND_RIGHT", "input"}, {"BIND_DOWN", "input"}, {"BIND_LSTICK_LEFT", "input"}, {"BIND_LSTICK_UP", "input"}, {"BIND_LSTICK_RIGHT", "input"}, {"BIND_LSTICK_DOWN", "input"}, {"BIND_RSTICK_LEFT", "input"}, {"BIND_RSTICK_UP", "input"}, {"BIND_RSTICK_RIGHT", "input"}, {"BIND_RSTICK_DOWN", "input"}, {"BIND_SHAKE", "input"}, - {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, {"REPLAY_PIPE", "performance"}, + {"ASYNC_PREEMPT", "performance"}, {"REPLAY_ASYNC", "performance"}, {"REPLAY_WORKERS", "performance"}, {"REPLAY_PIPE", "performance"}, {"TAIL_EARLY", "performance"}, {"REPLAY_SMALL", "performance"}, {"FREE_CORES", "performance"}, {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, @@ -284,6 +284,8 @@ static const CfgMeta metas[] = { // Advanced: scheduling {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, {"REPLAY_WORKERS", "0", "0..16", RESTART}, {"REPLAY_PIPE", "0", "0..4", RESTART}, + {"TAIL_EARLY", "0", "0|1", RESTART}, // vk_exec.c "the early tail" + {"REPLAY_SMALL", "0", "0..1024", RESTART}, // vk_exec.c group_start {"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART}, // dynamic resolution (dynres.c): the scene follows the GPU time toward DYN_RES_TARGET milliseconds {"DYN_RES", "0", "0|1", LIVE}, {"DYN_RES_TARGET", "16.7", "1..100", LIVE}, {"DYN_RES_STEP", "45", "8..600", LIVE}, diff --git a/runtime/gfx.h b/runtime/gfx.h index 7a630acd..6aa6a065 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -164,6 +164,10 @@ typedef struct { // As fence_wait, but only until the GPU has executed the submission's own commands: its write-backs to guest // memory and its fence's side effects may follow (vk_sync.c exec_tl). The same as fence_wait where there is no split. int (*exec_wait)(uint64_t value, uint64_t timeout_ns); + // The game's own wait for a fence (nvnSyncWait): fence_wait, or where the back end knows the rest of the value's + // GPU work reads nothing the game can change and hands it nothing, until its results are there (vk_sync.c + // vk_sync_game_wait). + int (*game_wait)(uint64_t value, uint64_t timeout_ns); uint64_t (*fence_done)(void); // highest value whose side effects are written // Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that // has passed or at_ns is 0. diff --git a/runtime/gfx_null.c b/runtime/gfx_null.c index b73010a4..bd4d1b61 100644 --- a/runtime/gfx_null.c +++ b/runtime/gfx_null.c @@ -147,6 +147,7 @@ const GfxBackend gfx_null_backend = { .submit = null_submit, .fence_wait = null_fence_wait, .exec_wait = null_fence_wait, + .game_wait = null_fence_wait, .fence_done = null_fence_done, .present = null_present, .finish = null_finish, diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 74f17a4a..977365f6 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -577,7 +577,7 @@ NVN_API(nvnSyncWait) { int ok; { NVN_BLOCKING(); - ok = b->fence_wait(s->fence, timeout); + ok = b->game_wait(s->fence, timeout); } if (ok) b->cpu_synced(s->fence); ret_x(c, ok ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED); diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index f4ad767d..f99457f1 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1447,8 +1447,17 @@ static void tab_game(float scale) { rows_saves(); // Testing: experimental rows, then the debug menu where the game version has it opt_group_text("Testing"); - // vk_exec.c rw_init reads it at start: the row notes the restart and the bar's Restart does it + // vk_exec.c rw_init and tail_enabled read these at start: each row notes the restart and the bar's Restart does + // it. A row whose setting does nothing without another one (rw_init) is greyed out until that one is set. The two + // rows under the workers are switches: on writes the value measured best (4 groups at once, groups from 16 draws), + // and any other value set in nx2navi.ini shows as on and stays. opt_slider("NX2NAVI_REPLAY_WORKERS", UI_MENU_REPLAY_WORKERS, 0, 8, UI_MENU_REPLAY_WORKERS_DESC, 1, "%.0f"); + const bool workers = cfg_int("NX2NAVI_REPLAY_WORKERS") > 0, pipe = workers && cfg_int("NX2NAVI_REPLAY_PIPE") > 0; + if (!workers) opt_disable_next(UI_MENU_NEEDS_WORKERS); + opt_toggle_value("NX2NAVI_REPLAY_PIPE", UI_MENU_REPLAY_PIPE, UI_MENU_REPLAY_PIPE_DESC, "4", "0"); + if (!pipe) opt_disable_next(workers ? UI_MENU_NEEDS_PIPE : UI_MENU_NEEDS_WORKERS); + opt_toggle_value("NX2NAVI_REPLAY_SMALL", UI_MENU_REPLAY_SMALL, UI_MENU_REPLAY_SMALL_DESC, "16", "0"); + opt_toggle("NX2NAVI_TAIL_EARLY", UI_MENU_TAIL_EARLY, UI_MENU_TAIL_EARLY_DESC); // The debug menu, a tool for testing with English text of its own: off unless switched on here if (dbg_supported()) { const bool on = cfg_bool("NX2NAVI_DEBUG_MENU"); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index bb787a42..11a3f207 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -506,11 +506,23 @@ static const char *const text_en[UI_TEXT_COUNT] = { "where the card has none, keeps video memory within what the driver reports free, waits less for the GPU and " "prepares frames on a thread of their own. Measured on a Radeon RX 580 only: it can show a dark or flickering " "frame now and then, so leave it off on a newer card.", - [UI_MENU_REPLAY_WORKERS] = "Replay workers (experimental)", - [UI_MENU_REPLAY_WORKERS_DESC] = "CPU threads that help translate each frame's draws for the GPU, beside the game's own " - "thread. 3 gave about 20% more frames a second in Hyrule Field on an 8-core CPU; more " - "than 3 gave nothing more. Some players' games crashed with them on: 0 (off) is the " - "safe choice. Takes effect after a restart.", + [UI_MENU_REPLAY_WORKERS] = "CPU helper threads (experimental)", + [UI_MENU_REPLAY_WORKERS_DESC] = "Uses more of your CPU's cores to prepare each frame for the graphics card. 0 turns it " + "off. 3 gave about 20% more frames a second on an 8-core CPU, and 5 a little more " + "with the settings below. Some players' games crashed with this on: set it back to " + "0 if yours does.", + [UI_MENU_REPLAY_PIPE] = "Helpers work ahead (experimental)", + [UI_MENU_REPLAY_PIPE_DESC] = "The game carries on with the rest of the frame while the helper threads are still " + "busy, instead of waiting for them. A small gain: 1 to 2% more frames a second.", + [UI_MENU_REPLAY_SMALL] = "Helpers take small jobs (experimental)", + [UI_MENU_REPLAY_SMALL_DESC] = "Gives the helper threads the smaller batches of drawing work too, not only the large " + "ones. A small gain: about 2% more frames a second.", + [UI_MENU_TAIL_EARLY] = "Start next frame early (experimental)", + [UI_MENU_TAIL_EARLY_DESC] = "The game starts on its next frame while the graphics card is still finishing the " + "current one. The biggest gain of these settings: 19 to 29% more frames a second with " + "FSR 4 on a Radeon RX 7900 XT, the only card it was measured on. It needs TAA, DLSS, " + "FSR 3.1 or FSR 4, and for now an AMD graphics card. Without them it changes nothing.", + [UI_MENU_NEEDS_WORKERS] = "needs CPU helper threads", [UI_MENU_NEEDS_PIPE] = "needs Helpers work ahead", [UI_MENU_DISCORD_RPC] = "Discord Rich Presence", [UI_MENU_DISCORD_RPC_DESC] = "Shows \"Playing NX2navi\" with the play time in your Discord status while the " "game runs. discord_app_id and discord_image in nx2navi.ini choose the application.", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index b5a1d41f..27801fd9 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -175,7 +175,10 @@ typedef enum { // Menu tab UI_MENU_MENU_LANG, UI_MENU_MENU_LANG_DESC, UI_MENU_MENU_LANG_AUTO, UI_MENU_DISCORD_RPC, UI_MENU_DISCORD_RPC_DESC, UI_MENU_LOW_END_GPU, UI_MENU_LOW_END_GPU_DESC, - UI_MENU_REPLAY_WORKERS, UI_MENU_REPLAY_WORKERS_DESC, // the Game tab, English only + // the Game tab's Testing rows (ui/menu.cpp tab_game), English only + UI_MENU_REPLAY_WORKERS, UI_MENU_REPLAY_WORKERS_DESC, UI_MENU_REPLAY_PIPE, UI_MENU_REPLAY_PIPE_DESC, + UI_MENU_REPLAY_SMALL, UI_MENU_REPLAY_SMALL_DESC, UI_MENU_TAIL_EARLY, UI_MENU_TAIL_EARLY_DESC, + UI_MENU_NEEDS_WORKERS, UI_MENU_NEEDS_PIPE, // why a row is greyed out // Language names, each in its own language and the same in every row (game language and menu language lists) UI_MENU_LANGNAME_EN, UI_MENU_LANGNAME_JA, UI_MENU_LANGNAME_FR, UI_MENU_LANGNAME_DE, UI_MENU_LANGNAME_IT, diff --git a/runtime/vk.h b/runtime/vk.h index aff9a232..9be27f5d 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -405,6 +405,23 @@ int vk_mem_take_live(void); // for the newest such submission up to the one it waits for. enum { VK_RES_EVENT = 1, VK_RES_COUNTER = 2, VK_RES_OQ = 4, VK_RES_WRITE = 8, VK_RES_WRITEBACK = 16, VK_RES_KINDS = 5 }; void vk_exec_note_results(int kind); // VK_RES_* +// The early tail (vk_exec.c "the early tail", NX2NAVI_TAIL_EARLY). vk_exec_tail: the replay is recording a submission's +// tail. vk_sync.c: the tail's command buffer set aside for the write-backs' one (hold), then the part with the +// write-backs handed over with the executed timeline's signal and the tail put behind it (release); whether a part can +// take that order now (ok); the game's wait for a fence (game_wait); for a value whose tail the GPU still runs, about +// how long that takes (tail_pending_ns, else 0). +struct VkExec; +void vk_exec_tail_begin(struct VkExec *ex); +int vk_exec_tail(void); +void vk_exec_tail_ub(uint64_t fence, uint64_t addr, const void *data, uint32_t size); // vk_exec_xfer.c +int vk_sync_tail_ok(void); +VkCommandBuffer vk_sync_tail_hold(void); +VkCommandBuffer vk_sync_tail_release(void); +int vk_sync_game_wait(uint64_t value, uint64_t timeout_ns); +uint64_t vk_sync_tail_pending_ns(uint64_t value); +int vk_mem_live_swap(int v); // vk_memory.c: the note of a late read of guest memory, exchanged +uint64_t vk_mem_gpu_wrote_count(void); // vk_memory.c: GPU writes of pool memory noted so far +int vk_mem_wb_lazy(void); // vk_memory.c: the lazy write-back runs int vk_exec_take_results(void); // the kinds noted since the last take void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over void vk_mem_lazy_handed(void); diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 6b40b6f0..6963c991 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -936,6 +936,7 @@ const GfxBackend gfx_vulkan_backend = { .submit = vk_be_submit, .fence_wait = vk_sync_fence_wait, .exec_wait = vk_be_exec_wait, + .game_wait = vk_sync_game_wait, .fence_done = vk_sync_fence_done, .present = vk_be_present, .frame_start = vk_be_frame_start, diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index 50685936..8810916e 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -2170,6 +2170,7 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { const VkExecTarget *t = &D.color, *z = &D.depth; if (!dst) dst = t; uint32_t w = t->width, h = t->height, ow = dst->width, oh = dst->height; + vk_exec_tail_begin(ex); // NX2NAVI_TAIL_EARLY: from here on the frame's tail, in a command buffer of its own VkCommandBuffer cb = ex->cb; vk_exec_barrier(ex); // what drew the scene is done (no pass is open here) vk_gpu_gap_note(VK_GAP_UPSCALER, (uint64_t)ow * oh); diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index a38050b0..439d2bf1 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1122,6 +1122,59 @@ static int split_n = -1, split_idle, split_in_pass, split_draws, split_at, part_ static uint64_t split_ns, part_t0; static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits, n_cuts; +// ---- the early tail (NX2NAVI_TAIL_EARLY, default 0) +// Standing in Hyrule Field with FSR 4 (RX 7900 XT, 2026-11-13) a frame's main submission ends with one part of 4.7 ms +// of GPU time, the temporal upscaler and the interface drawn after it, and one of 1.4 ms, the write-backs of what the +// shaders wrote into VRAM mirrors. The game waits for all of it before it writes the next frame's data (the wait after +// a frame's first submission, nvn_api_core.c, and its own fence waits), about 7 ms in which the replay thread has +// nothing to record. What the game reads of the submission is done before the upscaler runs: the occlusion and +// counter reports, the event values, and the write-backs' bytes, which only came last. After it the stream holds two +// timestamp reports, and inline uniform updates for the interface's draws (64 bytes into one mirrored block, some +// thirty times), and nothing else the CPU sees. +// So from the upscaler on (vk_dlaa.c evaluate) the replay records the tail into a command buffer of its own +// (vk_sync_cut), without splits or parallel groups. An inline uniform update of a mirrored block in it takes no +// write-back: the GPU copies the bytes into the mirror in stream order as ever, and guest memory gets them from the +// CPU with the submission's results (vk_exec_draw.c op_update_ub, vk_exec_xfer.c vk_exec_tail_ub). At the submission's +// end, when the tail turned out to hand the CPU nothing but timestamps (vk_exec_tail_results), to write no other pool +// memory (vk_mem_gpu_wrote_count) and to bind no memory the GPU reads from the game's own pages (vk_mem_live_swap: a +// pool without a mirror, a device address table), +// the write-backs are recorded into a command buffer that goes to the queue before the tail's, with the executed +// timeline's signal (vk_sync_tail_hold, vk_sync_tail_release): the GPU runs the same commands, the write-backs before +// the tail. The mirror copies the tail's first uses queued go with that part too, so they read the game's pages before +// the signal. The game's waits then end at that signal and the results the game reads (vk_sync.c vk_sync_exec_wait, +// vk_sync_game_wait), and the GPU runs the tail while the game and the replay are at the next frame. A timestamp +// reported in the tail is written then with the time the tail is expected to end at (vk_exec_xfer.c counter_cb), as +// the game measures the frame's GPU time with it. A tail that fails a test goes as before, the write-backs after it. +// Measured there, 150 s runs, each pair in one batch: 63.5 against 49.4 fps with three replay workers; without them +// 50.7 against 42.5 standing and 56.0 against 45.9 along the field route, and with FSR 3.1, whose tail the GPU runs +// for 1.2 ms where FSR 4's takes 3.5, 51.8 against 47.8 and 54.9 against 52.3. All but a few of the tails passed the +// tests there; in area loads and some places a tail hands the CPU results and goes as before. The pictures matched +// those without it standing, along the route and through warps to five areas. Not measured with the lazy write-back +// (NVIDIA's and Intel's default), where a tail's uniform updates are noted as writes and no tail goes early. +// Not with shadow storage, emulated mesh shaders, a frame dump or a pass capture, nor inside a chunk or a continuation. +static struct { + int on; // the submission's tail is being recorded + int live0; // vk_mem_live_swap's note from before it + uint64_t wrote0; // vk_mem_gpu_wrote_count when it began + // since the last perf line: tails; those that went early; those refused by results, writes, late reads, other + uint64_t n, early, why[4]; +} TL; +int vk_exec_tail_results(void); // vk_exec_xfer.c +void vk_sync_tail_stats(uint64_t *tails, uint64_t *waits, double *ms); +int vk_gpu_passes_capturing(void); +int vk_mesh_shader_native(void); // vk_device.c +int vk_exec_tail(void) { return TL.on; } + +static int tail_enabled(void) { + static int on = -1; + if (on < 0) { + on = cfg_bool("NX2NAVI_TAIL_EARLY"); + if (on) + rt_log("vk: the write-backs go before a frame's tail, and the game's waits end there (NX2NAVI_TAIL_EARLY)"); + } + return on; +} + int vk_exec_split_now(VkExec *ex) { int in_pass = ex->in_pass; if (in_pass) vk_exec_pass_end(ex); @@ -1153,7 +1206,7 @@ static void maybe_split(VkExec *ex) { split_idle ? "; a first part at once while the GPU is idle" : ""); split_at = split_n ? split_n : INT_MAX; } - if (!split_n) return; + if (!split_n || TL.on) return; // the tail stays one command buffer (the early tail) if (Q.drained < Q.out) { // groups the submitting thread went on past go into the part first, once done if (__atomic_load_n(&Q.done, __ATOMIC_ACQUIRE) < Q.out) return; // split_draws stays: a later op asks again group_drain(ex, PW_SPLIT); @@ -1784,7 +1837,7 @@ static void op_one(VkExec *ex, const op_hdr *h) { } // ---- replay workers (NX2NAVI_REPLAY_WORKERS, default 0: the replay records every op on the submitting thread; the -// Game tab's Replay workers row sets it) +// Game tab's CPU helper threads row sets it) // 3 workers: on an RX 7900 XT with a Ryzen 7 5800X3D, Hyrule Field standing went from 41.2 to 50.4 fps (2026-11-12, one // batch), with the same pictures there and in Kokiri Forest, Kakariko, Goron City, a graveyard house and Gerudo // Valley; 5 or 7 workers measured the same as 3. Default 0 again after players' games crashed with 3 on by default @@ -1877,6 +1930,8 @@ static const char *const pw_name[PW_N] = {"next group", "op", "pass hooks", "tur static struct { int n; // workers; 0 off, -1 not read yet int pipe; // NX2NAVI_REPLAY_PIPE: the groups out at once at most, 0 none + int small; // NX2NAVI_REPLAY_SMALL: the fewest draws of a group one worker takes whole + uint64_t small_groups, small_draws; // such groups and their draws since the last perf line void *help_dc; // the submitting thread's draw context for chunks it takes while it waits RwWorker *w[RW_MAX]; const op_hdr *skip_to[CALL_DEPTH + 1]; // per call depth: the walk read ahead to this op, a group too small @@ -2008,6 +2063,10 @@ static void rw_init(void) { int pipe = cfg_int("NX2NAVI_REPLAY_PIPE"); // the groups out at once RW.pipe = RW.n <= 0 || pipe <= 0 ? 0 : pipe > LIVE_GROUPS ? LIVE_GROUPS : pipe; if (RW.pipe) RW.help_dc = vk_exec_draw_ctx_new(); + RW.small = RW.pipe ? cfg_int("NX2NAVI_REPLAY_SMALL") : 0; + if (RW.small > 0) + rt_log("vk: a group of %d draws or more goes whole to one replay worker (NX2NAVI_REPLAY_SMALL)", RW.small); + else RW.small = 0; if (RW.pipe) rt_log("vk: groups of %d draws or more are recorded by %d replay worker%s while the submitting thread goes on " "past them, %d at once at most (NX2NAVI_REPLAY_WORKERS, NX2NAVI_REPLAY_PIPE)", 2 * CHUNK_DRAWS, RW.n, @@ -2319,7 +2378,14 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { int n = (int)(draws / CHUNK_DRAWS); if (n > RW.n + 1) n = RW.n + 1; if (n > GROUP_CHUNKS) n = GROUP_CHUNKS; - if (n < 2 || vk_dump_armed || !vk_sync_append_room(n)) { + // NX2NAVI_REPLAY_SMALL=N (default 0, with NX2NAVI_REPLAY_PIPE): a group of N draws or more that is too small to cut + // goes whole to one worker, and the submitting thread goes on with what follows it without taking the chunk + // itself. Kokiri Forest standing, 5 workers, 4 groups out, the early tail on (2026-11-13): about 700 draws a frame + // were in such groups, 3.6 ms of the submitting thread; with 16 the replay took 10.2 instead of 11.0 ms a frame + // (63.2 against 62.2 fps: the game then waits longer for the GPU, which is busy 83% of the frame there). + int small = RW.pipe && RW.small && n < 2 && draws >= (uint32_t)RW.small; + if (small) n = 1; + if (n < 1 + !small || vk_dump_armed || TL.on || !vk_sync_append_room(n)) { RW.skip_to[depth] = end, RW.serial_draws += draws, RW.small_end[why] += draws; return h; } @@ -2393,7 +2459,8 @@ static const op_hdr *group_start(VkExec *ex, const op_hdr *h, int depth) { // the submitting thread takes chunks too, as the workers wake up, and goes on once none is left to take: the // GPU waits for the group's part until its last chunk is done rw_behind = 1; - group_take(RW.n); + if (small) RW.small_groups++, RW.small_draws += draws; // a worker takes it: this thread goes on at once + else group_take(RW.n); uint64_t to = kern_now_ns(); RW.front_ns += tf - t0, RW.own_ns += to - tf, RW.wait_ns += to - t0; return end; @@ -2511,6 +2578,25 @@ static void rec_claim(void) { } } +// The tail begins here (the early tail): what was recorded so far waits in the chain, the tail gets a command buffer of +// its own. Not in a chunk or a continuation, nor with groups still out: the cut needs every piece of the part. +void vk_exec_tail_begin(VkExec *ex) { + if (!tail_enabled() || TL.on || ex != x_cur || rw_chunk || evo_in) return; + if (Q.drained != Q.out) group_drain(ex, PW_OP); // the cut needs every piece of the part + if (!vk_sync_wb_split() || !vk_mesh_shader_native() || vk_dump_armed || vk_gpu_passes_capturing()) return; + if (!vk_sync_tail_ok()) return; + if (ex->in_pass) vk_exec_pass_end(ex); + vk_exec_barrier(ex); // the command buffer so far ends behind a full barrier, every image back in GENERAL + VkCommandBuffer cb = vk_sync_cut(); + if (!cb) return; + ex->cb = cb; + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + n_cuts++; + TL.on = 1, TL.live0 = vk_mem_live_swap(0), TL.wrote0 = vk_mem_gpu_wrote_count(); + vk_exec_tail_results(); +} + int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): nothing is recorded uint64_t in_ns = kern_now_ns(); @@ -2528,6 +2614,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t nvn_pins_sweep(nvn_submit_count(), keep, 9); ex->cb = cb; ex->fence = fence; + TL.on = 0; ex->recorded = 0; ex->in_pass = 0; ex->mark = NULL; @@ -2594,12 +2681,47 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t // NX2NAVI_WB_SPLIT: the write-backs in a part of their own, after one that signals the executed timeline // (vk_sync.c exec_tl): the wait after a frame's first submission then ends when the submission's passes are // through, not its write-backs. Off by default: vk_sync.c says what else that wait then leaves out. - if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) { + int tail = 0; // the early tail: the write-backs go before the tail, with the executed timeline's signal + if (TL.on) { + int live = vk_mem_live_swap(0), res = vk_exec_tail_results(); + vk_mem_live_swap(live | TL.live0); // the submission's own note, as it would have been + int wrote = vk_mem_gpu_wrote_count() != TL.wrote0; + int other = !synced || !ex->recorded || vk_dump_armed || vk_gpu_passes_capturing() || !vk_sync_tail_ok(); + TL.on = 0, TL.n++; + if (res) TL.why[0]++; + else if (wrote) TL.why[1]++; + else if (live) TL.why[2]++; + else if (other || !vk_exec_pool_split(ex->fence)) TL.why[3]++; + else tail = 1, TL.early++; + if (tail) { + ex->cb = vk_sync_tail_hold(); + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + } + } + if (!tail && synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && + vk_exec_pool_split(ex->fence)) { ex->cb = vk_sync_split_exec(); ex->dirty = VK_EXEC_DIRTY_ALL; vk_exec_draw_new_cb(); } if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory + if (tail) { + ex->cb = vk_sync_tail_release(); + part_start = ex->recorded; // the last command buffer holds nothing: the tail is in the chain before it + } + if ((rt_trace & RT_TRACE_PERF) && TL.n >= 600) { + uint64_t tails, waits; + double ms; + vk_sync_tail_stats(&tails, &waits, &ms); + rt_log("perf: early tail: %llu tails, %llu with the write-backs before them (refused: %llu handed the CPU " + "results, %llu wrote pool memory, %llu read guest memory late, %llu other); %llu waits of the game " + "ended before their tail, which the GPU runs for about %.2f ms", + (unsigned long long)TL.n, (unsigned long long)TL.early, + (unsigned long long)TL.why[0], (unsigned long long)TL.why[1], (unsigned long long)TL.why[2], + (unsigned long long)TL.why[3], (unsigned long long)waits, ms); + TL.n = TL.early = 0, memset(TL.why, 0, sizeof TL.why); + } views_release(ex, fence); ex->cb = VK_NULL_HANDLE; if (n) n_subs++; @@ -2696,8 +2818,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t memset(RW.pw_n, 0, sizeof RW.pw_n), memset(RW.pw_ns, 0, sizeof RW.pw_ns); memset(RW.pd_op, 0, sizeof RW.pd_op); RW.behind_ns = 0; + if (RW.small) + rt_log("perf: replay workers: groups one worker took whole (NX2NAVI_REPLAY_SMALL): %.2f per " + "submission, %.1f draws each", (double)RW.small_groups / (double)n_subs, + RW.small_groups ? (double)RW.small_draws / (double)RW.small_groups : 0.0); } } + RW.small_groups = RW.small_draws = 0; RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; n_subs = n_parts = n_idle_parts = n_pass_splits = n_cuts = 0; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index c6034e6e..517392cd 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -7371,7 +7371,14 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { VK_MEM_PICK(sp); VkBufferCopy cp = {o, sp.offset, u->size}; vkCmdCopyBuffer(ex->cb, b, sp.buffer, 1, &cp); - vk_mem_gpu_wrote(&sp, u->size); + // In a submission's tail (vk_exec.c "the early tail") an update of a mirrored block reaches guest memory with the + // results the game reads, from the bytes here, and takes no write-back: one after the tail would land when the + // game is already writing the next frame's data. Not with the lazy write-back, where the block's pages may be + // armed and a store into them would wait for the GPU (memory.c lazy_fault): the update is noted as ever there, + // and the tail goes after the write-backs as before. + if (vk_exec_tail() && !vk_mem_wb_lazy() && vk_mem_mirrored(u->dst, u->size)) + vk_exec_tail_ub(ex->fence, u->dst, u->data, u->size); + else vk_mem_gpu_wrote(&sp, u->size); vk_usnap_drop(u->dst, u->size); ex->recorded++; vk_mem_count(sp.cls, VK_MUSE_GPU_WRITE, u->size); diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 79026a89..6a15e19a 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -67,12 +67,32 @@ typedef struct { uint64_t addr, size; int back; } Range; // back: written into // for the newest such submission). Write-backs of the mirrors note it in vk_memory.c; the lazily written ones fetch on // the CPU's first access. static volatile int32_t ex_results; // VK_RES_* bits +static int tail_results; // those noted while the submission's tail records (vk_exec.c "the early tail") +int vk_exec_tail_results(void) { int r = tail_results; tail_results = 0; return r; } void vk_exec_note_results(int kind) { + if (vk_exec_tail()) tail_results |= kind; __atomic_fetch_or(&ex_results, kind, __ATOMIC_RELAXED); vk_gpu_passes_result(kind); // NX2NAVI_GPU_PASSES: where in the submission it is done } int vk_exec_take_results(void) { return __atomic_exchange_n(&ex_results, 0, __ATOMIC_RELAXED); } +// An inline uniform update in a submission's tail (vk_exec_draw.c op_update_ub, vk_exec.c "the early tail"): the GPU +// copies the bytes into the pool's mirror where the stream has the update, and guest memory gets them here, with the +// results the game reads, in place of a write-back. +typedef struct { uint64_t addr; uint32_t size; uint8_t data[]; } UbWrite; +static void ub_write_cb(void *arg) { + UbWrite *w = arg; + memcpy((void *)(uintptr_t)w->addr, w->data, w->size); + free(w); +} +void vk_exec_tail_ub(uint64_t fence, uint64_t addr, const void *data, uint32_t size) { + UbWrite *w = malloc(sizeof *w + size); + if (!w) rt_fatal("vk: out of memory"); + w->addr = addr, w->size = size; + memcpy(w->data, data, size); + vk_completion_add_result(fence, ub_write_cb, w); +} + static void guest_written_cb(void *arg) { Range *r = arg; if (r->back) vk_gpu_written_back(r->addr, r->size); @@ -114,8 +134,9 @@ static void event_cb(void *arg) { // NVNcounterData = {counter, timestamp}; timestamps are Tegra 19.2 MHz ticks (the base of GetSystemTick). // keep: the 4-byte quarters of the record the write leaves as they are (bit n: bytes 4n to 4n + 3), 0 none. -static void write_counter(uint64_t addr, uint64_t counter, unsigned keep) { - uint64_t rec[2] = {counter, (uint64_t)((unsigned __int128)kern_now_ns() * 12 / 625)}; +// ahead_ns: the timestamp is that much past now (a tail timestamp the GPU has not reached yet, counter_cb), else 0. +static void write_counter(uint64_t addr, uint64_t counter, unsigned keep, uint64_t ahead_ns) { + uint64_t rec[2] = {counter, (uint64_t)((unsigned __int128)(kern_now_ns() + ahead_ns) * 12 / 625)}; if (!keep) { memcpy((void *)(uintptr_t)addr, rec, sizeof rec); return; @@ -127,14 +148,16 @@ static void write_counter(uint64_t addr, uint64_t counter, unsigned keep) { // Reports other than samples passed (timestamps; the statistics counters are not modelled) write a counter of 0. The // counter word matters: Ocarina writes {1, 0} into a record from the CPU, reports TIMESTAMP_TOP into it and gates // clears on it differing from a zero record (mode 1), clears that must not run. -static void report_write(uint64_t addr, uint64_t seq, uint64_t counter); // below, with the pending table +static void report_write(uint64_t addr, uint64_t seq, uint64_t counter, uint64_t ahead_ns); // below, with the pending table -typedef struct { uint64_t addr, seq; } CounterWrite; +// tail: the submission's value for a timestamp reported in its tail (vk_exec.c "the early tail"), else 0 +typedef struct { uint64_t addr, seq, tail; } CounterWrite; static void counter_cb(void *arg) { CounterWrite *w = arg; if (vk_sync_effect_late()) __atomic_fetch_add(&late_counters, 1, __ATOMIC_RELAXED); - report_write(w->addr, w->seq, 0); + // A timestamp of the tail, written while the GPU still runs the tail: the time the tail is expected to end at + report_write(w->addr, w->seq, 0, w->tail ? vk_sync_tail_pending_ns(w->tail) : 0); free(w); } @@ -308,7 +331,7 @@ static unsigned clear_quarters(const Pending *e) { return e->clr_seq >= e->seq ? // Completion thread, or the thread whose wait runs the results: report seq to addr goes into guest memory, but for // the quarters a small clear replayed after it has filled. -static void report_write(uint64_t addr, uint64_t seq, uint64_t counter) { +static void report_write(uint64_t addr, uint64_t seq, uint64_t counter, uint64_t ahead_ns) { int lk = __atomic_load_n(&clr_any, __ATOMIC_ACQUIRE); unsigned keep = 0; if (lk) { @@ -317,7 +340,7 @@ static void report_write(uint64_t addr, uint64_t seq, uint64_t counter) { if (e && seq <= e->clr_seq) keep = e->clr_mask; if (keep) __atomic_fetch_add(&CS.kept, 1, __ATOMIC_RELAXED); } - write_counter(addr, counter, keep); + write_counter(addr, counter, keep, ahead_ns); if (lk) plat_mutex_unlock(&clr_lock); pending_written(addr, seq); } @@ -644,7 +667,7 @@ static void oq_cb(void *arg) { if (s->lost && !oq_count) oq_count = 1; if (s->addr) { if (vk_sync_effect_late()) __atomic_fetch_add(&late_counts, 1, __ATOMIC_RELAXED); - report_write(s->addr, s->seq, oq_count); + report_write(s->addr, s->seq, oq_count, 0); } } free(s->q), free(s); @@ -893,10 +916,13 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { clear_note_report(p->addr); if (samples) oq_report(ex, p->addr); else { + // a timestamp in the tail is the one result a tail may hand the CPU: its time is written ahead (counter_cb) + int ts_tail = vk_exec_tail() && p->type == NVN_COUNTER_TYPE_TIMESTAMP, keep = tail_results; vk_exec_note_results(VK_RES_COUNTER); + if (ts_tail) tail_results = keep; CounterWrite *a = malloc(sizeof *a); if (!a) rt_fatal("vk: out of memory"); - *a = (CounterWrite){p->addr, report_seq}; + *a = (CounterWrite){p->addr, report_seq, ts_tail ? ex->fence : 0}; vk_completion_add_result(ex->fence, counter_cb, a); } Pending *e = pending_slot(p->addr, 1); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 91e75c13..2f2cf341 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -3712,7 +3712,11 @@ static uint64_t wb_skip(void) { } static int64_t m_wb_skipped; // bytes wb_skip kept in mirrors since the last summary +static volatile uint64_t n_gpu_wrote; // vk_mem_gpu_wrote_count +uint64_t vk_mem_gpu_wrote_count(void) { return __atomic_load_n(&n_gpu_wrote, __ATOMIC_RELAXED); } + static void gpu_wrote(const VkMemSpan *sp, uint64_t size, int shader) { + __atomic_fetch_add(&n_gpu_wrote, 1, __ATOMIC_RELAXED); PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size || !wb_lazy()) { uint64_t skip = r && r->mstate == MS_ON && sp->buffer == r->mbuf ? wb_skip() : 0; @@ -4087,6 +4091,9 @@ static uint64_t capture_cap(void) { } void vk_mem_note_live(void) { __atomic_store_n(&mirror_live, 1, __ATOMIC_RELAXED); } +int vk_mem_live_swap(int v) { return __atomic_exchange_n(&mirror_live, v, __ATOMIC_RELAXED); } +static int wb_lazy(void); +int vk_mem_wb_lazy(void) { return wb_lazy(); } // the lazy write-back runs (vk_exec_draw.c op_update_ub) int vk_mem_take_live(void) { int live = __atomic_exchange_n(&mirror_live, 0, __ATOMIC_RELAXED); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 9cdc2868..d864e659 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -63,6 +63,16 @@ static uint64_t live_last_of[LIVE_RING], live_last; // per value (index & LIVE_RING - 1): the newest value up to it whose completion hands results to the CPU (vk_exec_take_results) static uint64_t res_of[LIVE_RING], res_last; static uint8_t res_kinds[LIVE_RING]; // per value: its VK_RES_* kinds +// The early tail (vk_exec.c "the early tail"), per value under st_lock: early_of holds the value when its part with +// the write-backs signalled the executed timeline before its tail; empty_of when the submission handed the GPU nothing +// and the CPU no results (a submission held at its first event wait: Ocarina's first of a frame in the field). +static uint64_t early_of[LIVE_RING], empty_of[LIVE_RING]; +static VkCommandBuffer tail_cb; // sub_lock: the tail's command buffer between hold and release +// sub_lock: this submission's tail went behind its write-backs; the parts of it handed over before the last +static int tail_early, sub_parts; +static volatile uint64_t tail_ns; // about how long the GPU takes for a tail (a moving average) +static volatile uint64_t tail_t0, tail_val; // the first early return for tail_val, for tail_ns +static volatile uint64_t tail_n, tail_waits; // since the last vk_sync_tail_stats: early tails; waits they ended static int res_mask = 0x1f; // NX2NAVI_SUBMIT_FINISH_KINDS: the kinds the wait counts static int stop; static Pending *queue; // side effects in the order they were added @@ -920,6 +930,12 @@ static void completion_main(void *arg) { // finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each // checked against the timeline itself) is nothing they read, and it takes cmd_lock, which the upload // path holds, so done waited behind uploads. + uint64_t tv = __atomic_load_n(&tail_val, __ATOMIC_ACQUIRE); + if (tv && reached >= tv) { // the tail a wait went ahead of is through: how long it took from there + uint64_t d = kern_now_ns() - __atomic_load_n(&tail_t0, __ATOMIC_RELAXED), a = tail_ns; + if (d < 50000000ull) __atomic_store_n(&tail_ns, a ? (a * 7 + d) / 8 : d, __ATOMIC_RELAXED); + __atomic_compare_exchange_n(&tail_val, &tv, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED); + } uint64_t was = done; __atomic_store_n(&done, reached, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_done); @@ -1197,6 +1213,63 @@ VkCommandBuffer vk_sync_worker_begin(int w) { void vk_sync_worker_end(VkCommandBuffer cb) { vkEndCommandBuffer(cb); } static VkCommandBuffer split_part(int exec); + +// ---- the early tail (vk_exec.c "the early tail"). The replay cut the part where the tail began (vk_sync_cut), so the +// chain holds what came before and the command buffer being recorded is the tail's. vk_sync_tail_hold sets that one +// aside, ended and timed as a part of its own, and begins the one the write-backs are recorded into; +// vk_sync_tail_release hands the chain and the write-backs to the queue with the executed timeline's signal +// (split_part), and puts the tail into the chain for the submission's last part, which signals the timeline. The queue +// runs them in that order, so the GPU's work is what it was with the write-backs moved before the tail. +int vk_sync_tail_ok(void) { return !pre_cb && !tail_cb && exec_tl && exec_sig < cur_fence && nchain + 2 <= CHAIN_MAX; } + +VkCommandBuffer vk_sync_tail_hold(void) { + VkCtx *c = vk_ctx(); + VkCommandBuffer cb = slots[cur_slot].cb; + vk_gpu_passes_part(cb, 1); + ts_end(cb, cur_fence); + if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); + tail_cb = cb; + CmdSlot *s = take_slot(c, cur_fence); // the tail keeps its slot until the value is reached + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(s->cb, &bi); + ts_begin(s->cb); + rec_begin(s->cb, cur_fence); + vk_gpu_passes_part(s->cb, 0); + return s->cb; +} + +VkCommandBuffer vk_sync_tail_release(void) { + VkCommandBuffer cb = split_part(1); + chain[nchain++] = tail_cb; + tail_cb = VK_NULL_HANDLE; + tail_early = 1; + return cb; +} + +// The newest value up to `value` whose GPU work a wait of the game is for: the empty submissions after it are passed +// over. *early: that one's write-backs went before its tail, so the executed timeline reaches it there. +static uint64_t wait_target(uint64_t value, int *early) { + plat_mutex_lock(&st_lock); + uint64_t t = value; + while (t > done && empty_of[t & (LIVE_RING - 1)] == t) t--; + *early = t > done && early_of[t & (LIVE_RING - 1)] == t; + plat_mutex_unlock(&st_lock); + return t; +} + +uint64_t vk_sync_tail_pending_ns(uint64_t value) { + plat_mutex_lock(&st_lock); + int pending = value > done && early_of[value & (LIVE_RING - 1)] == value; + plat_mutex_unlock(&st_lock); + return pending ? __atomic_load_n(&tail_ns, __ATOMIC_RELAXED) : 0; +} + +void vk_sync_tail_stats(uint64_t *tails, uint64_t *waits, double *ms) { + *tails = __atomic_exchange_n(&tail_n, 0, __ATOMIC_RELAXED); + *waits = __atomic_exchange_n(&tail_waits, 0, __ATOMIC_RELAXED); + *ms = (double)__atomic_load_n(&tail_ns, __ATOMIC_RELAXED) / 1e6; +} VkCommandBuffer vk_sync_split(void) { return split_part(0); } // The part that ends here is the last with the submission's own commands: it signals the executed timeline. VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); } @@ -1218,6 +1291,7 @@ static VkCommandBuffer split_part(int exec) { si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence; ts_submitted(cbi, ncb); queue_submit(c, &si, 0); // held or handed to the queue (submit batching) + sub_parts++; vk_mem_lazy_handed(); vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted CmdSlot *s = take_slot(c, cur_fence); // the submitted part keeps its slot until the value is reached @@ -1259,6 +1333,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { vk_mem_dirty_submit(); // NX2NAVI_TRACE=perf: what a mirror would upload before this submit uint64_t fence = __atomic_load_n(&issued, __ATOMIC_ACQUIRE) + 1; cur_fence = fence; + tail_early = sub_parts = 0; CmdSlot *s = take_slot(c, fence); VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; @@ -1318,6 +1393,9 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { if (live) live_last = fence; live_last_of[fence & (LIVE_RING - 1)] = live_last; res_kinds[fence & (LIVE_RING - 1)] = (uint8_t)res; + early_of[fence & (LIVE_RING - 1)] = tail_early ? fence : 0; + empty_of[fence & (LIVE_RING - 1)] = !own && !sent && !sub_parts && !res ? fence : 0; + if (tail_early) __atomic_fetch_add(&tail_n, 1, __ATOMIC_RELAXED); if (res & res_mask) res_last = fence; res_of[fence & (LIVE_RING - 1)] = res_last; __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); @@ -1530,8 +1608,13 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { if (vk_device_is_lost()) return __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value; if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns); if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; + // The early tail: a value whose GPU work is that of an earlier submission with its write-backs before its tail (the + // submissions since handed the GPU nothing) is reached where that one's executed signal is, its tail still to run. + int early = 0; + uint64_t asked = value, target = wb_results ? wait_target(value, &early) : value; + if (early) value = target; uint64_t p0 = perf_begin(); - hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now + hold_release(asked, 0, 0); // a held submission the waiter needs goes to the queue now VkCtx *c = vk_ctx(); VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, .pSemaphores = &exec_tl, .pValues = &value}; @@ -1545,6 +1628,13 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { if (r == VK_SUCCESS && wb_results) results_run(value); perf_end(PERF_FENCE, p0); perf_gpu_wait(p0); + if (r == VK_SUCCESS && early && __atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) { + __atomic_fetch_add(&tail_waits, 1, __ATOMIC_RELAXED); + uint64_t none = 0; // the first wait back for this tail starts its clock (the completion thread ends it) + if (__atomic_load_n(&tail_val, __ATOMIC_ACQUIRE) != value && + __atomic_compare_exchange_n(&tail_val, &none, value, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) + __atomic_store_n(&tail_t0, kern_now_ns(), __ATOMIC_RELAXED); + } if (r == VK_SUCCESS) { __atomic_fetch_add(&exw_n, 1, __ATOMIC_RELAXED); if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) { @@ -1556,6 +1646,19 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { return r == VK_SUCCESS; } +// The game's own wait for a fence (nvn_api_core.c nvnSyncWait): the fence's wait, or with the early tail the wait for +// the results of the submission the value's GPU work belongs to (vk_sync_exec_wait), when that one's tail is all the +// GPU still has of it: the tail reads nothing the game can change and hands it nothing. +int vk_sync_game_wait(uint64_t value, uint64_t timeout_ns) { + vk_sync_start(); + if (vk_sync_wb_split() && wb_results && __atomic_load_n(&done, __ATOMIC_ACQUIRE) < value) { + int early = 0; + wait_target(value, &early); + if (early) return vk_sync_exec_wait(value, timeout_ns); + } + return vk_sync_fence_wait(value, timeout_ns); +} + uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); } uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }