From bfa486c2aa98b1b16d9b68a6141ae1269e5939c7 Mon Sep 17 00:00:00 2001 From: Vasilis Manetas Date: Thu, 12 Nov 2026 16:26:26 +0100 Subject: [PATCH] Hand every queue submission to the driver's own submit thread on Linux, so RADV's vkQueueSubmit2 leaves the replaying thread. Mesa's Vulkan drivers keep timeline semaphores as kernel sync objects, and their queues submit on the calling thread unless a submission waits on a value not submitted yet. The replay hands a part of the frame to the queue each time it splits one off (vk_exec.c maybe_split), so every such vkQueueSubmit2, with the command stream's checks and the kernel's ioctl, ran inside the replay, the thread that limits the frame rate on an RX 6800 XT under RADV. MESA_VK_ENABLE_SUBMIT_THREAD=1 sends every submission to a thread of the queue's own. vk_device.c mesa_submit_thread sets it before any thread can load the driver, unless the environment sets it already. submit_thread in nx2navi.ini (NX2NAVI_SUBMIT_THREAD) is 1 by default on Linux and 0 on Windows; 0 leaves the driver's own choice. On a Mesa driver the log says which one applies. Kokiri Forest, 0.9.1, RX 6800 XT, Mesa 26.2.6, 3 pairs of 45 s runs of one build with the setting on and off: the replay took 13.9 ms a frame against 15.0, lower in every pair; frames over 25 ms fell from 910 to 406; the frame rate went from 40.0 to 40.8 fps, within the runs' noise. --- nx2navi.ini.example | 4 ++++ runtime/cfg.c | 9 +++++++-- runtime/vk_device.c | 26 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index fe04570f..725cde21 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -89,6 +89,10 @@ # recorder_core = 0 # Submissions handed to the GPU queue together in one call, 1 to 16 (1 = each one at once). # submit_batch = 4 +# Mesa's Vulkan drivers (RADV, ANV, NVK): every submission to the GPU queue goes to a thread of the queue's own, so its +# kernel call leaves the thread that prepares the frames, 0 or 1. Default: on (Linux), off (Windows). 0 leaves the +# driver's own choice. Unused when MESA_VK_ENABLE_SUBMIT_THREAD is set. +# submit_thread = 1 # Occlusion query results are read back from the video card in batches, 0 or 1. Default: on. 0 reads each query with a # call of its own, as before, which on AMD's Windows driver held every frame's "GPU done" back by about 4 ms. # oq_batch = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index e389119a..a6524eb5 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -75,7 +75,7 @@ static const char *const known[] = { "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WATER_PROJECTIVE", "WAIT_WATCH_S", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", - "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", + "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "SUBMIT_THREAD", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_FINI_IDLE", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VMA_BLOCK_MB", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "STAGING_FAIL_AT", "VRAM_BALLAST_MB", @@ -126,7 +126,7 @@ static const struct { const char *key, *section; } sections[] = { {"FREE_CORES", "performance"}, {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, - {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, + {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, {"SUBMIT_THREAD", "performance"}, {"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, {"MEM_NEED_MB", "diagnostics"}, {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"ASTC_BC1", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, {"POOL_CREATE_MS", "performance"}, {"POOL_FINI_IDLE", "performance"}, {"FREEZE_BUDGET_US", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, @@ -259,6 +259,11 @@ static const CfgMeta metas[] = { {"RECORDER_CORE", "0", "auto|0|1", RESTART}, {"RECORDER_PRIO", "1", "0|1", RESTART}, {"REPLAY_TIME", "0", "0|1", RESTART}, {"LOCK_TIME", "0", "0|1", RESTART}, {"POOL_CREATE_BUDGET", "16", "1..4096", RESTART}, {"POOL_CREATE_MS", "6", "0..100", RESTART}, {"POOL_FINI_IDLE", "0", "0|1", RESTART}, {"FREEZE_BUDGET_US", "1000", "0..100000", RESTART}, {"POOL_LOAD_BACKLOG", "512", "0..65536", RESTART}, {"POOL_LOAD_MS", "50", "1..1000", RESTART}, // held submissions handed to the queue together (vk_sync.c, submit batching) {"SUBMIT_BATCH", "4", "1..16", RESTART}, +#ifdef _WIN32 + {"SUBMIT_THREAD", "0", "0|1", RESTART}, // vk_device.c mesa_submit_thread: Mesa's submit thread (RADV) +#else + {"SUBMIT_THREAD", "1", "0|1", RESTART}, // vk_device.c mesa_submit_thread: Mesa's submit thread (RADV) +#endif {"CACHE_DIE", "1", "0|1|ab", RESTART}, {"ONE_PER_CORE", "0", "0|1|ab", RESTART}, // perf.c cpu_placement // shaders and pipelines {"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "any", "strict|any|fullscreen", RESTART}, diff --git a/runtime/vk_device.c b/runtime/vk_device.c index b790a10c..9effdb76 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -123,6 +123,30 @@ static void nvidia_cache_beside_exe(void) { } #endif +// Mesa's Vulkan drivers on Linux (RADV, ANV, NVK) keep timeline semaphores as kernel sync objects, which cannot wait +// for a value not submitted yet. Their queues therefore submit on the calling thread and use a thread of the queue's +// own only for a submission that waits on such a value. Every other vkQueueSubmit2, the command stream's checks and +// the kernel's ioctl, runs on the replaying thread, once for every part the replay hands to the queue early (vk_exec.c +// maybe_split). MESA_VK_ENABLE_SUBMIT_THREAD=1 sends every submission to the queue's thread: in Kokiri Forest on an +// RX 6800 XT (Mesa 26.2) the replay took 13.9 ms a frame with it and 15.0 ms without. NX2NAVI_SUBMIT_THREAD (default +// 1 on Linux, 0 on Windows) sets it unless the environment already sets it; 0 leaves the driver's own choice. Drivers +// that do not read the variable ignore it. vk_driver_cache_env calls this before any thread can load the driver. +static const char *submit_thread_why; // create_device logs it on a Mesa driver + +static void mesa_submit_thread(void) { + char cur[16]; + if (plat_getenv("MESA_VK_ENABLE_SUBMIT_THREAD", cur, sizeof cur)) { + submit_thread_why = "MESA_VK_ENABLE_SUBMIT_THREAD is set in the environment"; + return; + } + if (!cfg_bool("NX2NAVI_SUBMIT_THREAD")) { + submit_thread_why = "NX2NAVI_SUBMIT_THREAD=0: the driver's own choice"; + return; + } + plat_setenv("MESA_VK_ENABLE_SUBMIT_THREAD", "1"); + submit_thread_why = "every one on the queue's own thread (NX2NAVI_SUBMIT_THREAD=0 leaves the driver's own choice)"; +} + // A driver may read these variables when it loads, and a thread can load the Vulkan driver before create_instance: // widescreen.c's wide_init asks for the monitor's size (fullscreen with a monitor chosen and the picture at the // window's size), and SDL's load of the Vulkan library there lists the instance extensions, which loads the drivers, @@ -134,6 +158,7 @@ void vk_driver_cache_env(void) { if (done) return; done = 1; mesa_cache_beside_exe(); + mesa_submit_thread(); #ifdef __linux__ nvidia_cache_beside_exe(); #endif @@ -507,6 +532,7 @@ static int create_device(void) { VkResult r = vk_dlssg_create_device(g_vk.instance, g_vk.phys, &di, &g_vk.device); free(c); if (r != VK_SUCCESS) { rt_log("vk: vkCreateDevice failed (%d)", (int)r); return 0; } + if (submit_thread_why && vk_caps.mesa) rt_log("vk: queue submissions: %s", submit_thread_why); vk_dlssg_device_ready(g_vk.phys, g_vk.device); if (vk_dlssg_dpa()) { // Streamline's, so its plugins hook every entry point the replay fetches vk_dpa = vk_dlssg_dpa();