diff --git a/nx2navi.ini.example b/nx2navi.ini.example index fe04570f..725cde21 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -89,6 +89,10 @@ # recorder_core = 0 # Submissions handed to the GPU queue together in one call, 1 to 16 (1 = each one at once). # submit_batch = 4 +# Mesa's Vulkan drivers (RADV, ANV, NVK): every submission to the GPU queue goes to a thread of the queue's own, so its +# kernel call leaves the thread that prepares the frames, 0 or 1. Default: on (Linux), off (Windows). 0 leaves the +# driver's own choice. Unused when MESA_VK_ENABLE_SUBMIT_THREAD is set. +# submit_thread = 1 # Occlusion query results are read back from the video card in batches, 0 or 1. Default: on. 0 reads each query with a # call of its own, as before, which on AMD's Windows driver held every frame's "GPU done" back by about 4 ms. # oq_batch = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index e389119a..a6524eb5 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -75,7 +75,7 @@ static const char *const known[] = { "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WATER_PROJECTIVE", "WAIT_WATCH_S", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", - "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", + "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "SUBMIT_THREAD", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_FINI_IDLE", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VMA_BLOCK_MB", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "STAGING_FAIL_AT", "VRAM_BALLAST_MB", @@ -126,7 +126,7 @@ static const struct { const char *key, *section; } sections[] = { {"FREE_CORES", "performance"}, {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, - {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, + {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, {"SUBMIT_THREAD", "performance"}, {"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, {"MEM_NEED_MB", "diagnostics"}, {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"ASTC_BC1", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, {"POOL_CREATE_MS", "performance"}, {"POOL_FINI_IDLE", "performance"}, {"FREEZE_BUDGET_US", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, @@ -259,6 +259,11 @@ static const CfgMeta metas[] = { {"RECORDER_CORE", "0", "auto|0|1", RESTART}, {"RECORDER_PRIO", "1", "0|1", RESTART}, {"REPLAY_TIME", "0", "0|1", RESTART}, {"LOCK_TIME", "0", "0|1", RESTART}, {"POOL_CREATE_BUDGET", "16", "1..4096", RESTART}, {"POOL_CREATE_MS", "6", "0..100", RESTART}, {"POOL_FINI_IDLE", "0", "0|1", RESTART}, {"FREEZE_BUDGET_US", "1000", "0..100000", RESTART}, {"POOL_LOAD_BACKLOG", "512", "0..65536", RESTART}, {"POOL_LOAD_MS", "50", "1..1000", RESTART}, // held submissions handed to the queue together (vk_sync.c, submit batching) {"SUBMIT_BATCH", "4", "1..16", RESTART}, +#ifdef _WIN32 + {"SUBMIT_THREAD", "0", "0|1", RESTART}, // vk_device.c mesa_submit_thread: Mesa's submit thread (RADV) +#else + {"SUBMIT_THREAD", "1", "0|1", RESTART}, // vk_device.c mesa_submit_thread: Mesa's submit thread (RADV) +#endif {"CACHE_DIE", "1", "0|1|ab", RESTART}, {"ONE_PER_CORE", "0", "0|1|ab", RESTART}, // perf.c cpu_placement // shaders and pipelines {"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "any", "strict|any|fullscreen", RESTART}, diff --git a/runtime/vk_device.c b/runtime/vk_device.c index b790a10c..9effdb76 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -123,6 +123,30 @@ static void nvidia_cache_beside_exe(void) { } #endif +// Mesa's Vulkan drivers on Linux (RADV, ANV, NVK) keep timeline semaphores as kernel sync objects, which cannot wait +// for a value not submitted yet. Their queues therefore submit on the calling thread and use a thread of the queue's +// own only for a submission that waits on such a value. Every other vkQueueSubmit2, the command stream's checks and +// the kernel's ioctl, runs on the replaying thread, once for every part the replay hands to the queue early (vk_exec.c +// maybe_split). MESA_VK_ENABLE_SUBMIT_THREAD=1 sends every submission to the queue's thread: in Kokiri Forest on an +// RX 6800 XT (Mesa 26.2) the replay took 13.9 ms a frame with it and 15.0 ms without. NX2NAVI_SUBMIT_THREAD (default +// 1 on Linux, 0 on Windows) sets it unless the environment already sets it; 0 leaves the driver's own choice. Drivers +// that do not read the variable ignore it. vk_driver_cache_env calls this before any thread can load the driver. +static const char *submit_thread_why; // create_device logs it on a Mesa driver + +static void mesa_submit_thread(void) { + char cur[16]; + if (plat_getenv("MESA_VK_ENABLE_SUBMIT_THREAD", cur, sizeof cur)) { + submit_thread_why = "MESA_VK_ENABLE_SUBMIT_THREAD is set in the environment"; + return; + } + if (!cfg_bool("NX2NAVI_SUBMIT_THREAD")) { + submit_thread_why = "NX2NAVI_SUBMIT_THREAD=0: the driver's own choice"; + return; + } + plat_setenv("MESA_VK_ENABLE_SUBMIT_THREAD", "1"); + submit_thread_why = "every one on the queue's own thread (NX2NAVI_SUBMIT_THREAD=0 leaves the driver's own choice)"; +} + // A driver may read these variables when it loads, and a thread can load the Vulkan driver before create_instance: // widescreen.c's wide_init asks for the monitor's size (fullscreen with a monitor chosen and the picture at the // window's size), and SDL's load of the Vulkan library there lists the instance extensions, which loads the drivers, @@ -134,6 +158,7 @@ void vk_driver_cache_env(void) { if (done) return; done = 1; mesa_cache_beside_exe(); + mesa_submit_thread(); #ifdef __linux__ nvidia_cache_beside_exe(); #endif @@ -507,6 +532,7 @@ static int create_device(void) { VkResult r = vk_dlssg_create_device(g_vk.instance, g_vk.phys, &di, &g_vk.device); free(c); if (r != VK_SUCCESS) { rt_log("vk: vkCreateDevice failed (%d)", (int)r); return 0; } + if (submit_thread_why && vk_caps.mesa) rt_log("vk: queue submissions: %s", submit_thread_why); vk_dlssg_device_ready(g_vk.phys, g_vk.device); if (vk_dlssg_dpa()) { // Streamline's, so its plugins hook every entry point the replay fetches vk_dpa = vk_dlssg_dpa();