From 5fee8d7ab262c0e0d0c39c6f60a73b1a0a90718a Mon Sep 17 00:00:00 2001 From: Vasilis Manetas Date: Thu, 12 Nov 2026 16:26:26 +0100 Subject: [PATCH 1/2] Hand every queue submission to the driver's own submit thread on Linux, so RADV's vkQueueSubmit2 leaves the replaying thread. Mesa's Vulkan drivers keep timeline semaphores as kernel sync objects, and their queues submit on the calling thread unless a submission waits on a value not submitted yet. The replay hands a part of the frame to the queue each time it splits one off (vk_exec.c maybe_split), so every such vkQueueSubmit2, with the command stream's checks and the kernel's ioctl, ran inside the replay, the thread that limits the frame rate on an RX 6800 XT under RADV. MESA_VK_ENABLE_SUBMIT_THREAD=1 sends every submission to a thread of the queue's own. vk_device.c mesa_submit_thread sets it before any thread can load the driver, unless the environment sets it already. submit_thread in nx2navi.ini (NX2NAVI_SUBMIT_THREAD) is 1 by default on Linux and 0 on Windows; 0 leaves the driver's own choice. On a Mesa driver the log says which one applies. Kokiri Forest, 0.9.1, RX 6800 XT, Mesa 26.2.6, 3 pairs of 45 s runs of one build with the setting on and off: the replay took 13.9 ms a frame against 15.0, lower in every pair; frames over 25 ms fell from 910 to 406; the frame rate went from 40.0 to 40.8 fps, within the runs' noise. --- nx2navi.ini.example | 4 ++++ runtime/cfg.c | 9 +++++++-- runtime/vk_device.c | 26 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index fe04570f..725cde21 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -89,6 +89,10 @@ # recorder_core = 0 # Submissions handed to the GPU queue together in one call, 1 to 16 (1 = each one at once). # submit_batch = 4 +# Mesa's Vulkan drivers (RADV, ANV, NVK): every submission to the GPU queue goes to a thread of the queue's own, so its +# kernel call leaves the thread that prepares the frames, 0 or 1. Default: on (Linux), off (Windows). 0 leaves the +# driver's own choice. Unused when MESA_VK_ENABLE_SUBMIT_THREAD is set. +# submit_thread = 1 # Occlusion query results are read back from the video card in batches, 0 or 1. Default: on. 0 reads each query with a # call of its own, as before, which on AMD's Windows driver held every frame's "GPU done" back by about 4 ms. # oq_batch = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index 61ec856d..e872afb5 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -75,7 +75,7 @@ static const char *const known[] = { "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WATER_PROJECTIVE", "WAIT_WATCH_S", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", - "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", + "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "SUBMIT_THREAD", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_FINI_IDLE", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VMA_BLOCK_MB", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "STAGING_FAIL_AT", "VRAM_BALLAST_MB", @@ -126,7 +126,7 @@ static const struct { const char *key, *section; } sections[] = { {"FREE_CORES", "performance"}, {"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"}, {"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"}, - {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, + {"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"}, {"SUBMIT_THREAD", "performance"}, {"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, {"MEM_NEED_MB", "diagnostics"}, {"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"ASTC_BC1", "performance"}, {"POOL_CREATE_BUDGET", "performance"}, {"POOL_CREATE_MS", "performance"}, {"POOL_FINI_IDLE", "performance"}, {"FREEZE_BUDGET_US", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, @@ -259,6 +259,11 @@ static const CfgMeta metas[] = { {"RECORDER_CORE", "0", "auto|0|1", RESTART}, {"RECORDER_PRIO", "1", "0|1", RESTART}, {"REPLAY_TIME", "0", "0|1", RESTART}, {"LOCK_TIME", "0", "0|1", RESTART}, {"POOL_CREATE_BUDGET", "16", "1..4096", RESTART}, {"POOL_CREATE_MS", "6", "0..100", RESTART}, {"POOL_FINI_IDLE", "0", "0|1", RESTART}, {"FREEZE_BUDGET_US", "1000", "0..100000", RESTART}, {"POOL_LOAD_BACKLOG", "512", "0..65536", RESTART}, {"POOL_LOAD_MS", "50", "1..1000", RESTART}, // held submissions handed to the queue together (vk_sync.c, submit batching) {"SUBMIT_BATCH", "4", "1..16", RESTART}, +#ifdef _WIN32 + {"SUBMIT_THREAD", "0", "0|1", RESTART}, // vk_device.c mesa_submit_thread: Mesa's submit thread (RADV) +#else + {"SUBMIT_THREAD", "1", "0|1", RESTART}, // vk_device.c mesa_submit_thread: Mesa's submit thread (RADV) +#endif {"CACHE_DIE", "1", "0|1|ab", RESTART}, {"ONE_PER_CORE", "0", "0|1|ab", RESTART}, // perf.c cpu_placement // shaders and pipelines {"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "any", "strict|any|fullscreen", RESTART}, diff --git a/runtime/vk_device.c b/runtime/vk_device.c index b790a10c..9effdb76 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -123,6 +123,30 @@ static void nvidia_cache_beside_exe(void) { } #endif +// Mesa's Vulkan drivers on Linux (RADV, ANV, NVK) keep timeline semaphores as kernel sync objects, which cannot wait +// for a value not submitted yet. Their queues therefore submit on the calling thread and use a thread of the queue's +// own only for a submission that waits on such a value. Every other vkQueueSubmit2, the command stream's checks and +// the kernel's ioctl, runs on the replaying thread, once for every part the replay hands to the queue early (vk_exec.c +// maybe_split). MESA_VK_ENABLE_SUBMIT_THREAD=1 sends every submission to the queue's thread: in Kokiri Forest on an +// RX 6800 XT (Mesa 26.2) the replay took 13.9 ms a frame with it and 15.0 ms without. NX2NAVI_SUBMIT_THREAD (default +// 1 on Linux, 0 on Windows) sets it unless the environment already sets it; 0 leaves the driver's own choice. Drivers +// that do not read the variable ignore it. vk_driver_cache_env calls this before any thread can load the driver. +static const char *submit_thread_why; // create_device logs it on a Mesa driver + +static void mesa_submit_thread(void) { + char cur[16]; + if (plat_getenv("MESA_VK_ENABLE_SUBMIT_THREAD", cur, sizeof cur)) { + submit_thread_why = "MESA_VK_ENABLE_SUBMIT_THREAD is set in the environment"; + return; + } + if (!cfg_bool("NX2NAVI_SUBMIT_THREAD")) { + submit_thread_why = "NX2NAVI_SUBMIT_THREAD=0: the driver's own choice"; + return; + } + plat_setenv("MESA_VK_ENABLE_SUBMIT_THREAD", "1"); + submit_thread_why = "every one on the queue's own thread (NX2NAVI_SUBMIT_THREAD=0 leaves the driver's own choice)"; +} + // A driver may read these variables when it loads, and a thread can load the Vulkan driver before create_instance: // widescreen.c's wide_init asks for the monitor's size (fullscreen with a monitor chosen and the picture at the // window's size), and SDL's load of the Vulkan library there lists the instance extensions, which loads the drivers, @@ -134,6 +158,7 @@ void vk_driver_cache_env(void) { if (done) return; done = 1; mesa_cache_beside_exe(); + mesa_submit_thread(); #ifdef __linux__ nvidia_cache_beside_exe(); #endif @@ -507,6 +532,7 @@ static int create_device(void) { VkResult r = vk_dlssg_create_device(g_vk.instance, g_vk.phys, &di, &g_vk.device); free(c); if (r != VK_SUCCESS) { rt_log("vk: vkCreateDevice failed (%d)", (int)r); return 0; } + if (submit_thread_why && vk_caps.mesa) rt_log("vk: queue submissions: %s", submit_thread_why); vk_dlssg_device_ready(g_vk.phys, g_vk.device); if (vk_dlssg_dpa()) { // Streamline's, so its plugins hook every entry point the replay fetches vk_dpa = vk_dlssg_dpa(); From f0d0fae13b40f9bd7a59592f370e957b8c06e453 Mon Sep 17 00:00:00 2001 From: FIocker <4461558+FIocker@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:17:31 +0100 Subject: [PATCH 2/2] Tell FSR 3.1 frame generation of camera cuts, so a generated frame is never blended from two unrelated pictures. vk_dlaa_fg_inputs hands out DLSS's new history with the frame's depth and motion, and FSR 4's and NVIDIA's frame generation pass it on, but vk_fg_frame dropped it: FSR 3.1 takes a cut only in the generation callback, which its swapchain calls later on its own thread. vk_fg_frame now notes each frame id's cut in a small ring (runtime/vk_fg_reset.h), and generate sets the callback's reset from it. A frame after one that was not prepared (the menu open, no inputs) counts as a cut too, and so does an id the ring no longer holds. Co-Authored-By: Claude Opus 5.5 --- docs/building.md | 5 +++++ runtime/vk_fg.cpp | 16 ++++++++++++-- runtime/vk_fg_reset.h | 32 ++++++++++++++++++++++++++++ tests/fg_reset_check.c | 46 +++++++++++++++++++++++++++++++++++++++++ tests/fg_reset_check.py | 40 +++++++++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 runtime/vk_fg_reset.h create mode 100644 tests/fg_reset_check.c create mode 100644 tests/fg_reset_check.py diff --git a/docs/building.md b/docs/building.md index 1c243528..c6c0e9d8 100644 --- a/docs/building.md +++ b/docs/building.md @@ -473,6 +473,11 @@ builds the header alone and replays an area change after a loading screen from a half-resolution passes are per-frame again at once, a lookup table, a volume's slices and an icon capture still wait, and an image taken over by another texture starts without the history of the one before. No GPU, no game files. +A change to which frames FSR 3.1 frame generation is told are camera cuts (`runtime/vk_fg_reset.h`, `runtime/vk_fg.cpp` +`generate` and `vk_fg_frame`) passes `python tests\fg_reset_check.py`, which builds the header alone and checks that each +frame id gets its own frame's cut, that an id never noted or no longer held counts as a cut, and that a slot rewritten +on another thread never answers an old id with a newer frame's value. No GPU, no game files. + The lifter (`recomp/`) and the CPU headers (`runtime/cpu.h`, `runtime/simd.h`) are checked against Unicorn. Every lifter change passes these, all run with `.venv`: diff --git a/runtime/vk_fg.cpp b/runtime/vk_fg.cpp index 3ed69bef..25f03cc8 100644 --- a/runtime/vk_fg.cpp +++ b/runtime/vk_fg.cpp @@ -28,6 +28,7 @@ extern "C" { #include "vk_dlssg.h" #include "vk_ffx.h" #include "vk_fg.h" +#include "vk_fg_reset.h" #include "ui/menu.h" } @@ -45,6 +46,9 @@ struct { int failed = 0; uint64_t generated = 0, paused = 0, no_inputs = 0, hudless = 0, no_hudless = 0; int on = 0; // the last configured frame had generation on (vk_fg_generating) + FgrRing cuts; // each frame id's camera cut, for the generation callback (vk_fg_reset.h) + uint64_t cut_frames = 0; // frames handed to the callback as cuts + int prepared = 0; // the last frame was prepared (generation on with its inputs) } F; VkResult submit_game(uint32_t n, const VkSubmitInfo *si, VkFence fence) { @@ -56,6 +60,8 @@ VkResult submit_game(uint32_t n, const VkSubmitInfo *si, VkFence fence) { ffxReturnCode_t generate(ffxDispatchDescFrameGeneration *d, void *user) { F.generated++; + // the swapchain's own reset (a frame id that did not follow the last) stays; a cut vk_fg_frame noted is added + if (fgr_cut(&F.cuts, d->frameID)) d->reset = true; return vk_ffx_dispatch((ffxContext *)user, &d->header); } @@ -267,6 +273,11 @@ extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, else if (!hudless) F.no_hudless++; else if (!on) F.paused++; F.frame++; + // A cut: DLSS started a new history (vk_dlaa_fg_inputs), or the frame before was not prepared (generation off for + // the menu, no inputs), so the frame between the two would be made from a stale depth and motion + fgr_note(&F.cuts, F.frame, reset || !F.prepared); + if ((reset || !F.prepared) && on) F.cut_frames++; + F.prepared = on; ffxConfigureDescFrameGeneration cfg = {}; cfg.header.type = FFX_API_CONFIGURE_DESC_TYPE_FRAMEGENERATION; cfg.swapChain = (void *)sc; @@ -312,9 +323,9 @@ extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, if (r != FFX_API_RETURN_OK) return fail("frame generation preparation", r); if (F.frame == 2 || F.frame % 3600 == 0) rt_log("fg: %llu frames presented, %llu generated (%llu with the HUD kept out); paused %llu (menu), %llu without depth " - "and motion, %llu without the HUD-less picture", (unsigned long long)F.frame, (unsigned long long)F.generated, + "and motion, %llu without the HUD-less picture; %llu after a camera cut", (unsigned long long)F.frame, (unsigned long long)F.generated, (unsigned long long)F.hudless, (unsigned long long)F.paused, (unsigned long long)F.no_inputs, - (unsigned long long)F.no_hudless); + (unsigned long long)F.no_hudless, (unsigned long long)F.cut_frames); } extern "C" void vk_fg_release(void) { @@ -322,4 +333,5 @@ extern "C" void vk_fg_release(void) { if (F.fg) vk_ffx_destroy(&F.fg), F.fg = nullptr; if (F.sc) vk_ffx_destroy(&F.sc), F.sc = nullptr; __atomic_store_n(&F.on, 0, __ATOMIC_RELAXED); + F.prepared = 0; } diff --git a/runtime/vk_fg_reset.h b/runtime/vk_fg_reset.h new file mode 100644 index 00000000..280a82cc --- /dev/null +++ b/runtime/vk_fg_reset.h @@ -0,0 +1,32 @@ +// Camera cuts for FSR 3.1 frame generation (vk_fg.cpp). DLSS starts a new history on a cut, another scene size or +// another target, and vk_dlaa.c vk_dlaa_fg_inputs hands that out with the frame's depth and motion vectors. FSR 3.1 +// takes it only in the generation callback (ffxDispatchDescFrameGeneration.reset), which its swapchain calls later, +// on its own present thread, for the frame id the configuration gave. So vk_fg_frame notes each frame's cut here +// under its id, and the callback looks it up. Without it every generated frame of a cut was blended from two unrelated +// pictures: an area load or a cutscene's cut flashed a frame of both. +// +// A ring of FGR_SLOTS frames; each slot holds the frame id and its cut as one atomic word, so a slot being rewritten +// never pairs an old id with a new frame's answer. An id the ring no longer holds (the callback fell that far behind, +// or the slot was never written) counts as a cut: a lost history costs one plain frame, a wrong one blends two. +// Own header so tests/fg_reset_check.c checks it without a GPU. +#pragma once +#include + +#define FGR_SLOTS 32 + +typedef struct { + uint64_t slot[FGR_SLOTS]; // (frame id << 1) | cut, written by vk_fg_frame; 0 never matches (ids start at 1) +} FgrRing; + +// Frame ids above this do not fit beside the cut bit; they always count as a cut. +#define FGR_MAX_ID (UINT64_MAX >> 1) + +static inline void fgr_note(FgrRing *r, uint64_t id, int cut) { + if (id && id <= FGR_MAX_ID) __atomic_store_n(&r->slot[id % FGR_SLOTS], id << 1 | (cut != 0), __ATOMIC_RELEASE); +} + +static inline int fgr_cut(const FgrRing *r, uint64_t id) { + if (!id || id > FGR_MAX_ID) return 1; + uint64_t v = __atomic_load_n(&r->slot[id % FGR_SLOTS], __ATOMIC_ACQUIRE); + return v >> 1 != id || (v & 1); +} diff --git a/tests/fg_reset_check.c b/tests/fg_reset_check.c new file mode 100644 index 00000000..c4aaa7e2 --- /dev/null +++ b/tests/fg_reset_check.c @@ -0,0 +1,46 @@ +// runtime/vk_fg_reset.h: the camera cut FSR 3.1 frame generation's callback is told of for a frame id. Built and run +// by tests/fg_reset_check.py; prints each failed check and exits 1 on any. +#include +#include +#include + +#include "vk_fg_reset.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +static FgrRing ring; +static volatile int stop; + +// Rewrites one slot over and over: frame 3, a cut, and frame 3 + FGR_SLOTS, none. +static void *writer(void *arg) { + (void)arg; + while (!stop) fgr_note(&ring, 3, 1), fgr_note(&ring, 3 + FGR_SLOTS, 0); + return NULL; +} + +int main(void) { + // nothing noted: every id is a cut, id 0 and ids past the range too + CHECK(fgr_cut(&ring, 0) && fgr_cut(&ring, 1) && fgr_cut(&ring, 77), "an id never noted is a cut"); + CHECK(fgr_cut(&ring, FGR_MAX_ID + 1) && fgr_cut(&ring, UINT64_MAX), "an id past the range is a cut"); + // a run of frames, a cut at 40: the callback for each id gets that frame's answer + for (uint64_t f = 1; f <= 60; f++) fgr_note(&ring, f, f == 40); + int wrong = 0; + for (uint64_t f = 60 - FGR_SLOTS + 1; f <= 60; f++) wrong += fgr_cut(&ring, f) != (f == 40); + CHECK(!wrong, "%d of the last %d frames answered wrong", wrong, FGR_SLOTS); + // the callback fell behind by more than the ring: its frame's slot holds a later one, so it is a cut + CHECK(fgr_cut(&ring, 60 - FGR_SLOTS), "a frame the ring no longer holds is a cut"); + CHECK(fgr_cut(&ring, 61), "a frame not noted yet is a cut"); + // the id and its cut are one word: a slot rewritten on another thread never answers frame 3 with the next one's + // "no cut" + memset(&ring, 0, sizeof ring); + pthread_t t; + pthread_create(&t, NULL, writer, NULL); + int paired = 0; + for (int i = 0; i < 2000000; i++) paired += !fgr_cut(&ring, 3); + stop = 1; + pthread_join(t, NULL); + CHECK(!paired, "frame 3 was answered with the other frame's value %d times", paired); + printf(fails ? "fg_reset_check: %d failed\n" : "fg_reset_check: ok\n", fails); + return fails != 0; +} diff --git a/tests/fg_reset_check.py b/tests/fg_reset_check.py new file mode 100644 index 00000000..1f2e960c --- /dev/null +++ b/tests/fg_reset_check.py @@ -0,0 +1,40 @@ +"""Build and run tests/fg_reset_check.c: runtime/vk_fg_reset.h, which frames FSR 3.1 frame generation's callback is +told are camera cuts. + +No GPU, no game files, any Python. + +Usage: fg_reset_check.py +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') + + +def compiler(): + """$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn).""" + if os.environ.get('CC'): + return os.environ['CC'] + if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists(): + return str(MSYS_BIN / 'clang.exe') + return shutil.which('clang') or 'clang' + + +def main(): + work = ROOT / 'build' / 'test' / 'fg_reset' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('fg_reset_check.exe' if os.name == 'nt' else 'fg_reset_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-Wall', '-Werror', '-pthread', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'fg_reset_check.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main())