diff --git a/runtime/frame_rate.c b/runtime/frame_rate.c index 962e8bcc..6bfcffd0 100644 --- a/runtime/frame_rate.c +++ b/runtime/frame_rate.c @@ -55,6 +55,7 @@ #include "platform/platform.h" #include "vk_dlssg.h" #include "vk_fg.h" +#include "vk_lowlat.h" #define STEP_SAMPLES 5 #define STEP_MAX_S (1.0 / 15.0) // longest step; a longer frame slows the game down @@ -233,7 +234,9 @@ double frame_step_game_hz(void) { } // RivaTuner Statistics Server (rtss_limit.c) gets the same limit; while it limits this process the pacer holds no -// frames, so the two never stack. NX2NAVI_RTSS=0 and frame generation (RTSS would count the generated frames) keep it out. +// frames, so the two never stack. NX2NAVI_RTSS=0 and frame generation (RTSS would count the generated frames) keep it +// out, and so does a driver's low latency while it is in effect (vk_lowlat.c): the driver holds the limit where the +// frame starts, and RTSS would hold the finished picture, the wait that mode is there to remove. static double paced_hz; // the game frames a second the newest frame_step_cap_ns paces at (ocarina_step); 0: none static int movie_paced; // the movie rate the newest frame_step_cap_ns paces at (ocarina_step, frame_step_movie_hz); 0: none int frame_step_movie_hz(void) { return movie_paced; } @@ -244,7 +247,7 @@ uint64_t frame_step_cap_ns(void) { const char *e = cfg_get("NX2NAVI_RTSS"); rtss_allowed = !(e && !strcmp(e, "0")); } - int rtss = rtss_allowed && !vk_fg_wanted() && !vk_dlssg_active(); + int rtss = rtss_allowed && !vk_fg_wanted() && !vk_dlssg_active() && !vk_lowlat_on(); int gen = generated_factor(); int movie_hz = movie_cap_hz(); static int movie_clock = -1; // NX2NAVI_MOVIE_CLOCK=0: the steady step and the usual pacing through a movie too diff --git a/runtime/gfx.h b/runtime/gfx.h index 658163a5..7a630acd 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -172,6 +172,14 @@ typedef struct { // and the GPU's): the back end may hold it longer (vk_dlssg.c vk_dlssg_frame_start, Reflex's sleep for DLSS Frame // Generation). NULL: nothing to do. void (*frame_start)(void); + // The game is about to read its pad (ov_input.c npad_states), several times a frame: where the pad read is the + // first thing a frame does, the back end holds the frame's start here, before the input is sampled (vk_lowlat.c + // vk_lowlat_frame_start). NULL: nothing to do. + void (*input_read)(void); + // nvnQueuePresentTexture, with the pacer's period in ns (0: it holds nothing): 1 when the back end holds the next + // frame's start to that period itself (vk_lowlat.c vk_lowlat_pace, a driver's low-latency sleep), so the pacer + // waits for neither the present's slot nor the window image. NULL: never. + int (*frame_paced)(uint64_t period_ns); void (*finish)(void); void (*read_texels)(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img); void (*write_texels)(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row, uint64_t img); diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index a1f46b98..5040d168 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -32,6 +32,7 @@ #include "vk_fg.h" #include "vk_dlaa.h" #include "vk_dlssg.h" +#include "vk_lowlat.h" #define FRAMES 2 #define MAX_IMAGES 16 @@ -189,6 +190,7 @@ static VkDeviceSize g_cap_size; static int g_cap_w, g_cap_h, g_cap_valid; static int g_flip_on; // vk_present_blit submitted the blit of swapchain image g_flip_idx; vk_present_flip presents it static uint32_t g_flip_idx; +static uint64_t g_flip_frame; // ... of this game frame, when a driver's low latency follows it (vk_lowlat.c); else 0 // ---- NX2NAVI_FLIGHT thumbnails (flight.c; g_pl): each present also blits the swapchain image down into slot frame % g_th_n // of a host-visible ring. A slot's tag names the frame it holds complete; recording a new frame clears the tag first, @@ -285,6 +287,7 @@ unsigned gfx_window_focus_messages(void) { return (unsigned)g_focus_msgs; } // Present path (g_pl held): a settings change (cfg_version) reaches the window thread, which applies a changed size or // fullscreen; a changed NX2NAVI_VK_PRESENT or NX2NAVI_HDR rebuilds the swapchain at this present. static int g_sc_hdr_wanted; // vk_hdr_wanted when the swapchain was made (create_swapchain) +static int g_sc_ll_wanted; // vk_lowlat_swapchain_wanted then static void window_settings_live(void) { static unsigned seen; static int started; @@ -302,6 +305,8 @@ static void window_settings_live(void) { // HDR switched in the menu: a swapchain of the other format. Compared with what the swapchain was made for, not // with what it got: on a display without HDR10 the setting stays on and the swapchain SDR, and that is settled. if (started && vk_hdr_wanted() != g_sc_hdr_wanted) plat_xchg32(&g_dirty, 1); + // NVIDIA's low latency switched in the menu (vk_lowlat.c): its mode is asked for when the swapchain is made + if (started && vk_lowlat_swapchain_wanted() != g_sc_ll_wanted) plat_xchg32(&g_dirty, 1); ov_operation_mode_live(); audio_channels_live(); if (started) gfx_window_apply_settings(); @@ -353,10 +358,17 @@ static void flip_locked(void) { VkPresentInfoKHR pi = {.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR, .waitSemaphoreCount = 1, .pWaitSemaphores = &g_render_done[g_flip_idx], .swapchainCount = 1, .pSwapchains = &g_sc, .pImageIndices = &g_flip_idx}; + // A driver's low latency (vk_lowlat.c): its present stage or marker right before the present of a game frame it + // follows, the frame's number on the present where the plain swapchain takes one, and the end marker after it. + // Nothing for any other present (the boot screen, a frame the mode did not start, the mode off). + VkLowlatPresent ll; vk_queue_lock(); + pi.pNext = vk_lowlat_present_begin(g_flip_frame, &ll); + g_flip_frame = 0; VkResult r = vk_dlssg_active() ? vk_dlssg_present(vk_ctx()->gfx_queue, &pi) : vk_fg_active() ? vk_fg_present(&pi) : vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi); vk_queue_unlock(); + vk_lowlat_present_end(&ll); vk_device_lost_check(r, "the present"); if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1); } @@ -429,6 +441,7 @@ static int create_swapchain(void) { vk_queue_lock(); vkDeviceWaitIdle(c->device); + vk_lowlat_swapchain(VK_NULL_HANDLE, 0); // NVIDIA's low-latency calls name the swapchain: none while it is replaced VkSwapchainKHR old = g_sc; VkSwapchainCreateInfoKHR ci = { .sType = VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR, .surface = g_surface, .minImageCount = count, @@ -465,8 +478,22 @@ static int create_swapchain(void) { return 0; } } + // NVIDIA's low latency (vk_lowlat.c): the plain swapchain is made with the latency mode while the setting is on, + // the only structure in its chain. A frame generation's swapchain above is not: the calls cannot name it. + const VkSwapchainLatencyCreateInfoNV lat = {.sType = VK_STRUCTURE_TYPE_SWAPCHAIN_LATENCY_CREATE_INFO_NV, + .latencyModeEnable = VK_TRUE}; + int ll_sc = 0; + g_sc_ll_wanted = vk_lowlat_swapchain_wanted(); if (r != VK_SUCCESS && vk_dlssg_active()) r = vk_dlssg_create_swapchain(c->device, &ci, &g_sc); // Streamline's hook - else if (r != VK_SUCCESS) r = vkCreateSwapchainKHR(c->device, &ci, NULL, &g_sc); + else if (r != VK_SUCCESS) { + if (g_sc_ll_wanted) ci.pNext = &lat, ll_sc = 1; + r = vkCreateSwapchainKHR(c->device, &ci, NULL, &g_sc); + } + { // three creates in a row that fail with the latency mode asked for: the next is made without it + static int ll_failed; + ll_failed = r != VK_SUCCESS && ll_sc ? ll_failed + 1 : 0; + if (ll_failed >= 3) vk_lowlat_swapchain_refused((int)r); + } if (r != VK_SUCCESS) { static volatile int32_t n_failed; int32_t nf_log = plat_inc32(&n_failed); @@ -492,6 +519,7 @@ static int create_swapchain(void) { vk_hdr_set_active(hdr); if (hdr) vk_hdr_metadata(g_sc, 1); g_sc_ext = ext; + vk_lowlat_swapchain(ll_sc ? g_sc : VK_NULL_HANDLE, 1); // the sleep mode is set on it; none: not one for that path g_sc_n = MAX_IMAGES; if (vk_fg_active()) { // AMD's swapchain: the count first (a larger capacity is not cut down to it) uint32_t n = 0; @@ -1437,6 +1465,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w uint32_t idx = 0; // NGX frame generation: the frame is composed into ng.bb and the presenter shows it (ng_show); no acquire here const int ngp = ng_ready(); + vk_lowlat_fg_presents(ngp); // the presenter presents several times a frame: a driver's low latency steps aside VkResult r = ngp ? VK_SUCCESS : vk_dlssg_active() ? vk_dlssg_acquire(c->device, g_sc, 1000000000ull, g_acquire[slot], &idx) : vk_fg_active() ? vk_fg_acquire(g_sc, 1000000000ull, g_acquire[slot], &idx) @@ -1624,6 +1653,11 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .waitSemaphoreInfoCount = 1, .pWaitSemaphoreInfos = &wait, .commandBufferInfoCount = 1, .pCommandBufferInfos = &cbi, .signalSemaphoreInfoCount = 1, .pSignalSemaphoreInfos = &sig}; + // A driver's low latency (vk_lowlat.c): the game frame this blit shows, and NVIDIA's tag with its number + VkLatencySubmissionPresentIdNV ll_tag; + int ll_tagged = 0; + const uint64_t ll_frame = src ? vk_lowlat_blit_frame(&ll_tag, &ll_tagged) : 0; + if (ll_tagged) si.pNext = &ll_tag; const uint64_t ng_n = ng.n + 1; // through the presenter: this frame's number on its timelines if (via_ng) { // No acquire to wait for. The frame writes one of the presenter's two image sets: it waits on the GPU until @@ -1647,6 +1681,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w } else if (r == VK_SUCCESS) { g_flip_on = 1; g_flip_idx = idx; + g_flip_frame = ll_frame; if (!defer) flip_locked(); } else { vk_device_lost_check(r, "the present's submit"); @@ -1712,6 +1747,7 @@ void gfx_window_release_vk(void) { vk_queue_lock(), vkDeviceWaitIdle(c->device), vk_queue_unlock(); vk_queue_unlock(); g_flip_on = 0; // its blit is done (device idle); the swapchain goes away unpresented + vk_lowlat_swapchain(VK_NULL_HANDLE, 0); ng_release(); // NGX frame generation's presenter shows nothing more; its images go before the device menu_vk_release(); vk_fsr_release(); diff --git a/runtime/nvn_api_obj.c b/runtime/nvn_api_obj.c index 8ba65eb5..243f5a7d 100644 --- a/runtime/nvn_api_obj.c +++ b/runtime/nvn_api_obj.c @@ -893,6 +893,11 @@ NVN_API(nvnQueuePresentTexture) { } if (hitch_every && o->presents % (uint64_t)hitch_every == 0) plat_wait_ns((uint64_t)hitch_ms * 1000000); uint64_t period = present_period(o), now = kern_now_ns(); + // A driver's low-latency sleep (vk_lowlat.c) holds the next frame to the period where the frame starts, before + // its pad read, so this pacer holds nothing: the present goes out as soon as it is drawn and the image is free at + // once. A wait here as well would be a second one, after the picture is drawn. Not while a movie plays: its slots + // are kept through a late frame (pace), which only this pacer does, so the back end is told there is no period. + if (gfx_backend()->frame_paced && gfx_backend()->frame_paced(frame_step_movie_hz() > 0 ? 0 : period)) period = 0; uint64_t slot = now + pace(&o->pace, now, period); gfx_backend()->present(t, o->crop, slot); o->ticket[idx] = o->presents; diff --git a/runtime/ov_input.c b/runtime/ov_input.c index 16340ec8..19e52caf 100644 --- a/runtime/ov_input.c +++ b/runtime/ov_input.c @@ -13,6 +13,7 @@ #include "debug_tools.h" #include "cfg.h" +#include "gfx.h" #include "host.h" #include "input.h" #include "ov.h" @@ -91,6 +92,9 @@ static void npad_states(Cpu *c, uint32_t want) { NpadState s = {.sampling = ++sample}; int live = want && enabled && allowed && id == pad_id() && want == pad_style() && (styles & want); plat_mutex_unlock(&lock); + // A frame's first pad read is where a driver's low-latency sleep belongs (vk_lowlat.c vk_lowlat_frame_start): the + // wait comes before the input is sampled, so the frame is drawn from the newest input. Nothing while that is off. + if (gfx_backend()->input_read) gfx_backend()->input_read(); if (live) { s.attributes = ATTR_CONNECTED; input_poll(&s.buttons, &s.lx, &s.ly, &s.rx, &s.ry); diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 167518af..6b40b6f0 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -9,7 +9,9 @@ // // Back-edge pacing: the present thread submits the blit at once, then holds the host present (vkQueuePresentKHR) until // the frame's display time from the pacer (nvn_api_obj.c), so present-to-present intervals stay even whatever the -// replay took. The blit is already on the queue, so the wait blocks no submission. +// replay took. The blit is already on the queue, so the wait blocks no submission. While a driver's low latency holds +// the frames where they start (vk_lowlat.c), the pacer's display time is the time of the game's present, and the host +// present goes out as soon as the blit is submitted. // // Asynchronous replay (NX2NAVI_REPLAY_ASYNC, default 0; 1 turns it on): nvnQueueSubmitCommands // and nvnQueuePresentTexture only queue. Each item gets its timeline value at enqueue (vk_sync_reserve), in queue @@ -34,6 +36,7 @@ #include "vk_dlaa.h" #include "vk_dlssg.h" #include "vk_fg.h" +#include "vk_lowlat.h" #define WEAK __attribute__((weak)) @@ -86,6 +89,7 @@ static void present_main(void *arg) { plat_mutex_unlock(&pl); if (flight_on || vk_dump_white) vk_thumbs_frame(j.frame); // NX2NAVI_DUMP_WHITE reads them too vk_dlssg_present_frame(j.frame); + vk_lowlat_present_frame(j.frame); // the frame's number for the blit's submit and the present's markers vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.format, j.w, j.h, j.has_crop ? j.crop : NULL); uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too if (j.value && f != j.value) @@ -530,6 +534,7 @@ static void replay_main(void *arg) { } else { uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); vk_dlssg_submitted(frame); // the frame's submissions were replayed before its present + vk_lowlat_submitted(frame); present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value); sub_idx = 0; } @@ -663,9 +668,11 @@ static void vk_be_present(const tex_t *t, const int crop[4], uint64_t at_ns) { vk_sync_start(); gfx_tl_present(); vk_dlssg_game_present(); + vk_lowlat_game_present(); if (!gfx_replay_async()) { uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); vk_dlssg_submitted(frame); + vk_lowlat_submitted(frame); vk_exec_evt_frame(frame); perf_present(frame); // before vk_exec_frame_log, which resets the frame's exec counts the flight recorder reads uint64_t queued = kern_now_ns(); @@ -766,6 +773,7 @@ static void vk_be_program_release(const prog_t *p) { replay_drain(DR_PROG_RELEAS // Asynchronous: the handles (nvn_submit_prepare_async ran under the front end's submit lock) are copied and queued; // the value returned is the one the replay will signal. static uint64_t vk_be_submit(const uint64_t *handles, int n) { + vk_lowlat_submit_start(); // a frame's first submission: its simulation is over (nothing while low latency is off) if (!gfx_replay_async()) { vk_sub_in_frame = __atomic_load_n(&nvn_submits_in_frame, __ATOMIC_RELAXED); // the front end counts it up after the submit returns return submit_sync(handles, n); @@ -907,6 +915,15 @@ static void vk_be_cpu_synced(uint64_t fence) { vk_texture_write_back_small(fence); } +// A frame of the game starts. DLSS Frame Generation's low latency sleeps at the acquire (vk_dlssg.c); the direct path +// (vk_lowlat.c) at the acquire or the pad read, whichever the game makes first in the frame. At most one of the two +// is in effect in a run, and each returns at once while it is not. +static void vk_be_frame_start(void) { + vk_dlssg_frame_start(); + vk_lowlat_frame_start(0); +} +static void vk_be_input_read(void) { vk_lowlat_frame_start(1); } + const GfxBackend gfx_vulkan_backend = { .name = "vulkan", .pool_init = vk_be_pool_init, @@ -921,7 +938,9 @@ const GfxBackend gfx_vulkan_backend = { .exec_wait = vk_be_exec_wait, .fence_done = vk_sync_fence_done, .present = vk_be_present, - .frame_start = vk_dlssg_frame_start, + .frame_start = vk_be_frame_start, + .input_read = vk_be_input_read, + .frame_paced = vk_lowlat_pace, .finish = vk_be_finish, .read_texels = vk_be_read_texels, .write_texels = vk_be_write_texels, diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 56e80544..9cdc2868 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -18,6 +18,7 @@ #include "nvn_api.h" #include "prof.h" #include "vk.h" +#include "vk_lowlat.h" #include "platform/platform.h" #define WEAK __attribute__((weak)) @@ -96,6 +97,7 @@ static struct { VkSubmitInfo2 si[HOLD_CAP]; VkCommandBufferSubmitInfo cbi[HOLD_CAP][SUB_CBS]; VkSemaphoreSubmitInfo sig[HOLD_CAP][2]; + VkLatencySubmissionPresentIdNV lat[HOLD_CAP]; // the frame's number for the driver's low latency (vk_lowlat.c) uint64_t last; // the newest timeline value a held submission signals, 0 none uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried } H; @@ -312,6 +314,9 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) { memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount); H.si[i].pSignalSemaphoreInfos = H.sig[i]; } + // NVIDIA's low latency on (vk_lowlat.c): the submission carries the number of the frame being replayed, so the + // driver counts its GPU time to that frame. The callers' submissions have no chain of their own. + if (vk_lowlat_submit_tag(&H.lat[i])) H.si[i].pNext = &H.lat[i]; if (value) H.last = value; uint64_t now = kern_now_ns(); if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE); diff --git a/tests/frame_step_check.c b/tests/frame_step_check.c index 69b32475..f4909c44 100644 --- a/tests/frame_step_check.c +++ b/tests/frame_step_check.c @@ -37,6 +37,7 @@ int vk_fg_wanted(void) { return 0; } static int fg_mult = 1; // frames on screen per game frame int vk_dlssg_multiplier(void) { return fg_mult; } int vk_dlssg_active(void) { return fg_mult > 1; } +int vk_lowlat_on(void) { return 0; } void rtss_limit_want(int64_t millihz) { (void)millihz; } int rtss_limit_active(void) { return 0; } int plat_mem_read(uint64_t a, void *out, size_t n) { (void)a, (void)out, (void)n; return 0; }