From 22c86f08a038711e6bc304482f8c8fc89c857f0e Mon Sep 17 00:00:00 2001 From: chrg6 Date: Fri, 13 Nov 2026 19:05:14 -0500 Subject: [PATCH] Let DLSS free what it lets go of once the GPU is past it, instead of waiting for the device on the replaying thread. vk_dlaa.c waited for the whole device to be idle, with the queue lock held, before it freed a size's images and feature (free_size: a third scene size, a DLSS model change; stale_sizes), the features release_kept releases before a create, a reactive-mask snapshot (free_snap), the HUD-less picture of another size (hudless_from), prebloom's images (pb_images) and what retire_idle gives back. Those run on the thread that replays the game's submissions, and with the queue lock held the present's flip, the completion thread's hold release and every upload submit waited as well, at each pause-screen size, reload, window size, dynamic resolution step or mode change; AUDIT F-4-02 measured 16.4 to 44.9 ms for release_kept's wait together with the create that followed. Now nothing there waits: - images only the replay uses go to vk_memory.c's deferred destruction, stamped with the submission being recorded; - a DLSS feature goes into a list (vk_up_defer.h) with the timeline value of the last submission that created or evaluated it (Size.feat_fence) and is released on the same thread once the GPU is past that value, which is the rule NGX gives for releasing a feature and keeps every NGX call on one thread; - a size's motion vectors and the HUD-less picture, which frame generation reads at a present, go into that list behind a present gate of 8 presents and only then to the deferred destruction; the device's idle never covered a present still to come, so this covers more than the wait did; - the list is taken at the top of every evaluation, at every pass while no evaluation runs (retire_idle), and where something is let go of (vk_upscaler_let_go). A feature creation refused while released features are still in flight waits on the timeline from the host for those features (vk_upscaler_gpu_wait, no queue touched), takes them and tries once more, with a log line; a full list waits for the device (vk_upscaler_settle). release_kept's features whose last frame the GPU still runs are now released after the create that follows, not before it: none of them is evaluated again, and its log line says how many. NX2NAVI_UPSCALER_IDLE_WAIT (upscaler_idle_wait in [display], default 0, read at start) set to 1 waits for the device at each of these places as up to 0.10.1, for comparison: rows in cfg.c's known[], metas[] and sections[], and an entry in nx2navi.ini.example. retire_idle still waits for the device before the FSR contexts it gives back while there are any; vk_ffx.cpp and vk_ffx4.cpp follow in the next commit. Not reproduced before the change: no DLSS session log here (C-6 rests on the code and on F-4-02's mark). Every runtime file compiles, vk_dlaa.c and cfg.c with -Wall -Wextra and no warning; cfg_rows_check reports 0 problems. Co-Authored-By: Claude Opus 5.5 --- nx2navi.ini.example | 5 + runtime/cfg.c | 4 + runtime/vk_dlaa.c | 272 ++++++++++++++++++++++++++++++++++++++------ runtime/vk_ffx.h | 12 ++ 4 files changed, 259 insertions(+), 34 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 0deb0ae3..43497e14 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -715,6 +715,11 @@ # 0: none is ever freed (about 0.5 GB more video memory after the first pause). 2: as 1, and the networks that stay # are not made again when another is made (shows striped pause screens; for tests). # dlss_idle_free = 1 +# When DLSS or FSR lets go of a network, an FSR context or a picture (a new picture size, the pause screen, a mode +# change). 0: it is freed once the graphics card has finished the frames that used it, and the game does not wait. +# 1: the game waits for the graphics card to finish all its work first, as up to 0.10.1: a hitch of a frame or more +# at each such change. For comparison, or if 0 shows a fault. Read at start. +# upscaler_idle_wait = 0 # One DLSS network for the render sizes of a DLSS mode. 1: the network made for the mode's own render size also takes # the smaller render sizes DLSS allows on it (dynamic resolution's steps with dyn_res = 1, a screen that renders # smaller than play), so a change of size makes no new network and keeps the picture DLSS has built up. 0: every diff --git a/runtime/cfg.c b/runtime/cfg.c index bfce4d7d..1f08b1b8 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -46,6 +46,7 @@ static const char *const known[] = { "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CACHE_DIE", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COMPUTE_KEEP", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DMABUF", "DMABUF_MB", "DRAW_CRUMBS", "GPA_LATE", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_PREBLOOM", "DLSS_RANGE", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", + "UPSCALER_IDLE_WAIT", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "GPU_TIMELINE", "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FREEZE_QUIET_MS", "FREEZE_LOAD_HOLD_MS", "STUTTER_MAP", "PAR_JOBS", "PIPE_RELEASE", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLIGHT_FLASH", "FLIGHT_FLASH_DIFF", "FLIGHT_FLASH_CELLS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "GPL_MESH", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "DRAW_WRITERS_WAIT", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", @@ -127,6 +128,7 @@ static const struct { const char *key, *section; } sections[] = { {"COLOR_GAMMA", "display"}, {"HDR", "display"}, {"HDR_LIVE", "display"}, {"HDR_PEAK_NITS", "display"}, {"HDR_GAME_NITS", "display"}, {"HDR_UI_NITS", "display"}, {"SHADOW_RES", "display"}, {"DLAA", "display"}, {"FSR4", "display"}, {"FSR4_FG", "display"}, {"DLSS", "display"}, {"DLSS_PRESET", "display"}, {"DLSS_IDLE_FREE", "display"}, {"DLSS_RANGE", "display"}, {"DLSS_PREBLOOM", "display"}, {"DLSS_CHAR_MASK", "display"}, {"FXAA", "display"}, {"AA", "display"}, + {"UPSCALER_IDLE_WAIT", "display"}, {"FRAMEGEN", "display"}, {"FRAMEGEN_MULTIPLIER", "display"}, {"FG_MULT", "display"}, {"SKIN_MOTION", "display"}, {"LOW_LATENCY", "display"}, {"REFLEX_LIMIT", "display"}, {"DEVICE_PROFILE", "display"}, {"ASSUME_FOCUS", "display"}, @@ -215,6 +217,8 @@ static const CfgMeta metas[] = { {"SCENE_SHARE", "0.75", "0..1", LIVE}, {"DLAA", "0", "0|1", LIVE}, {"DLSS", "dlaa", "dlaa|quality|balanced|performance|ultra_performance", LIVE}, {"DLSS_PRESET", "default", "default|k|j", LIVE}, {"DLSS_IDLE_FREE", "1", "0|1|2", RESTART}, {"TAA", "dlss", "dlss|fsr3", LIVE}, // vk_dlaa.c fsr3 + // vk_dlaa.c vk_upscaler_idle_wait: 1 waits for the device to be idle before an upscaler frees anything, as up to 0.10.1 + {"UPSCALER_IDLE_WAIT", "0", "0|1", RESTART}, {"DLSS_RANGE", "0", "0|1", RESTART}, // vk_dlaa.c range_on: one DLSS feature for the render sizes of a mode {"DLSS_PREBLOOM", "0", "0|1", LIVE}, // vk_dlaa.c prebloom_run: DLSS or FSR 3.1 before the bloom, not after the tonemap {"FSR4", "0", "0|1", LIVE}, // vk_ffx.cpp vk_ffx_fsr4 diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index e20bf06c..5bf4105c 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -34,6 +34,7 @@ #include "vk_fg.h" #include "vk_ffx.h" #include "vk_rec.h" +#include "vk_up_defer.h" #include "dlaa_mv_spv.h" #include "dlaa_sharpen_spv.h" @@ -495,6 +496,7 @@ static struct { uint32_t min_w, min_h; uint64_t used; // evals count when it last ran uint64_t feat_used; // evals count when feat last ran: behind used while the size runs on another's + uint64_t feat_fence; // the submission that last created or evaluated feat (vk_exec_recording_fence) } sizes[DLSS_SIZES], *last; // last: the one the previous DLAA frame ran with struct Size *host; // the one whose feature that frame ran on: last, or the DLSS mode's (range_host) int stepped; // this evaluation: another render size than the last one's, on the same feature @@ -742,8 +744,148 @@ static void feat_release(NgxHandle *f) { perf_mark_end(PERF_M_UPREL, t0, 0); } -// The images and views of a size, then its feature. The device must be idle: free_size waits for it, vk_dlaa_release -// has. Whatever was not made is skipped, so a half-made size goes the same way. +// ---- what the upscalers let go of while the GPU may still use it +// A size's images and feature (a third scene size, a DLSS model change, a size unused for SIZE_STALE evaluations), the +// features release_kept and stale_sizes release, a snapshot of the reactive mask, the HUD-less picture of another size, +// prebloom's images, what retire_idle gives back, and FSR 3.1's and FSR 4's contexts (vk_ffx.cpp, vk_ffx4.cpp +// context_for) were freed after a wait for the whole device to be idle, on the replaying thread with the queue lock +// held: the present's flip, the completion thread's hold release and every upload submit waited as long, at each +// pause-screen size, reload, window size, dynamic resolution step or mode change (16 to 45 ms in play together with a +// DLSS feature's create). Nothing waits for them now: +// - images only the replay uses go to vk_memory.c's deferred destruction (vk_defer_image, vk_defer_view), stamped with +// the submission being recorded, which comes after every submission that can name them; +// - a DLSS feature goes into dead with the fence of the last submission that created or evaluated it (feat_fence) and +// is released once the GPU is past it, which is what NGX asks of a feature's release; on this thread, which makes +// every NGX call (NGX is not thread safe); +// - an image frame generation can read at a present (a size's motion vectors, the HUD-less picture) goes into dead +// behind a present gate (FG_LATE_PRESENTS), and from there to the deferred destruction; +// - FSR contexts go into lists of their own in vk_ffx.cpp and vk_ffx4.cpp, taken at the same points. +// The lists are taken at every evaluation (evaluate), at every pass while none runs (retire_idle), and where something +// is let go of (vk_upscaler_let_go). What still waits: a feature or context creation refused while ones let go of are +// in flight waits for the GPU to pass those (vk_upscaler_gpu_wait), and a full list for the device (vk_upscaler_settle). +// NX2NAVI_UPSCALER_IDLE_WAIT=1 waits for the device wherever something is let go of, as up to 0.10.1. +enum { UPD_FEAT = 1, UPD_IMG, UPD_VIEW }; // dead's kinds: a DLSS feature, an image with its allocation (aux), a view +static UpdList dead; // DLAA's hook thread only (vk_dlaa_pass and what it runs), as evaluate's state +// A vk_dlaa.c image frame generation can read at a present goes this many presents after it was let go of. present_image +// (vk_backend.c) calls vk_dlaa_frame before it waits for the present thread to take the job before it, so 3 presents +// on, the present job that last read it has been submitted and flipped; 1 more for a present from another thread than +// the submitting one; and 4 more for frame generation that reads it during the flip on queues of its own (vk_dlssg.c: +// inputs valid until the present), which no timeline value of ours covers. The deferred destruction then stamps it with +// the submission being recorded, which comes after those presents. +#define FG_LATE_PRESENTS 8 + +// D.presents from the hook thread (vk_dlaa_frame counts it on the presenting thread). +static uint64_t presents_now(void) { return __atomic_load_n(&D.presents, __ATOMIC_RELAXED); } + +// NX2NAVI_UPSCALER_IDLE_WAIT, read once: 1 waits for the device wherever an upscaler lets go of something. +int vk_upscaler_idle_wait(void) { + static int m = -1; + if (m < 0) m = cfg_int("NX2NAVI_UPSCALER_IDLE_WAIT") == 1; + return m; +} + +// One entry of dead, as upd_take hands it out, timed as "upscaler releases". teardown (vk_dlaa_release: the device is +// idle and about to go): images and views are destroyed at once. +static void dead_release(const UpdEntry *e, void *teardown) { + const VkCtx *c = vk_ctx(); + uint64_t t0 = perf_mark_begin(); + if (e->kind == UPD_FEAT) { + if (N.release) N.release((NgxHandle *)e->handle); + } else if (e->kind == UPD_VIEW) { + if (teardown) vkDestroyImageView(c->device, (VkImageView)e->handle, NULL); + else vk_defer_view((VkImageView)e->handle); + } else if (e->kind == UPD_IMG) { + if (teardown) vmaDestroyImage(c->vma, (VkImage)e->handle, (VmaAllocation)e->aux); + else vk_defer_image((VkImage)e->handle, (VmaAllocation)e->aux); + } + perf_mark_end(PERF_M_UPREL, t0, 0); +} + +// What the upscalers let go of goes once the GPU is past it: DLSS's here, FSR's in vk_ffx.cpp and vk_ffx4.cpp. +static void upscalers_drain(void) { + if (dead.n) upd_take(&dead, vk_sync_fence_done(), presents_now(), dead_release, NULL); +} + +// A wait for the device, then everything the upscalers let go of that a submission made so far names goes. An image a +// present can read keeps its present gate: the device's idle does not cover a present still to come. +void vk_upscaler_settle(void) { + const uint64_t issued = vk_sync_issued(); // read before the wait: every submission up to it is done after it + vk_upscaler_device_idle(); + upd_take(&dead, issued, presents_now(), dead_release, NULL); +} + +void vk_upscaler_let_go(void) { + if (vk_upscaler_idle_wait()) vk_upscaler_settle(); + else upscalers_drain(); +} + +uint64_t vk_upscaler_gpu_wait(uint64_t v) { + const uint64_t issued = vk_sync_issued(); + if (vk_upscaler_idle_wait()) { + vk_upscaler_device_idle(); + return issued; + } + if (v > issued) v = issued; // a submission still being recorded cannot finish while this thread records it + if (v <= vk_sync_fence_done()) return v; + uint64_t t0 = perf_mark_begin(); + vk_queue_lock(), vk_queue_unlock(); // the replay's held submissions go to the queue (gfx_present.c vk_queue_lock) + // a wait on the timeline from the host touches no queue, so no queue's lock is needed (unlike the device's idle) + const VkCtx *c = vk_ctx(); + VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, + .pSemaphores = &c->timeline, .pValues = &v}; + VkResult r = vkWaitSemaphores(c->device, &wi, 2000000000ull); + perf_mark_end(PERF_M_UPWAIT, t0, 0); + if (r == VK_SUCCESS) return v; + vk_device_lost_check(r, "the upscaler's wait for the GPU"); + rt_log("dlaa: the wait for the GPU to pass submission %llu failed (%d): waiting for the device instead", + (unsigned long long)v, (int)r); + vk_upscaler_device_idle(); + return issued; +} + +static void dead_put(int kind, void *h, void *aux, uint64_t fence, uint64_t presents) { + if (upd_put(&dead, kind, h, aux, fence, presents)) return; + // Full, which the GPU's progress and the presents do not let happen: room after a wait for the device, and if a + // present gate still holds every entry, they all go as before this list, the device idle. + static int told; + if (told++ < 4) rt_log("dlaa: %d things DLSS let go of still wait for the GPU or a present: a wait for the device", + UPD_SLOTS); + vk_upscaler_settle(); + if (upd_put(&dead, kind, h, aux, fence, presents)) return; + upd_take_all(&dead, dead_release, NULL); + upd_put(&dead, kind, h, aux, fence, presents); +} + +// A size let go of while the GPU may still use it: the images only the replay uses to the deferred destruction, the +// motion vectors (frame generation's, D.fg_mv) behind the present gate, the feature once the GPU is past its last +// create or evaluation. The entry is free at once. +static void retire_size(struct Size *e) { + vk_defer_view(e->out_view), vk_defer_image(e->out, e->out_mem); + vk_defer_view(e->sharp_view), vk_defer_image(e->sharp, e->sharp_mem); + vk_defer_view(e->mask_view), vk_defer_image(e->mask, e->mask_mem); + for (int i = 0; i < 2; i++) { + vk_defer_view(e->chars_view[i]), vk_defer_image(e->chars[i], e->chars_mem[i]); + vk_defer_view(e->dist_view[i]), vk_defer_image(e->dist[i], e->dist_mem[i]); + } + const uint64_t gate = presents_now() + FG_LATE_PRESENTS; + dead_put(UPD_VIEW, e->mv_view, NULL, 0, gate); // a view before its image: upd_take and vk_memory.c keep the order + dead_put(UPD_IMG, e->mv, e->mv_mem, 0, gate); + if (e->feat) dead_put(UPD_FEAT, e->feat, NULL, e->feat_fence, 0), feat_released = 1; + if (D.host == e) D.host = NULL; + memset(e, 0, sizeof *e); +} + +// A size's feature alone (release_kept, stale_sizes), released once the GPU is past its last create or evaluation. +// count: it counts as a release for release_kept's next call (feat_released). +static void retire_feat(struct Size *s, int count) { + dead_put(UPD_FEAT, s->feat, NULL, s->feat_fence, 0); + s->feat = NULL; + if (count) feat_released = 1; +} + +// The images and views of a size, then its feature. The device must be idle, or the GPU past every use of them: +// vk_dlaa_release (teardown), idle_release (unused for IDLE_FREE_PRESENTS presents). Whatever was not made is +// skipped, so a half-made size goes the same way. static void destroy_size(struct Size *e) { const VkCtx *c = vk_ctx(); if (e->out_view) vkDestroyImageView(c->device, e->out_view, NULL); @@ -765,9 +907,10 @@ static void destroy_size(struct Size *e) { memset(e, 0, sizeof *e); } +// A third scene size takes the least recently used one's entry (size_for): its images may still be in flight. static void free_size(struct Size *e) { - vk_upscaler_device_idle(); // a third scene size: the evicted images may still be in flight - destroy_size(e); + retire_size(e); + vk_upscaler_let_go(); } // The entry for a scene and output size: its output (RGBA16F at the output size, written by DLSS, then blitted into @@ -1425,7 +1568,8 @@ fail: return 0; } -// A snapshot's images and views. The device must be idle: free_snap waits for it, vk_dlaa_release has. +// A snapshot's images and views. The device must be idle (vk_dlaa_release), or the GPU never used them (snap_for's +// half-made pair). static void destroy_snap(struct Snap *s) { const VkCtx *c = vk_ctx(); if (s->before_view) vkDestroyImageView(c->device, s->before_view, NULL); @@ -1435,9 +1579,16 @@ static void destroy_snap(struct Snap *s) { memset(s, 0, sizeof *s); } +// The same for a snapshot the GPU may still use (only the replay reads it): to the deferred destruction. +static void retire_snap(struct Snap *s) { + vk_defer_view(s->before_view), vk_defer_view(s->mask_view); + vk_defer_image(s->before, s->before_mem), vk_defer_image(s->mask, s->mask_mem); + memset(s, 0, sizeof *s); +} + static void free_snap(struct Snap *s) { - vk_upscaler_device_idle(); // a fourth size: the evicted images may still be in flight - destroy_snap(s); + retire_snap(s); // a fourth size: the evicted images may still be in flight + vk_upscaler_let_go(); } static struct Snap *snap_for(VkCommandBuffer cb, uint32_t w, uint32_t h, VkFormat f) { @@ -1452,7 +1603,7 @@ static struct Snap *snap_for(VkCommandBuffer cb, uint32_t w, uint32_t h, VkForma VkImageUsageFlags u = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; if (!image2d(w, h, f, u, &e->before, &e->before_mem, &e->before_view) || !image2d(w, h, VK_FORMAT_R8_UNORM, u | VK_IMAGE_USAGE_STORAGE_BIT, &e->mask, &e->mask_mem, &e->mask_view)) { - free_snap(e); + destroy_snap(e); // what was made of the pair, never used (an evicted one's images went above) return NULL; } VkImageMemoryBarrier2 ib[2]; @@ -1526,6 +1677,17 @@ static void hl_destroy(void) { D.hl_view = VK_NULL_HANDLE, D.hl = D.hl_srgb = VK_NULL_HANDLE; } +// The same for a HUD-less picture the GPU may still use: D.hl, which a present copies for frame generation +// (gfx_present.c hudless_record), behind the present gate; the sRGB copy, which only the replay uses, to the deferred +// destruction. +static void retire_hl(void) { + const uint64_t gate = presents_now() + FG_LATE_PRESENTS; + dead_put(UPD_VIEW, D.hl_view, NULL, 0, gate); + dead_put(UPD_IMG, D.hl, D.hl_mem, 0, gate); + vk_defer_image(D.hl_srgb, D.hl_srgb_mem); + D.hl_view = VK_NULL_HANDLE, D.hl = D.hl_srgb = VK_NULL_HANDLE; +} + // The device is idle here. Release NGX features and parameters before their Vulkan resources, then shut down NGX even // if initialization succeeded but the layout check failed. The Linux core's Shutdown1 takes an extra output pointer // compared with the SDK's public wrapper; it reports the number of devices still using NGX. @@ -1583,6 +1745,7 @@ void vk_dlaa_release(void) { #ifndef _WIN32 const VkCtx *c = vk_ctx(); if (!c) return; + upd_take_all(&dead, dead_release, (void *)1); // what was let go of: the features before NGX goes, the images at once for (int i = 0; i < 5; i++) if (D.sizes[i].feat) N.release(D.sizes[i].feat), D.sizes[i].feat = NULL; if (FG.feat) N.release(FG.feat); // frame generation's feature and parameters (vk_ngxfg_evaluate), before NGX goes @@ -1871,11 +2034,13 @@ static void query_vram(void) { } // Frees the sizes other than cur that no evaluation used for SIZE_STALE evaluations: the GPU finished with them long -// ago (the CPU is never that many presents ahead), so their images and feature go at once. host is the size whose -// feature cur ran on (cur itself, or the DLSS mode's: range_host), which stays as well. A size that is in use on -// another's feature can still hold one of its own from before that other was made: unused as long, it goes alone. +// ago (the CPU is never that many presents ahead), so their images and feature go at once (retire_size, the drain in +// vk_upscaler_let_go). host is the size whose feature cur ran on (cur itself, or the DLSS mode's: range_host), which +// stays as well. A size that is in use on another's feature can still hold one of its own from before that other was +// made: unused as long, it goes alone. static void stale_sizes(const struct Size *cur, const struct Size *host) { if (!idle_free()) return; + int any = 0; for (int i = 0; i < DLSS_SIZES; i++) { struct Size *s = &D.sizes[i]; if (s == host || !s->out) continue; @@ -1883,32 +2048,38 @@ static void stale_sizes(const struct Size *cur, const struct Size *host) { rt_log("dlaa: the DLSS feature at %ux%u -> %ux%u went unused for %d frames: freed", s->w, s->h, s->ow, s->oh, SIZE_STALE); if (D.last == s) D.last = NULL; - free_size(s); + retire_size(s), any = 1; } else if (s->feat && D.evals >= s->feat_used + SIZE_STALE) { rt_log("dlaa: the DLSS feature at %ux%u -> %ux%u went unused for %d frames while its size ran on another " "size's: released", s->w, s->h, s->ow, s->oh, SIZE_STALE); - vk_upscaler_device_idle(); - feat_release(s->feat), s->feat = NULL, feat_released = 1; + retire_feat(s, 1), any = 1; } } + if (any) vk_upscaler_let_go(); } // Before a feature is made: the features of the other sizes with a larger input go, or all of them (all: a feature was // released since the last creation, feat_released); their images stay. The GPU may still be running an evaluation of -// theirs, so it finishes first, as in free_size. +// theirs: those whose last submission it has passed are released here, before the creation, and the others once it +// has (retire_feat, vk_upscaler_let_go), after it. None of them is evaluated again: its size makes a new one. static void release_kept(const struct Size *making, int all) { int n = 0; for (int i = 0; i < DLSS_SIZES; i++) { struct Size *s = &D.sizes[i]; if (s == making || !s->feat) continue; if (!all && (uint64_t)s->w * s->h <= (uint64_t)making->w * making->h) continue; - if (!n++) vk_upscaler_device_idle(); - feat_release(s->feat), s->feat = NULL; + retire_feat(s, 0), n++; if (D.last == s) D.last = NULL; if (D.host == s) D.host = NULL; } - if (n) rt_log("dlaa: %d DLSS feature(s) released before the one for %ux%u is made (%s); made again when used", n, - making->w, making->h, all ? "after a release" : "larger inputs"); + if (!n) return; + vk_upscaler_let_go(); + int left = upd_count(&dead, UPD_FEAT); // older ones have older fences and went first: those left are this call's + if (left > n) left = n; + char later[80] = ""; + if (left) snprintf(later, sizeof later, "; %d of them once the GPU is past their last frame", left); + rt_log("dlaa: %d DLSS feature(s) released before the one for %ux%u is made (%s); made again when used%s", n, + making->w, making->h, all ? "after a release" : "larger inputs", later); } // The outer columns and rows of what a jittered pass draws into. The jitter moves the scene's viewport by under half a @@ -2208,6 +2379,7 @@ static volatile int32_t size_changed; static void fg_only_frame(VkExec *ex, const VkExecTarget *dst); static void evaluate(VkExec *ex, const VkExecTarget *dst) { + upscalers_drain(); // once a frame while the upscaler or frame generation's inputs run: what the GPU is past goes if (D.fgo) { // no upscaler: frame generation's inputs alone, at the same point of the frame fg_only_frame(ex, dst); return; @@ -2221,7 +2393,8 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { vk_gpu_gap_note(VK_GAP_UPSCALER, (uint64_t)ow * oh); if (D.preset_dirty) { // another model: features are created with it, so every size gets a new one for (int i = 0; i < DLSS_SIZES; i++) - if (D.sizes[i].out) free_size(&D.sizes[i]); + if (D.sizes[i].out) retire_size(&D.sizes[i]); + vk_upscaler_let_go(); D.last = NULL, D.preset_dirty = 0; } struct Size *e = size_for(cb, w, h, ow, oh); @@ -2335,7 +2508,25 @@ own_feature: uint64_t t0 = perf_mark_begin(); NgxResult r = N.create(cb, NGX_FEATURE_SUPERSAMPLING, p, &host->feat); perf_mark_end(PERF_M_NGX, t0, 0); + const int inflight = !NGX_OK(r) || !host->feat ? upd_count(&dead, UPD_FEAT) : 0; + if (inflight) { + // Refused while features let go of are still in flight (release_kept, retire_size): their video memory, or + // NGX's refusal of a feature with the parameters of a live one. The GPU finishes with them, they go, and the + // creation is tried once more: the only wait left on this path, for those features, not for the device. + const NgxResult r0 = r; + const uint64_t w0 = plat_now_ns(); + upd_take(&dead, vk_upscaler_gpu_wait(upd_max_fence(&dead)), presents_now(), dead_release, NULL); + const double waited = (plat_now_ns() - w0) / 1e6; + host->feat = NULL; + t0 = perf_mark_begin(); + r = N.create(cb, NGX_FEATURE_SUPERSAMPLING, p, &host->feat); + perf_mark_end(PERF_M_NGX, t0, 0); + rt_log("dlaa: DLSS feature creation refused (%08x) with %d released feature(s) still in flight: waited %.1f " + "ms for them, %s", (unsigned)r0, inflight, waited, + NGX_OK(r) && host->feat ? "made at the second try" : "refused again"); + } if (!NGX_OK(r) || !host->feat) return dl_fail("DLSS feature creation", r); + host->feat_fence = vk_exec_recording_fence(); // NGX records the creation into cb D.reset = 1, D.steps_made += step; int q = perf_quality(fw, ow), pr = q == 0 ? pm : q == 3 ? pl : pk; rt_log("dlaa: DLSS feature created at %ux%u -> %ux%u, quality value %d, preset %c (scene format %d, depth format " @@ -2388,6 +2579,7 @@ own_feature: p_setf(p, "FrameTimeDeltaInMsec", frame_ms); c0 = timed ? kern_now_ns() : 0; NgxResult r = N.evaluate(cb, host->feat, p, NULL); + host->feat_fence = vk_exec_recording_fence(); // recorded into cb (a refusal too, for all this side knows) if (timed) T.cpu_ngx += kern_now_ns() - c0; if (!NGX_OK(r) && host != e) { // DLSS refused this render size on the feature of another, inside the range it reported (range_note). Every @@ -2485,8 +2677,8 @@ static void hudless_from(VkCommandBuffer cb, VkImage src, uint32_t level, uint32 if (vk_hdr_native()) return; VkFormat fmt = hdr ? VK_FORMAT_R16G16B16A16_SFLOAT : VK_FORMAT_R8G8B8A8_UNORM; if (D.hl && (D.hl_w != w || D.hl_h != h || D.hl_fmt != fmt)) { - vk_upscaler_device_idle(); - hl_destroy(); + retire_hl(); // the presents of the last frames may still read it + vk_upscaler_let_go(); } if (!D.hl) { VkImageView v2 = VK_NULL_HANDLE; @@ -2603,13 +2795,22 @@ static void pb_destroy(void) { memset(&PB, 0, sizeof PB); } -// The two images at w x h, made again for another picture size (the old ones after the device is idle, as hudless). +// The same for images the GPU may still use (only the replay reads them): to the deferred destruction. PB.used stays. +static void retire_pb(void) { + vk_defer_view(PB.scene_view), vk_defer_view(PB.comp_view); + vk_defer_image(PB.scene, PB.scene_mem), vk_defer_image(PB.comp, PB.comp_mem); + const uint64_t used = PB.used; + memset(&PB, 0, sizeof PB); + PB.used = used; +} + +// The two images at w x h, made again for another picture size (the old ones once the GPU is past them, as hudless). static int pb_images(VkCommandBuffer cb, uint32_t w, uint32_t h) { PB.used = D.presents; if (PB.scene && PB.w == w && PB.h == h) return 1; if (PB.scene || PB.comp) { - vk_upscaler_device_idle(); - pb_destroy(); + retire_pb(); + vk_upscaler_let_go(); } VkImageUsageFlags u = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; @@ -2633,8 +2834,8 @@ static int pb_images(VkCommandBuffer cb, uint32_t w, uint32_t h) { VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 2, .pImageMemoryBarriers = ib}; vkCmdPipelineBarrier2(cb, &di); - // used again: pb_destroy above cleared it with the old images, and idle_release took the new ones, which this frame - // uses, for 600 presents unused and freed them at the next present, without a wait for the GPU + // used again for the new images, which this frame uses: left at 0 by a clear of the old ones, idle_release took + // them for 600 presents unused and freed them at the next present, without a wait for the GPU PB.w = w, PB.h = h, PB.used = D.presents; rt_log("dlaa: prebloom: the passes after the upscaler draw at %ux%u (two B10G11R11 images, %.0f MiB)", w, h, 2.0 * w * h * 4 / 1048576.0); @@ -2825,9 +3026,9 @@ void vk_dlaa_upscaler(VkExec *ex, VkImage out, VkFormat fmt, uint32_t w, uint32_ // reach stayed for the process: the reactive mask's snapshots, the FSR 3.1 and FSR 4 upscale contexts, and the DLSS // sizes after a failure (dl_fail turns the upscaler off without the setting changing, so D.on stays set). They go // here, RETIRE_PRESENTS presents after the upscaler stopped running and frame generation stopped making inputs of its -// own from a size (inputs_on), on the replay thread, which makes every evaluation, after a wait for the device to be -// idle, as free_size does for a size it replaces. Switched on again, the upscaler makes them anew, as at its first -// frame in a run. +// own from a size (inputs_on), on the replay thread, which makes every evaluation, once the GPU is past them, as +// free_size does for a size it replaces. Switched on again, the upscaler makes them anew, as at its first frame in a +// run. #define RETIRE_PRESENTS 600 static uint64_t retire_off_at; // replay: D.presents when the upscaler stopped running static uint64_t retired_n; // replay: times this gave memory back @@ -2842,7 +3043,9 @@ static int up_holds(void) { } static void retire_idle(void) { - uint64_t now = __atomic_load_n(&D.presents, __ATOMIC_RELAXED); + uint64_t now = presents_now(); + // no evaluation runs to take what the upscalers let go of (evaluate): it goes here once the GPU is past it + if (!inputs_on()) upscalers_drain(); if (!idle_free()) return; // as idle_release if (inputs_on()) { // the upscaler runs, or frame generation alone makes its inputs from a size (D.fgo) retire_off_at = 0; @@ -2850,13 +3053,14 @@ static void retire_idle(void) { } if (!retire_off_at) retire_off_at = now ? now : 1; if (now < retire_off_at + RETIRE_PRESENTS || !up_holds()) return; - vk_upscaler_device_idle(); - for (int i = 0; i < SNAPS; i++) destroy_snap(&R.s[i]); + for (int i = 0; i < SNAPS; i++) retire_snap(&R.s[i]); + if (vk_ffx_holds()) vk_upscaler_device_idle(); // vk_ffx_release_idle destroys the FSR contexts at once vk_ffx_release_idle(); if (D.failed) { // as idle_release does for a size: frame generation is handed nothing of one from here on - for (int i = 0; i < DLSS_SIZES; i++) destroy_size(&D.sizes[i]); + for (int i = 0; i < DLSS_SIZES; i++) retire_size(&D.sizes[i]); D.last = NULL, D.host = NULL, D.fg_fresh = 0, D.fg_mv_view = VK_NULL_HANDLE, D.fg_mv = (VkFfxImage){0}; } + vk_upscaler_let_go(); retired_n++; rt_log("dlaa: the upscaler has not run for %d presents (%s): its mask snapshots and FSR contexts%s are freed (%llu " "time(s)); made again when it runs", RETIRE_PRESENTS, D.failed ? "it failed" : "switched off", diff --git a/runtime/vk_ffx.h b/runtime/vk_ffx.h index 188d75c6..f1e554eb 100644 --- a/runtime/vk_ffx.h +++ b/runtime/vk_ffx.h @@ -19,6 +19,18 @@ float vk_dlaa_fsr3_boost(uint32_t w, uint32_t ow); // vk_dlaa.c: a wait for the device to be idle, the queue lock held, before an upscaler frees what the GPU may still // use (NX2NAVI_TRACE=perf: "upscaler device waits") void vk_upscaler_device_idle(void); +// vk_dlaa.c "what the upscalers let go of while the GPU may still use it", on the thread that makes the upscalers' +// calls (vk_dlaa.c's pass hooks): +int vk_upscaler_idle_wait(void); // NX2NAVI_UPSCALER_IDLE_WAIT: 1 waits for the device wherever one lets go of something +// After something was let go of: what the GPU is past goes (with NX2NAVI_UPSCALER_IDLE_WAIT=1 after a wait for the +// device, vk_upscaler_settle). +void vk_upscaler_let_go(void); +// A wait for the device, then everything let go of that a submission made so far names goes (a present gate stays). +void vk_upscaler_settle(void); +// A refused creation: waits, without the device's idle, until the GPU is past timeline value v (no further than the +// submissions made so far) and returns the value it is past; with NX2NAVI_UPSCALER_IDLE_WAIT=1, or when that wait +// fails, a wait for the device. +uint64_t vk_upscaler_gpu_wait(uint64_t v); int vk_ffx_load(void); // 1 when the library is there with its entry points (loaded once) // Whether FSR 3.1 upscaling can run in this build, for the settings menu: after vk_ffx_load ran, what it found; before,