diff --git a/nx2navi.ini.example b/nx2navi.ini.example index f323a759..d80b29b3 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -99,6 +99,9 @@ # A pipeline whose fragment shader reads the picture it draws into (soft particles, water, some full-screen passes) is # linked from the shader libraries like any other. 0: as before, such a draw waits for the whole compile. # gpl_feedback = 1 +# The threads that build shader libraries also link the pipelines of draws that were left out of a frame, several at +# once, instead of one thread linking them one after another while the others build optimised pipelines. 0: as before. +# pipe_fast_workers = 1 # The graphics driver's cache of shader libraries is checked at every start and, when it lost them (a driver update, # a cache the driver emptied), filled again in the background while the game runs. 0: no check; force: at every start. # lib_warm = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index 4395ce3a..02e9320c 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -43,7 +43,7 @@ static const char *const known[] = { "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLSS_IDLE_FREE", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES", - "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST", + "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_FAST_WORKERS", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST", "HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE", "HDR_LIVE", "HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", @@ -114,7 +114,7 @@ static const struct { const char *key, *section; } sections[] = { {"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"}, - {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"}, + {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"}, {"COMPUTE_KEEP", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, @@ -219,6 +219,7 @@ static const CfgMeta metas[] = { {"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "any", "strict|any|fullscreen", RESTART}, {"PIPE_LINK_MS", "6", "0..100", RESTART}, // vk_exec_draw.c link_budget {"GPL_FEEDBACK", "1", "0|1", RESTART}, // vk_pipeline.c gpl_feedback + {"PIPE_FAST_WORKERS", "1", "0|1", RESTART}, // vk_pipeline.c lib_worker, fast_take {"FRAME_HOLD_MS", "120", "0..500", RESTART}, // vk_backend.c frame_hold {"PACE_SMOOTH", "1", "0|1", RESTART}, // vk_backend.c pace_smooth {"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART}, diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 25973185..7000ae2c 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -75,7 +75,7 @@ static struct { int indep; // stage-independent layouts (vk_caps.gpl): vertex set 0, fragment set 1 int push_stage; // GFX_STAGE_* whose set is pushed with independent layouts int gpl; // draws whose pipeline is building get a fast-linked one (vk_caps.gpl) - struct { uint64_t links, fb_links, wlinks, dlinks, pr_added, lib_slow[4], ready_n[6], ready_nofast, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double pr_added_ms, lib_slow_ms[4], ready_ms, ready_max, link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g; + struct { uint64_t links, fb_links, wlinks, flinks, dlinks, pr_added, lib_slow[4], ready_n[6], ready_nofast, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double pr_added_ms, lib_slow_ms[4], ready_ms, ready_max, link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g; int vs_outs_all; // NX2NAVI_VS_OUTS: 0 union of paired fragment inputs, 1 (all) all 32 output locations int dyn_vi, dyn_clamp, dyn_poly, dyn_blend_en, dyn_blend_eq, dyn_wmask, dyn_a2c; int f_logic_op, f_depth_clamp, f_fill_non_solid, f_dual_src, f_full_subgroups; @@ -3238,17 +3238,74 @@ static void u_run(LibJob *lj) { free(lj); } -// A worker for waiting draws' jobs only (U): a draw that waits finds one free even while the pool builds pipelines, -// so the drawing thread never builds a library itself. +// NX2NAVI_PIPE_FAST_WORKERS (default 1; 0 = as before): the library workers also take the fast link of a draw-queue job +// (fast_take). A draw that is skipped rather than waiting leaves its pipeline to the pool, where a job is first +// fast-linked and then goes back in the queue for its optimised build. The pool keeps one worker out of optimised +// builds, so with the others inside one (about 100 ms each with the driver's cache warm, 350 ms cold) that single +// worker linked every new pipeline of a burst, one after another at 5 to 10 ms each. On a first visit to Kokiri +// Forest (RTX 5090, libraries in the driver's cache) 180 of the 444 pipelines draws missed took over 50 ms to become +// usable, the longest 1.3 s, and their draws were missing from every frame until then; with the library workers +// linking, 23 did, 19 of them mesh pipelines, which have no fast link. +static int fast_workers(void) { + static int on = -1; + if (on < 0) on = cfg_bool("NX2NAVI_PIPE_FAST_WORKERS"); + return on; +} + +// The first draw-queue job whose pipeline still lacks its fast link, taken out of the queue; NULL when there is none. +// Caller holds P.lock. +static Job *fast_take(void) { + if (!P.gpl || !fast_workers()) return NULL; + for (Job *j = A.head, *prev = NULL; j; prev = j, j = j->next) { + Pipe *p = j->p; + if (p->info.pipeline || p->fast_tried || p->link_queued || !p->lay || is_mesh(&j->s)) continue; + if (prev) prev->next = j->next; + else A.head = j->next; + if (A.tail == j) A.tail = prev; + j->next = j->prev = NULL; + return j; + } + return NULL; +} + +// A worker for stage libraries and fast links only: a waiting draw's jobs (U) first, so a draw that waits finds one +// free even while the pool builds pipelines and the drawing thread never builds a library itself; then the fast link +// of a draw-queue job (fast_take), whose optimised build goes on to the pool. static void lib_worker(void *arg) { (void)arg; plat_mutex_lock(&P.lock); for (;;) { - while (!U.head) plat_cond_wait(&lib_cv, &P.lock); - LibJob *lj = U.head; - if (!(U.head = lj->next)) U.tail = NULL; - if (lj->b) lj->b->job = NULL; - u_run(lj); + Job *j = NULL; + while (!U.head && !(j = fast_take())) plat_cond_wait(&lib_cv, &P.lock); + if (!j) { + LibJob *lj = U.head; + if (!(U.head = lj->next)) U.tail = NULL; + if (lj->b) lj->b->job = NULL; + u_run(lj); + continue; + } + Pipe *p = j->p; + int final; + p->fast_tried = 1; // no other thread links it meanwhile; a draw that waits for it waits for this link + VkPipeline f = gpl_link(&j->s, &j->k, p->lay, 1, &final); // releases P.lock while building + if (f && !p->info.pipeline) { + p->fast = p->info.pipeline = f, P.g.wlinks++, P.g.flinks++; + ready_note(p, 1); + plat_cond_broadcast(&done_cv); + } else if (f) { + vkDestroyPipeline(P.device, f, NULL); + } + if (p->info.pipeline) { // the optimised build: ahead of the prebuild, behind other draws' misses + j->low = 1, P.g.requeued++; + j->next = A.low; + if (A.low) A.low->prev = j; + else A.low_tail = j; + A.low = j; + } else { // no fast link for this one: its whole build, next in the draw queue + if (!(j->next = A.head)) A.tail = j; + A.head = j; + } + plat_cond_signal(&work_cv); } } @@ -3349,8 +3406,11 @@ static int workers_start(void) { if (!t) break; A.thread[A.threads++] = t; // kept: promote() raises a worker running a prebuild job } - // two for waiting draws' libraries and links (the vertex and fragment libraries of one draw in parallel) - for (int i = 0; A.threads && U.on && P.gpl && i < 2; i++) + // Workers for stage libraries and fast links alone: two (the vertex and fragment libraries of one waiting draw in + // parallel), and as many as the pool has with the fast links of skipped draws (lib_worker). They sleep unless a + // library or a fast link is queued. + int nlib = fast_workers() ? n : 2; + for (int i = 0; A.threads && U.on && P.gpl && i < nlib; i++) if (plat_thread_new(lib_worker, NULL, "pipe lib worker")) U.lib_workers++; if (A.threads) rt_log("vk_pipeline: %d background graphics pipeline workers, %d for waiting draws' libraries", A.threads, @@ -3406,6 +3466,7 @@ static void promote(Pipe *p) { else A.head = j; A.tail = j; plat_cond_signal(&work_cv); + if (U.lib_workers) plat_cond_signal(&lib_cv); // its fast link (lib_worker) } else if (A.prio_low[j->worker]) { A.prio_low[j->worker] = 0; plat_thread_set_priority(A.thread[j->worker], PLAT_PRIO_NORMAL); @@ -3476,6 +3537,7 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode if (A.tail) A.tail->next = j; else A.head = j; A.tail = j; + if (U.lib_workers && fast_workers()) plat_cond_signal(&lib_cv); // its fast link (lib_worker) } A.inflight++; plat_cond_signal(&work_cv); @@ -5784,12 +5846,13 @@ static int cache_save(uint32_t wait_ms) { static void save_with(uint32_t wait_ms) { if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log(); if (P.ready && P.gpl) { - rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of pipelines with a feedback loop; avg %.2f ms, " + rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; " + "%llu of pipelines with a feedback loop; avg %.2f ms, " "max %.2f ms, %llu failed), %llu draws " "lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued " "low after their fast link), %llu drawing-thread links over 5 ms; %llu waiting draws built their libraries " "(avg %.2f ms, max %.2f ms)", - (unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, + (unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, (unsigned long long)P.g.flinks, (unsigned long long)P.g.fb_links, P.g.links ? P.g.link_ms / P.g.links : 0.0, P.g.link_max, (unsigned long long)P.g.link_fail, (unsigned long long)P.g.defer,