diff --git a/docs/architecture.md b/docs/architecture.md index 4561fc0c..69b58511 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -131,7 +131,14 @@ keeps per-core data without locks and relies on that, so the host models it (`ru ring of Vulkan `{x, y, z}` records filled on the GPU. - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link at a new state, an optimised pipeline replacing it from a background thread. A draw whose pipeline is not ready is - skipped when one of its pass's targets was rendered in the previous frame, and waits otherwise. Compute pipelines + skipped when one of its pass's targets was rendered in the previous frame, and waits otherwise. Three things keep + that short on a first visit (`runtime/vk_pipeline.c`): a pipeline whose fragment shader reads a target of its own + pass is fast-linked like any other, the feedback loop flags being part of the fragment output library alone + (`gpl_feedback`); the library workers also link the pipelines of skipped draws, so a burst of new materials is + linked on several threads while the pool is inside optimised builds (`lib_worker`, `fast_take`); and the first + draw of a submission that misses has the rest of the submission read ahead, which queues the stage libraries of + every pipeline the later draws need before the replay reaches them (`runtime/vk_exec.c` `vk_exec_ahead`, + `vk_pipeline_hint`). Mesh pipelines have no libraries and are built whole. Compute pipelines are compiled outside the pipeline lock, and one compile serves every thread that needs it. The boot builds the pipeline of every compute shader play recorded or the shader preparation listed, but keeps only those of the shaders dispatched most recently (`compute-used.bin`, `compute_keep`): the others go into the pipeline cache and are diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 4cfc2fd6..f0b1fc0b 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -96,6 +96,16 @@ # How many milliseconds the draws of one frame may wait, in all, for a missing pipeline to be linked from the shader # libraries in the graphics driver's cache before they are dropped (0 to 100; 0: dropped at once, as before). # pipe_link_ms = 6 +# When a draw finds its pipeline missing, the rest of the commands the game submitted with it are read ahead and the +# shader libraries of every other pipeline they will need are built at once on background threads, so the later draws +# find theirs ready instead of waiting one after another or being left out of a frame. 0: off, as before. +# pipe_ahead = 1 +# A pipeline whose fragment shader reads the picture it draws into (soft particles, water, some full-screen passes) is +# linked from the shader libraries like any other. 0: as before, such a draw waits for the whole compile. +# gpl_feedback = 1 +# The threads that build shader libraries also link the pipelines of draws that were left out of a frame, several at +# once, instead of one thread linking them one after another while the others build optimised pipelines. 0: as before. +# pipe_fast_workers = 1 # The graphics driver's cache of shader libraries is checked at every start and, when it lost them (a driver update, # a cache the driver emptied), filled again in the background while the game runs. 0: no check; force: at every start. # After two cold starts in a row warming stops being tried on that driver; once its cache can hold the libraries (on diff --git a/runtime/cfg.c b/runtime/cfg.c index 9101a189..2605166f 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -42,8 +42,8 @@ static const char *const known[] = { "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLSS_IDLE_FREE", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", - "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES", - "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST", + "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES", + "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_LINK_DRAW_MS", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST", "HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE", "HDR_LIVE", "HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", @@ -115,7 +115,7 @@ static const struct { const char *key, *section; } sections[] = { {"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"}, - {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"}, + {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"}, {"COMPUTE_KEEP", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, @@ -228,6 +228,10 @@ static const CfgMeta metas[] = { // shaders and pipelines {"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "any", "strict|any|fullscreen", RESTART}, {"PIPE_LINK_MS", "6", "0..100", RESTART}, // vk_exec_draw.c link_budget + {"PIPE_LINK_DRAW_MS", "3", "1..50", RESTART}, // vk_exec_draw.c link_draw_ms + {"PIPE_AHEAD", "1", "0|1", RESTART}, // vk_exec.c vk_exec_ahead, vk_pipeline.c vk_pipeline_hint + {"GPL_FEEDBACK", "1", "0|1", RESTART}, // vk_pipeline.c gpl_feedback + {"PIPE_FAST_WORKERS", "1", "0|1", RESTART}, // vk_pipeline.c lib_worker, fast_take {"FRAME_HOLD_MS", "120", "0..500", RESTART}, // vk_backend.c frame_hold {"PACE_SMOOTH", "1", "0|1", RESTART}, // vk_backend.c pace_smooth {"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART}, diff --git a/runtime/flight.c b/runtime/flight.c index 0e363ed7..85f33204 100644 --- a/runtime/flight.c +++ b/runtime/flight.c @@ -2,8 +2,10 @@ // render-thread blocked time per perf kind, event counts) and a 320x180 thumbnail each (gfx_present.c, blitted on the // GPU by the present thread). A dump goes to /-f/: frames.tsv and f.png per // thumbnail, written by a background thread. Triggers: F8 while the game window has focus, a frame interval over -// NX2NAVI_FLIGHT_SPIKE_MS (100; 0 off; at most one dump per 10 s), and NX2NAVI_FLIGHT_AT=[,...]. A dump holds -// the frames up to FLIGHT_POST presents after the trigger, so the aftermath is in it too. tools/flight_view.py reads it. +// NX2NAVI_FLIGHT_SPIKE_MS (100; 0 off; at most one dump per 10 s), NX2NAVI_FLIGHT_AT=[,...] and +// NX2NAVI_FLIGHT_EVERY= (every n-th frame: with n under the ring's length the dumps of a run hold every frame of it +// but the last n). A dump holds the frames up to FLIGHT_POST presents after the trigger, so the aftermath is in it +// too. tools/flight_view.py reads it. // // Records come from perf_present (prof_win.c) on the render thread; the counts are differences of the NX2NAVI_TRACE=perf // running totals and of vk_exec_frame. Off, the only cost is the flight_on branch in perf_present and vk_backend.c. @@ -53,7 +55,7 @@ static uint64_t recorded; // records written so far static uint64_t spike_ns, t0_ns; static uint64_t dump_at, trig_frame, trig_interval_us, last_spike_ns; static const char *trig_why; -static uint64_t at_frames[16]; +static uint64_t at_frames[16], every; static int n_at; static char base_dir[PLAT_PATH_MAX]; static uint64_t prev_cnt[PERF_CNT_N], prev_xlated; @@ -79,6 +81,7 @@ void flight_init(void) { at_frames[n_at++] = f; p = *end == ',' ? end + 1 : end; } + every = env_u("NX2NAVI_FLIGHT_EVERY", 0); const char *d = cfg_get("NX2NAVI_FLIGHT_DIR"); char tmp[PLAT_PATH_MAX]; if (d && *d) snprintf(tmp, sizeof tmp, "%s", d); @@ -354,6 +357,7 @@ void flight_frame(uint64_t frame, uint64_t now_ns, uint64_t interval_ns, const u key_was = key; for (int i = 0; i < n_at; i++) if (at_frames[i] == frame) trigger(frame, interval_ns, "NX2NAVI_FLIGHT_AT"); + if (every && frame && frame % every == 0) trigger(frame, interval_ns, "NX2NAVI_FLIGHT_EVERY"); if (spike_ns && interval_ns > spike_ns && recorded > 30 && (!last_spike_ns || now_ns - last_spike_ns >= FLIGHT_SPIKE_GAP_NS)) { if (!dump_at) last_spike_ns = now_ns; trigger(frame, interval_ns, "spike"); diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index dbc6e367..fd87ad2e 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -42,6 +42,7 @@ typedef struct { // NVNmappingRequest _Static_assert(sizeof(nvn_mapping_t) == 0x28, "NVNmappingRequest stride"); const pool_t *nvn_pool_find(uint64_t addr, uint64_t len); // GPU-visible pool holding [addr, addr+len], NULL if none +uint64_t nvn_prog_made_ns(uint64_t serial); // NX2NAVI_PROG_LEAD: when the program was made, 0 unknown uint64_t nvn_new_serial(void); // next tex_t / prog_t serial, from 1 uint64_t nvn_fence_last(void); // fence value of the newest submit (0 before the first) int nvn_depth_minus_w(void); // 1 after DeviceSetDepthMode(NEAR_IS_MINUS_W), else 0 diff --git a/runtime/nvn_api_obj.c b/runtime/nvn_api_obj.c index 87f40703..55758efe 100644 --- a/runtime/nvn_api_obj.c +++ b/runtime/nvn_api_obj.c @@ -443,6 +443,15 @@ NVN_API(nvnProgramFinalize) { prog_release(o); o->magic = 0; } +// NX2NAVI_PROG_LEAD=1 (diagnostics): when each program was made, by serial (direct-mapped; a serial that lost its +// slot reads 0). vk_exec_draw.c prog_lead compares it with the program's first draw. +#define PROG_MADE_N 16384 +static uint64_t prog_made_serial[PROG_MADE_N], prog_made_at[PROG_MADE_N]; +uint64_t nvn_prog_made_ns(uint64_t serial) { + uint32_t i = (uint32_t)serial & (PROG_MADE_N - 1); + return __atomic_load_n(&prog_made_serial[i], __ATOMIC_RELAXED) == serial ? __atomic_load_n(&prog_made_at[i], __ATOMIC_RELAXED) : 0; +} + static int prog_set_shaders(prog_t *o, int n, const NVNshaderData *sd) { if (n < 0 || n > 6 || (n && !sd)) { rt_log("nvn: ProgramSetShaders with %d stages at %p", n, (const void *)sd); @@ -452,6 +461,13 @@ static int prog_set_shaders(prog_t *o, int n, const NVNshaderData *sd) { o->nstage = (uint32_t)n; if (n) memcpy(o->sd, sd, (size_t)n * sizeof *sd); o->serial = nvn_new_serial(); + static int lead = -1; + if (lead < 0) lead = cfg_get("NX2NAVI_PROG_LEAD") != NULL && strcmp(cfg_get("NX2NAVI_PROG_LEAD"), "0") != 0; + if (lead) { + uint32_t i = (uint32_t)o->serial & (PROG_MADE_N - 1); + __atomic_store_n(&prog_made_at[i], plat_now_ns(), __ATOMIC_RELAXED); + __atomic_store_n(&prog_made_serial[i], o->serial, __ATOMIC_RELAXED); + } NVN_LOG("ProgramSetShaders %p n=%d data0=%#llx", (void *)o, n, n ? (unsigned long long)sd[0].data : 0ull); return 1; } diff --git a/runtime/vk.h b/runtime/vk.h index 7aa2a4a5..8a1face5 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -714,6 +714,9 @@ VkPipeline vk_pipeline_graphics(const VkPipeState *s, const VkPipeInfo **info); // wait_ms > 0 waits up to that long for a pending pipeline first. Pipelines built in a run are recorded to // pipeline-keys.bin next to the pipeline cache and prebuilt at the next boot (NX2NAVI_PIPE_PREBUILD=0: not). VkPipeline vk_pipeline_graphics_async(const VkPipeState *s, const VkPipeInfo **info, int *pending, uint32_t wait_ms); +// The stage libraries of a draw the replay has not reached yet, queued to the workers when no pipeline is known for its +// state (vk_pipeline.c "stage libraries ahead of their draws"). Returns the libraries queued. +int vk_pipeline_hint(const VkPipeState *s); // rt_boot calls it unless --shot is given; NX2NAVI_PIPE_SYNC=1 makes it a no-op. Of the probes only --probe-prewarm // calls it. void vk_pipeline_async_enable(void); diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index ce15cf6c..6ac73408 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1216,11 +1216,125 @@ static void evo_resume(VkExec *ex) { } } +// ---- looking ahead for pipelines (NX2NAVI_PIPE_AHEAD, default 1; 0 off) +// A draw whose pipeline is not there yet waits for its fast link or is skipped until a worker has built it, and the +// next draw with a new pipeline is only found when the replay reaches it: an area's first frame, or a turn that brings +// a dozen new materials into view, paid for them one after another (each wait a few milliseconds with the stage +// libraries in the driver's cache, 100 ms and more without), or showed frames with those draws missing while the +// workers that could have built them all at once sat idle. +// The submission is recorded before it is replayed, so what comes after the draw is known. The first draw of a +// submission that finds a shader or pipeline missing has the rest read here (vk_exec_ahead), from the op after it to +// the end of the submission's last command buffer: the scan applies the binds that decide a draw's shaders and its +// shader libraries (program, state objects, vertex attributes, textures and images, render targets, depth clamp) to a +// copy of the replay state and, at each draw, looks its shaders up as the draw will and names the state to +// vk_pipeline_hint, which queues the libraries not built yet. Nothing is recorded and no Vulkan object is touched; +// a shader not in memory is queued for its disk read or translation by the lookup itself. The draws then find their +// libraries built, or building on several workers at once. +// What the scan does not follow: the op stream past a wait for another queue's signal (what follows may be held), a +// change of the texture pool, mesh draws (their pipelines have no libraries), and whatever would take it past +// AHEAD_NS. Render targets are read from their descriptors (format, view format, sample count), which is what the +// replay resolves them to unless a target is left out; a guess that misses costs two libraries at most. +#define AHEAD_NS 3000000ull +#define AHEAD_OPS 400000u +static struct { + int on, live; // on: -1 until read; live: vk_exec_record walks handles[i] now + const uint64_t *handles; + int n, i; + const op_hdr *at[CALL_DEPTH + 1]; // the op being replayed at each call depth + int depth; + uint64_t seq; // the submission scanned last + uint64_t scans, ops, cut, ns, max_ns, logged_scans; +} AH = {.on = -1}; + +static void ahead_set_rt(VkExec *sx, const p_set_rt *r) { + sx->rt = *r; + int n = r->n < 0 ? 0 : r->n > 8 ? 8 : r->n, samples = 0; + for (int i = 0; i < 8; i++) { + const tex_t *t = i < n ? r->color[i] : NULL; + sx->pipe.color_fmt[i] = !t ? 0 : r->has_cview[i] && r->cview[i].set_format ? (int32_t)r->cview[i].format : t->d.format; + if (t && !samples) samples = t->d.samples > 1 ? t->d.samples : 1; + } + sx->pipe.depth_fmt = r->depth ? r->depth->d.format : 0; + if (r->depth && !samples) samples = r->depth->d.samples > 1 ? r->depth->d.samples : 1; + sx->pipe.samples = (uint8_t)(samples ? samples : 1); +} + +void vk_exec_ahead(VkExec *ex) { + if (AH.on < 0) { + AH.on = cfg_bool("NX2NAVI_PIPE_AHEAD"); + rt_log("vk: a draw that finds its pipeline missing has the rest of its submission read ahead for the stage " + "libraries the later draws need: %s (NX2NAVI_PIPE_AHEAD)", AH.on ? "on" : "off"); + } + if (!AH.on || !AH.live || evo_in || AH.seq == vk_exec_submit_seq) return; + AH.seq = vk_exec_submit_seq; + uint64_t t0 = kern_now_ns(); + uint32_t ops = 0; + static VkExec sx; // a copy of the replay state: too large for the stack + sx = *ex; + vk_exec_ahead_begin(); + const op_hdr *ret[CALL_DEPTH + 1]; + int depth = AH.depth, i = AH.i; + for (int d = 0; d < depth; d++) ret[d] = AH.at[d]; + const op_hdr *h = nvn_op_next(AH.at[depth]); + for (;;) { + if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the replay reports it when it gets there + if (++ops >= AHEAD_OPS || (!(ops & 63) && kern_now_ns() - t0 > AHEAD_NS)) { + AH.cut++; + break; + } + const void *a = nvn_op_payload(h); + int stop = 0; + switch (h->op) { + case OP_END: + if (depth > 0) h = ret[--depth]; // back in the caller: on from the op after its call, below + else if (++i < AH.n) { + h = nvn_op_first(AH.handles[i]); + continue; + } else stop = 1; + break; + case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue; + case OP_CALL: + if (depth >= CALL_DEPTH) { + stop = 1; + break; + } + ret[depth++] = h; + h = nvn_op_first(((const p_jump *)a)->target); + continue; + case OP_WAIT_EVENT: stop = 1; break; + case OP_SET_TEX_POOL: stop = ((const p_set_pool *)a)->pool != sx.tex_pool; break; + case OP_SET_RT: + if (memcmp(&sx.rt, a, sizeof sx.rt)) ahead_set_rt(&sx, a); + break; + case OP_DEPTH_CLAMP: sx.pipe.depth_clamp = ((const p_flag *)a)->value != 0; break; + case OP_RASTER_DISCARD: sx.pipe.raster_discard = ((const p_flag *)a)->value != 0; break; + case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_TEX: case OP_BIND_IMG: + case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: + vk_exec_ahead_op(&sx, h->op, a); + break; + default: break; + } + if (stop) break; + h = nvn_op_next(h); + } + uint64_t ns = kern_now_ns() - t0; + AH.scans++, AH.ops += ops, AH.ns += ns; + if (ns > AH.max_ns) AH.max_ns = ns; + if ((rt_trace & RT_TRACE_PERF) && AH.scans - AH.logged_scans >= 100) { + rt_log("perf: looking ahead: %llu scans so far, %.0f ops and %.2f ms each (longest %.2f ms, %llu ended at their " + "limit); %llu draws seen, %llu looked up", (unsigned long long)AH.scans, (double)AH.ops / (double)AH.scans, + (double)AH.ns / 1e6 / (double)AH.scans, (double)AH.max_ns / 1e6, (unsigned long long)AH.cut, + (unsigned long long)vk_exec_ahead_draws, (unsigned long long)vk_exec_ahead_lookups); + AH.logged_scans = AH.scans; + } +} + static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) { for (;;) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size); const void *a = nvn_op_payload(h); + AH.at[depth] = h, AH.depth = depth; // where vk_exec_ahead reads on from switch (h->op) { case OP_END: return NULL; case OP_BEGIN: break; @@ -1405,7 +1519,9 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t if (n) cont_push(slot, cont_new(handles, n, 0, NULL)), evo_held++, vk_exec_evt('p', NULL, 0); } else { evo_queue = evo ? gfx_submit_queue : 0; + AH.handles = handles, AH.n = n, AH.live = 1; for (int i = 0; i < n; i++) { + AH.i = i; const op_hdr *stop = walk(ex, nvn_op_first(handles[i]), 0); if (!stop) continue; cont_push(slot, cont_new(handles, n, i, stop)); // the rest of this submission waits for its signal @@ -1413,6 +1529,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_evt('h', NULL, 0); break; } + AH.live = 0; evo_queue = 0; } vk_exec_pass_end(ex); diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 6dbbfee6..6ce192c6 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -162,6 +162,14 @@ void vk_exec_noatt_dims(const VkExec *ex, uint32_t *w, uint32_t *h); // pass s // rendered for the first time) was rendered in the previous presented frame (per-frame targets); 0 otherwise, or when // the pass has none. int vk_exec_targets_recent(VkExec *ex, int any); +// Looking ahead (vk_exec.c): called by a draw that found its shader or pipeline missing, before it waits or is skipped. +// The rest of the submission being replayed is read once and the stage libraries of the pipelines its later draws need +// are queued to the workers (vk_pipeline_hint). vk_exec_ahead_op (vk_exec_draw.c) applies one bind or draw op of that +// scan to sx, a copy of the replay state; vk_exec_ahead_begin starts a scan there. +void vk_exec_ahead(VkExec *ex); +void vk_exec_ahead_begin(void); +void vk_exec_ahead_op(VkExec *sx, uint32_t op, const void *payload); +extern uint64_t vk_exec_ahead_draws, vk_exec_ahead_lookups; // draws the scans saw, and those whose shaders they looked up // The current pass's targets with their formats, extents and the frames they were last rendered in (diagnostics). void vk_exec_targets_describe(VkExec *ex, char *buf, size_t cap); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 66685a5d..4f375ebe 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -3341,7 +3341,15 @@ static int pipe_skip_mode(void) { // cold it runs out after a few draws and the rest are skipped as before, so a burst of cold pipelines costs a frame // this much and no more. Measured on an RX 7900 XT, first walk through a village with no pipeline recorded: 2,349 // draws skipped without the budget, with the libraries warm in both runs. -#define LINK_DRAW_MS 3u +static uint32_t link_draw_ms(void) { + static int ms = -1; + if (ms < 0) { + long long v = cfg_int("NX2NAVI_PIPE_LINK_DRAW_MS"); + ms = v < 1 ? 1 : v > 50 ? 50 : (int)v; + } + return (uint32_t)ms; +} +#define LINK_DRAW_MS link_draw_ms() // How long a full-screen draw waits on for its pipeline once its frame budget is spent and the previous picture // cannot be held any longer (prepare_draw_ex): the longest optimised build of such a pipeline measured was 194 ms. #define FULLSCREEN_LAST_MS 500u @@ -3351,7 +3359,8 @@ static uint64_t *link_budget(void) { long long ms = cfg_int("NX2NAVI_PIPE_LINK_MS"); per_frame = (uint64_t)(ms < 0 ? 0 : ms > 100 ? 100 : ms) * 1000000ull; rt_log("vk_exec: draws wait up to %llu ms a frame in all for the fast link of a missing pipeline " - "(NX2NAVI_PIPE_LINK_MS)", (unsigned long long)(per_frame / 1000000)); + "(NX2NAVI_PIPE_LINK_MS), %u ms at most each (NX2NAVI_PIPE_LINK_DRAW_MS)", + (unsigned long long)(per_frame / 1000000), LINK_DRAW_MS); } uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (f != frame) frame = f, left = per_frame; @@ -3457,6 +3466,7 @@ uint32_t vk_exec_pending_draws_take(void) { static int draw_skipped(const VkExec *ex, int mesh, int why) { DRS.skipped[mesh][why]++; if (why == SK_SHADER_PENDING || why == SK_PIPE_PENDING) pending_draws++; + if (why == SK_PIPE_PENDING) perf_count(PERF_CNT_PIPE_SKIP); // skipped draws only: a lookup that then waited is none if (!(DRS.first[mesh] >> why & 1)) { DRS.first[mesh] |= 1u << why; rt_log("vk: first %s draw skipped because: %s (program %llu)", mesh ? "mesh" : "vertex", sk_names[why], @@ -3489,6 +3499,40 @@ static int skip_prog_wanted(uint64_t serial) { return 0; } +// NX2NAVI_PROG_LEAD=1 (diagnostics): how long before its first draw the game made each program +// (nvnProgramSetShaders, nvn_api_obj.c nvn_prog_made_ns): the head start a build begun at that moment would have over +// one begun at the draw, which is where shaders are looked up and pipelines built today. A line for vertex, mesh and +// compute programs (their first dispatch) each, after every 200 programs first used. +#define PROG_LEAD_N 16384 +static struct { int on; uint64_t seen[PROG_LEAD_N]; uint64_t n[3][6], logged; } PLD = {-1}; + +static int prog_lead_on(void) { + if (PLD.on < 0) PLD.on = cfg_get("NX2NAVI_PROG_LEAD") != NULL && strcmp(cfg_get("NX2NAVI_PROG_LEAD"), "0") != 0; + return PLD.on; +} + +// mesh: 0 a vertex program's draw, 1 a mesh program's, 2 a compute program's dispatch. +static void prog_lead(const prog_t *p, int mesh) { + uint32_t i = (uint32_t)p->serial & (PROG_LEAD_N - 1); + if (PLD.seen[i] == p->serial) return; + PLD.seen[i] = p->serial; + uint64_t made = nvn_prog_made_ns(p->serial); + if (!made) return; + double ms = (double)(plat_now_ns() - made) / 1e6; + PLD.n[mesh][ms < 17 ? 0 : ms < 100 ? 1 : ms < 1000 ? 2 : ms < 5000 ? 3 : ms < 30000 ? 4 : 5]++; + uint64_t tot = 0; + for (int m = 0; m < 3; m++) + for (int k = 0; k < 6; k++) tot += PLD.n[m][k]; + if (tot - PLD.logged < 200) return; + PLD.logged = tot; + static const char *const kind[3] = {"vertex", "mesh", "compute"}; + for (int m = 0; m < 3; m++) + rt_log("vk: %s programs were made before their first use by: under 17 ms %llu, 17-100 ms %llu, 0.1-1 s %llu, 1-5 s " + "%llu, 5-30 s %llu, longer %llu (NX2NAVI_PROG_LEAD)", kind[m], (unsigned long long)PLD.n[m][0], + (unsigned long long)PLD.n[m][1], (unsigned long long)PLD.n[m][2], (unsigned long long)PLD.n[m][3], + (unsigned long long)PLD.n[m][4], (unsigned long long)PLD.n[m][5]); +} + // mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing // vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) { @@ -3506,6 +3550,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int VkRect2D sc = vk_exec_scissor(ex); if (!sc.extent.width || !sc.extent.height) return draw_skipped(ex, mesh, SK_SCISSOR); if (!ex->prog) return draw_skipped(ex, mesh, SK_NO_PROGRAM); + if (prog_lead_on()) prog_lead(ex->prog, mesh); if (n_skip_prog && skip_prog_wanted(ex->prog->serial)) return draw_skipped(ex, mesh, SK_DISABLED); // -1 until read int has_vs, has_fs, vs_wait = 0, fs_wait = 0, memo = mesh ? 0 : shader_memo_on(); int has_ts = 0, ts_wait = 0, vst = mesh ? NVN_STAGE_MESH : NVN_STAGE_VS; @@ -3533,6 +3578,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int // session. A draw into per-frame targets is skipped until they are ready and comes back next frame. int oneshot = -1, skip = pipe_skip_mode(); // -1: not looked up if (vs_wait || fs_wait || ts_wait) { + vk_exec_ahead(ex); // the later draws' shaders and stage libraries are queued while this one waits or is skipped if (!(oneshot = !vk_exec_targets_recent(ex, skip >= 1))) // skipped until ready return perf_count(PERF_CNT_SHADER_SKIP), draw_skipped(ex, mesh, SK_SHADER_PENDING); perf_count(PERF_CNT_ONESHOT_WAIT); @@ -3595,15 +3641,15 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int info = PM.info, pending = 0; p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE); } else { - if (linking) { // nearly every draw finds its pipeline: the clock is read only for one that did not - p = vk_pipeline_graphics_async(&s, &info, &pending, 0); - if (pending) { - t0 = kern_now_ns(); - p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); - } - } else { - w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait - p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); + // Nearly every draw finds its pipeline, so the first lookup waits for nothing and the clock is read only for a + // draw that missed. A miss first has the rest of the submission read ahead (vk_exec_ahead: the stage libraries + // of the later draws' pipelines build on the workers from here on), inside this draw's wait budget. + p = vk_pipeline_graphics_async(&s, &info, &pending, 0); + if (pending) { + if (linking) t0 = kern_now_ns(); + else w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; + vk_exec_ahead(ex); + if (wait_ms) p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms); } if (t0 && budget) { uint64_t spent = kern_now_ns() - t0; @@ -3641,9 +3687,10 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int char tg[512]; wait_logs++; vk_exec_targets_describe(ex, tg, sizeof tg); - rt_log("vk_exec: frame %llu: a draw waited %.1f ms for its pipeline (%s%s): %u vertices, pass %ux%u, %s", + rt_log("vk_exec: frame %llu: a draw waited %.1f ms for its pipeline (%s%s%s): %u vertices, pass %ux%u, %s", (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1, waited / 1e6, why, - pending ? ", skipped after all" : "", count, ex->pass_w, ex->pass_h, tg); + s.feedback ? ", feedback loop" : "", pending ? ", skipped after all" : "", count, ex->pass_w, ex->pass_h, + tg); } if (pending) return draw_skipped(ex, mesh, SK_PIPE_PENDING); // a background worker is building it: skipped until it is ready if (!p || !info) return draw_skipped(ex, mesh, SK_PIPE_FAILED); @@ -3692,6 +3739,89 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { return prepare_draw_ex(ex, prim, count, restart, 0); } +// ---- looking ahead (vk_exec.c vk_exec_ahead): the binds and draws of ops the replay has not reached, applied to sx, a +// copy of the replay state. A draw looks its shaders up as prepare_draw_ex will (one not in memory is queued for its +// disk read or translation by that lookup) and names its state to vk_pipeline_hint. The last draw looked up is kept: +// one with the same program, texture binds and state repeats it, as most draws of a pass do. +uint64_t vk_exec_ahead_draws, vk_exec_ahead_lookups; +static struct { int valid; const prog_t *prog; uint64_t serial; uint32_t tex_serial; VkPipeState pipe; } AHL; + +void vk_exec_ahead_begin(void) { AHL.valid = 0; } + +static void ahead_draw(VkExec *sx) { + vk_exec_ahead_draws++; + if (!sx->prog) return; + if (AHL.valid && AHL.prog == sx->prog && AHL.serial == sx->prog->serial && AHL.tex_serial == sx->tex_serial && + !memcmp(&AHL.pipe, &sx->pipe, sizeof AHL.pipe)) + return; + AHL.valid = 1, AHL.prog = sx->prog, AHL.serial = sx->prog->serial, AHL.tex_serial = sx->tex_serial, AHL.pipe = sx->pipe; + vk_exec_ahead_lookups++; + int has_vs, has_fs, vs_wait = 0, fs_wait = 0; + const GfxShader *vs = shader_for(sx, NVN_STAGE_VS, &has_vs, &vs_wait, 0); + if (!has_vs) return; // a mesh program, or none: no libraries + const GfxShader *fs = shader_for(sx, NVN_STAGE_FS, &has_fs, &fs_wait, 0); + if (!vs || vs_wait || fs_wait || (has_fs && !fs) || shader_skipped(vs) || shader_skipped(fs)) return; + VkPipeState s = sx->pipe; + s.vs = vs, s.fs = fs, s.ts = NULL; + s.feedback = 0; + s.prim = NVN_DRAW_PRIMITIVE_TRIANGLES; + s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0); + s.y_flip = 1; + s.prim_restart = 0; + vk_pipeline_hint(&s); +} + +void vk_exec_ahead_op(VkExec *sx, uint32_t op, const void *payload) { + switch (op) { + case OP_BIND_PROGRAM: { + const p_bind_program *b = payload; + sx->prog = b->prog, sx->prog_stages = b->stages; + break; + } + case OP_BIND_STATE: { + const p_bind_state *b = payload; + blend_st bs; + switch (b->kind) { + case 0: + memcpy(&bs, b->bytes, sizeof bs); + if (bs.target < 8) sx->pipe.blend[bs.target] = bs; + break; + case 1: memcpy(&sx->pipe.color, b->bytes, sizeof sx->pipe.color); break; + case 2: memcpy(&sx->pipe.chmask, b->bytes, sizeof sx->pipe.chmask); break; + case 3: memcpy(&sx->pipe.ms, b->bytes, sizeof sx->pipe.ms); break; + case 4: memcpy(&sx->pipe.poly, b->bytes, sizeof sx->pipe.poly); break; + case 5: memcpy(&sx->pipe.ds, b->bytes, sizeof sx->pipe.ds); break; + } + break; + } + case OP_BIND_VATTR: { + const p_bind_vattr *b = payload; + if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break; + memset(sx->pipe.vattr, 0, sizeof sx->pipe.vattr); + memcpy(&sx->pipe.vattr[b->first], b->st, sizeof(uint32_t) * (size_t)b->n); + break; + } + case OP_BIND_VSTREAM: { + const p_bind_vstream *b = payload; + if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break; + memset(sx->pipe.vstream, 0, sizeof sx->pipe.vstream); + memcpy(&sx->pipe.vstream[b->first], b->st, sizeof(vs_st) * (size_t)b->n); + break; + } + case OP_BIND_TEX: case OP_BIND_IMG: { + const p_bind_handles *b = payload; + int cap = op == OP_BIND_TEX ? 128 : 32; + if (b->stage < 0 || b->stage >= 8 || b->first < 0 || b->n < 0 || b->n > 32 || b->first + b->n > cap) break; + memcpy(op == OP_BIND_TEX ? &sx->tex[b->stage][b->first] : &sx->img[b->stage][b->first], b->h, + sizeof(uint64_t) * (size_t)b->n); + sx->tex_serial++; + break; + } + case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: ahead_draw(sx); break; + default: break; + } +} + // ---- per-object motion // The camera pass (vk_dlaa.c mv_pass) gives everything the camera's motion, so a character that moves with the camera, // or on its own, gets the wrong one and DLSS throws its history away there. Each solid scene draw whose position reads @@ -5020,6 +5150,7 @@ static void untranslated_dispatch_fallback(VkExec *ex) { static void op_dispatch(VkExec *ex, const p_dispatch *d) { if (d->x <= 0 || d->y <= 0 || d->z <= 0) return; if (!ex->prog) return warn_once(W_PROGRAM, "a dispatch without a bound program is skipped"); + if (prog_lead_on()) prog_lead(ex->prog, 2); vk_contact_flush(ex); cb_res(ex); dummies_init(); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 4e6f3d97..2524c5b5 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -76,7 +76,7 @@ static struct { int indep; // stage-independent layouts (vk_caps.gpl): vertex set 0, fragment set 1 int push_stage; // GFX_STAGE_* whose set is pushed with independent layouts int gpl; // draws whose pipeline is building get a fast-linked one (vk_caps.gpl) - struct { uint64_t links, wlinks, dlinks, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g; + struct { uint64_t links, fb_links, wlinks, flinks, dlinks, pr_added, lib_slow[4], ready_n[6], ready_nofast, hints, hint_libs, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double pr_added_ms, lib_slow_ms[4], ready_ms, ready_max, link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g; int vs_outs_all; // NX2NAVI_VS_OUTS: 0 union of paired fragment inputs, 1 (all) all 32 output locations int dyn_vi, dyn_clamp, dyn_poly, dyn_blend_en, dyn_blend_eq, dyn_wmask, dyn_a2c; int f_logic_op, f_depth_clamp, f_fill_non_solid, f_dual_src, f_full_subgroups; @@ -134,6 +134,7 @@ typedef struct Pipe { uint8_t link_queued; // a waiting draw queued its fast link to the workers (LibJob p), not yet run uint8_t used; // compute: a dispatch of this run asked for it (vk_pipeline_compute) uint8_t pre; // compute: built and kept by the boot prebuild (lb_compute) or bg_main + uint64_t t_miss; // graphics: when a draw first missed it (ns), 0 once it is usable (ready_note) size_t key_size; uint8_t key[]; } Pipe; @@ -2619,8 +2620,8 @@ static Lib *libs[LIB_BUCKETS]; typedef struct { uint32_t kind, topo_class, attr_fmt[32], attr_off[32]; uint8_t attr_bind[32], rate[17], pad[3]; } ViKey; typedef struct { uint32_t kind, missing; const GfxShader *vs; uint8_t clamp, poly_mode, minus_w, pad; GfxShaderIo fin[32]; } PrKey; typedef struct { uint32_t kind; uint8_t dsb, samples, a2c, pad; const GfxShader *fs; } FsKey; -typedef struct { uint32_t kind; uint8_t n_color, samples, a2c, logic_op, blend_en, pad[3]; uint32_t color[8], depth, stencil, wmask; - uint8_t blend[8][6]; } FoKey; +typedef struct { uint32_t kind; uint8_t n_color, samples, a2c, logic_op, blend_en, fbl, pad[2]; uint32_t color[8], depth, stencil, wmask; + uint8_t blend[8][6]; } FoKey; // fbl: GKey.fbl, the feedback loop flags of the library (gpl_feedback) static uint64_t qpc(void) { return plat_now_ns(); } static double ms_since(uint64_t t0) { return (double)(qpc() - t0) / 1e6; } @@ -2732,6 +2733,11 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi ri.colorAttachmentCount = k->n_color, ri.pColorAttachmentFormats = (const VkFormat *)k->color; ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil; pi.pColorBlendState = &cb, pi.pMultisampleState = &ms; + // Whether the pipeline may sample an attachment it renders to is part of the fragment output state: the flags + // go on this library alone, and a pipeline linked from it has them (gpl_feedback). + const FoKey *ok = key; + if (ok->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; + if (ok->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; } VkPipelineCreationFeedbackCreateInfo fci; VkPipelineCreationFeedback fb; @@ -2814,6 +2820,8 @@ static VkPipeline lib_build(Lib *b, int kind, const VkPipeState *s, const GKey * plat_mutex_lock(&P.lock); P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms; if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms; + if (kind == LIB_PR && ((const PrKey *)(const void *)b->key)->missing) P.g.pr_added++, P.g.pr_added_ms += ms; + if (ms >= 25) P.g.lib_slow[kind]++, P.g.lib_slow_ms[kind] += ms; } if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind); b->pipe = pipe, b->building = 0; @@ -2897,19 +2905,48 @@ static void lib_keys(const VkPipeState *s, const GKey *k, ViKey *v, PrKey *pk, F fk->kind = LIB_FS, fk->fs = s->fs, fk->dsb = k->dsb, fk->samples = k->samples, fk->a2c = k->a2c; memset(ok, 0, sizeof *ok); ok->kind = LIB_FO, ok->n_color = k->n_color, ok->samples = k->samples, ok->a2c = k->a2c, ok->logic_op = k->logic_op; - ok->blend_en = k->blend_en, ok->depth = k->depth, ok->stencil = k->stencil, ok->wmask = k->wmask; + ok->blend_en = k->blend_en, ok->fbl = k->fbl, ok->depth = k->depth, ok->stencil = k->stencil, ok->wmask = k->wmask; memcpy(ok->color, k->color, sizeof ok->color), memcpy(ok->blend, k->blend, sizeof ok->blend); } +// p, which a draw missed, has its first usable pipeline: how long that took is how long its draws were skipped or +// waited. fast 0: it had no fast link and this is the optimised build. Caller holds P.lock. +static void ready_note(Pipe *p, int fast) { + if (!p->t_miss) return; + double ms = (double)(plat_now_ns() - p->t_miss) / 1e6; + p->t_miss = 0; + P.g.ready_n[ms < 4 ? 0 : ms < 8 ? 1 : ms < 17 ? 2 : ms < 50 ? 3 : ms < 150 ? 4 : 5]++; + P.g.ready_ms += ms; + if (ms > P.g.ready_max) P.g.ready_max = ms; + P.g.ready_nofast += !fast; +} + +// NX2NAVI_GPL_FEEDBACK (default 1; 0 = as before): a pipeline whose fragment shader samples an attachment of its own pass +// (VkPipeState.feedback: soft particles and water reading the depth buffer, decals, full-screen passes that read the +// target they write) gets a fast link like any other. The feedback loop flags belong to the fragment output state, so +// they are part of that library's key (FoKey.fbl) and of nothing else; the shader libraries are the ones the same +// shaders use without a loop. Before, such a pipeline had no fast link at all and its draws waited for, or were +// skipped during, the whole optimised build. On a first visit to Kokiri Forest (RTX 5090, every stage library in the +// driver's cache, DLSS Frame Generation on, so no frame is held back) full-screen draws of such pipelines waited +// 267 ms at the area's first frame and 410 ms in the middle of a walk; linked, 3 ms. +static int gpl_feedback(void) { + static int on = -1; + if (on < 0) { + on = cfg_bool("NX2NAVI_GPL_FEEDBACK"); + if (!on) rt_log("vk_pipeline: gpl: pipelines with a feedback loop get no fast link (NX2NAVI_GPL_FEEDBACK=0)"); + } + return on; +} + // A fast-linked pipeline for s from its stage libraries. Caller holds P.lock. // build 0 (drawing thread): links only when the shader libraries exist (vertex input and fragment output libraries, // which hold no shader, are built here); *final 0 = not yet, a worker builds them. build 1 (worker): builds the // missing libraries and links with P.lock released. VK_NULL_HANDLE with *final 1 when GPL cannot express the pipeline -// (shared layout, feedback loop) or a build failed. +// (shared layout; a feedback loop with NX2NAVI_GPL_FEEDBACK=0) or a build failed. static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc, int build, int *final) { Layout *l = (Layout *)lc; *final = 1; - if (!l->set1 || k->fbl) return VK_NULL_HANDLE; + if (!l->set1 || (k->fbl && !gpl_feedback())) return VK_NULL_HANDLE; uint64_t t0 = qpc(); ViKey vk_; PrKey pk; @@ -2942,7 +2979,7 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc P.g.link_ms += ms; if (ms > P.g.link_max) P.g.link_max = ms; } - if (pipe) P.g.links++; + if (pipe) P.g.links++, P.g.fb_links += k->fbl != 0; else if (!P.g.link_fail++) rt_log("vk_pipeline: gpl: fast link failed; such draws wait or skip as without GPL"); double total = ms_since(t0); if (!build && total > 5.0 && P.g.slow++ < 16) @@ -3186,6 +3223,7 @@ static void u_run(LibJob *lj) { VkPipeline f = gpl_link(&lj->s, &lj->k, lj->l, 1, &final); // releases P.lock while linking if (f && !p->info.pipeline) { p->fast = p->info.pipeline = f; + ready_note(p, 1); job_lower(p->job); // its job now only does the optimised build } else if (f) { vkDestroyPipeline(P.device, f, NULL); // the job published one meanwhile @@ -3201,17 +3239,74 @@ static void u_run(LibJob *lj) { free(lj); } -// A worker for waiting draws' jobs only (U): a draw that waits finds one free even while the pool builds pipelines, -// so the drawing thread never builds a library itself. +// NX2NAVI_PIPE_FAST_WORKERS (default 1; 0 = as before): the library workers also take the fast link of a draw-queue job +// (fast_take). A draw that is skipped rather than waiting leaves its pipeline to the pool, where a job is first +// fast-linked and then goes back in the queue for its optimised build. The pool keeps one worker out of optimised +// builds, so with the others inside one (about 100 ms each with the driver's cache warm, 350 ms cold) that single +// worker linked every new pipeline of a burst, one after another at 5 to 10 ms each. On a first visit to Kokiri +// Forest (RTX 5090, libraries in the driver's cache) 180 of the 444 pipelines draws missed took over 50 ms to become +// usable, the longest 1.3 s, and their draws were missing from every frame until then; with the library workers +// linking, 23 did, 19 of them mesh pipelines, which have no fast link. +static int fast_workers(void) { + static int on = -1; + if (on < 0) on = cfg_bool("NX2NAVI_PIPE_FAST_WORKERS"); + return on; +} + +// The first draw-queue job whose pipeline still lacks its fast link, taken out of the queue; NULL when there is none. +// Caller holds P.lock. +static Job *fast_take(void) { + if (!P.gpl || !fast_workers()) return NULL; + for (Job *j = A.head, *prev = NULL; j; prev = j, j = j->next) { + Pipe *p = j->p; + if (p->info.pipeline || p->fast_tried || p->link_queued || !p->lay || is_mesh(&j->s)) continue; + if (prev) prev->next = j->next; + else A.head = j->next; + if (A.tail == j) A.tail = prev; + j->next = j->prev = NULL; + return j; + } + return NULL; +} + +// A worker for stage libraries and fast links only: a waiting draw's jobs (U) first, so a draw that waits finds one +// free even while the pool builds pipelines and the drawing thread never builds a library itself; then the fast link +// of a draw-queue job (fast_take), whose optimised build goes on to the pool. static void lib_worker(void *arg) { (void)arg; plat_mutex_lock(&P.lock); for (;;) { - while (!U.head) plat_cond_wait(&lib_cv, &P.lock); - LibJob *lj = U.head; - if (!(U.head = lj->next)) U.tail = NULL; - if (lj->b) lj->b->job = NULL; - u_run(lj); + Job *j = NULL; + while (!U.head && !(j = fast_take())) plat_cond_wait(&lib_cv, &P.lock); + if (!j) { + LibJob *lj = U.head; + if (!(U.head = lj->next)) U.tail = NULL; + if (lj->b) lj->b->job = NULL; + u_run(lj); + continue; + } + Pipe *p = j->p; + int final; + p->fast_tried = 1; // no other thread links it meanwhile; a draw that waits for it waits for this link + VkPipeline f = gpl_link(&j->s, &j->k, p->lay, 1, &final); // releases P.lock while building + if (f && !p->info.pipeline) { + p->fast = p->info.pipeline = f, P.g.wlinks++, P.g.flinks++; + ready_note(p, 1); + plat_cond_broadcast(&done_cv); + } else if (f) { + vkDestroyPipeline(P.device, f, NULL); + } + if (p->info.pipeline) { // the optimised build: ahead of the prebuild, behind other draws' misses + j->low = 1, P.g.requeued++; + j->next = A.low; + if (A.low) A.low->prev = j; + else A.low_tail = j; + A.low = j; + } else { // no fast link for this one: its whole build, next in the draw queue + if (!(j->next = A.head)) A.tail = j; + A.head = j; + } + plat_cond_signal(&work_cv); } } @@ -3259,6 +3354,7 @@ static void pipe_worker(void *arg) { j->p->fast_tried = 1; if (f && !j->p->info.pipeline) { j->p->fast = j->p->info.pipeline = f, P.g.wlinks++; + ready_note(j->p, 1); plat_cond_broadcast(&done_cv); j->low = 1, j->worker = -1, P.g.requeued++; j->prev = NULL, j->next = A.low; @@ -3275,6 +3371,7 @@ static void pipe_worker(void *arg) { VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off); plat_mutex_lock(&P.lock); if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one + ready_note(j->p, 0); if (pipe && j->p->fast) P.g.swaps++; j->p->building = 0; j->p->job = NULL; @@ -3300,6 +3397,8 @@ static void pipe_worker(void *arg) { return; } +static int hint_on(void); + // Starts the pool on first use: a quarter of the logical processors, 2 to 4. Caller holds P.lock. static int workers_start(void) { if (A.threads) return 1; @@ -3310,8 +3409,12 @@ static int workers_start(void) { if (!t) break; A.thread[A.threads++] = t; // kept: promote() raises a worker running a prebuild job } - // two for waiting draws' libraries and links (the vertex and fragment libraries of one draw in parallel) - for (int i = 0; A.threads && U.on && P.gpl && i < 2; i++) + // Workers for stage libraries and fast links alone: two (the vertex and fragment libraries of one waiting draw in + // parallel), and as many as the pool has with the look-ahead, which queues the libraries of a whole submission's + // new pipelines at once (vk_pipeline_hint), or with the fast links of skipped draws (lib_worker). They sleep + // unless a library or a fast link is queued. + int nlib = hint_on() || fast_workers() ? n : 2; + for (int i = 0; A.threads && U.on && P.gpl && i < nlib; i++) if (plat_thread_new(lib_worker, NULL, "pipe lib worker")) U.lib_workers++; if (A.threads) rt_log("vk_pipeline: %d background graphics pipeline workers, %d for waiting draws' libraries", A.threads, @@ -3355,6 +3458,7 @@ int vk_pipeline_async_on(void) { return A.on; } static void promote(Pipe *p) { Job *j = p->job; if (!j || !j->low) return; + if (j->prebuild && !j->demanded && !p->info.pipeline) p->t_miss = plat_now_ns(); j->low = 0, j->demanded = 1; if (j->worker < 0) { if (j->prev) j->prev->next = j->next; @@ -3366,6 +3470,7 @@ static void promote(Pipe *p) { else A.head = j; A.tail = j; plat_cond_signal(&work_cv); + if (U.lib_workers) plat_cond_signal(&lib_cv); // its fast link (lib_worker) } else if (A.prio_low[j->worker]) { A.prio_low[j->worker] = 0; plat_thread_set_priority(A.thread[j->worker], PLAT_PRIO_NORMAL); @@ -3425,6 +3530,7 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode } j->p = p; p->building = 1, p->job = j; + if (mode != 2) p->t_miss = plat_now_ns(); if (mode == 2) { j->prev = A.low_tail; if (A.low_tail) A.low_tail->next = j; @@ -3435,6 +3541,7 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode if (A.tail) A.tail->next = j; else A.head = j; A.tail = j; + if (U.lib_workers && fast_workers()) plat_cond_signal(&lib_cv); // its fast link (lib_worker) } A.inflight++; plat_cond_signal(&work_cv); @@ -3536,6 +3643,42 @@ static void lib_wait(const VkPipeState *s, const GKey *k, Layout *l, double end) } } +// ---- stage libraries ahead of their draws (NX2NAVI_PIPE_AHEAD, default 1; 0 off; vk_exec.c "looking ahead") +// The replay names the state of a draw it has not reached yet. When no pipeline is known for it, its pre-rasterization +// and fragment shader libraries are queued to the library workers now, so they build while the draws before it are +// recorded, in parallel with each other and with those of the other draws named: the draw's own lookup then finds both +// and links on its thread in microseconds instead of waiting for them or being skipped while they build. Nothing else +// is entered, no pipeline, no optimised build and no key record: a state the scan guessed wrong costs two libraries +// at most. The feedback loop bits, the topology and the targets do not matter here, as no shader library depends on +// them. Returns the libraries queued. +static int hint_on(void) { + static int on = -1; + if (on < 0) on = cfg_bool("NX2NAVI_PIPE_AHEAD"); + return on; +} + +int vk_pipeline_hint(const VkPipeState *s) { + if (!A.on || !U.on || !hint_on() || !s->vs || !ensure_init() || !P.gpl || !P.indep || is_mesh(s)) return 0; + if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) return 0; + GKey k; + gkey_fill(s, &k); + if (k.topo_class == 3) return 0; + uint64_t h = key_hash(&k, sizeof k); + int queued = 0; + plat_mutex_lock(&P.lock); + if (!pipe_find(&k, sizeof k, h)) { + Bindings B = {0}, F = {0}; + Layout *l = add_stage(&B, s->vs, 0) && (!s->fs || add_stage(&F, s->fs, 0)) ? layout_pair(&B, &F) : NULL; + Lib *b[2]; + uint64_t q0 = U.queued, w0 = U.waits; + if (l && l->set1 && workers_start() && lib_queue(s, &k, l, b)) queued = (int)(U.queued - q0); + U.waits = w0, U.queued = q0; // those count waiting draws' jobs + P.g.hints++, P.g.hint_libs += (uint64_t)queued; + } + plat_mutex_unlock(&P.lock); + return queued; +} + static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int async, int *pending, uint32_t wait_ms) { if (info) *info = NULL; if (pending) *pending = 0; @@ -3594,6 +3737,7 @@ static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int as if (final) p->fast_tried = 1; if (f && !p->info.pipeline) { p->fast = p->info.pipeline = f; + ready_note(p, 1); job_lower(p->job); plat_cond_broadcast(&done_cv); } else if (f) { @@ -3614,7 +3758,6 @@ static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int as VkPipeline pipe = p ? p->info.pipeline : VK_NULL_HANDLE; plat_mutex_unlock(&P.lock); if (wait) { - perf_count(PERF_CNT_PIPE_SKIP); if (pending) *pending = 1; return VK_NULL_HANDLE; } @@ -3650,6 +3793,7 @@ static void warm_builder(void *arg) { VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off); plat_mutex_lock(&P.lock); if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one + ready_note(j->p, 0); if (pipe && j->p->fast) P.g.swaps++; j->p->building = 0; j->p->job = NULL; @@ -5749,14 +5893,27 @@ static int cache_save(uint32_t wait_ms) { static void save_with(uint32_t wait_ms) { if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log(); if (P.ready && P.gpl) { - rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws " + rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; " + "%llu of pipelines with a feedback loop; avg %.2f ms, " + "max %.2f ms, %llu failed), %llu draws " "lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued " "low after their fast link), %llu drawing-thread links over 5 ms; %llu waiting draws built their libraries " "(avg %.2f ms, max %.2f ms)", - (unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, P.g.links ? P.g.link_ms / P.g.links : 0.0, + (unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, (unsigned long long)P.g.flinks, + (unsigned long long)P.g.fb_links, + P.g.links ? P.g.link_ms / P.g.links : 0.0, P.g.link_max, (unsigned long long)P.g.link_fail, (unsigned long long)P.g.defer, (unsigned long long)P.g.swaps, (unsigned long long)P.g.requeued, (unsigned long long)P.g.slow, (unsigned long long)P.g.dlinks, P.g.dlinks ? P.g.dlink_ms / P.g.dlinks : 0.0, P.g.dlink_max); + uint64_t rn = P.g.ready_n[0] + P.g.ready_n[1] + P.g.ready_n[2] + P.g.ready_n[3] + P.g.ready_n[4] + P.g.ready_n[5]; + rt_log("vk_pipeline: gpl: pipelines a draw missed were usable after under 4 ms %llu, 4-8 ms %llu, 8-17 ms %llu, " + "17-50 ms %llu, 50-150 ms %llu, longer %llu (avg %.1f ms, longest %.0f ms); %llu of them had no fast link", + (unsigned long long)P.g.ready_n[0], (unsigned long long)P.g.ready_n[1], (unsigned long long)P.g.ready_n[2], + (unsigned long long)P.g.ready_n[3], (unsigned long long)P.g.ready_n[4], (unsigned long long)P.g.ready_n[5], + rn ? P.g.ready_ms / (double)rn : 0.0, P.g.ready_max, (unsigned long long)P.g.ready_nofast); + if (P.g.hints) + rt_log("vk_pipeline: gpl: looking ahead named %llu pipeline states not known yet and queued %llu stage libraries " + "for them", (unsigned long long)P.g.hints, (unsigned long long)P.g.hint_libs); if (U.waits || U.queued) rt_log("vk_pipeline: gpl: library jobs for waiting draws: %llu queued, %llu built by a worker, %llu by the " "thread that needed them (no worker free), %llu waits ended at their limit; %llu fast links on the %d " @@ -5765,8 +5922,13 @@ static void save_with(uint32_t wait_ms) { (unsigned long long)U.timeouts, (unsigned long long)U.links, U.lib_workers); static const char *const kn[4] = {"vertex input", "pre-rasterization", "fragment shader", "fragment output"}; for (int i = 0; i < 4; i++) - rt_log("vk_pipeline: gpl: %s libraries %llu (avg %.2f ms, max %.2f ms)", kn[i], (unsigned long long)P.g.lib_n[i], - P.g.lib_n[i] ? P.g.lib_ms[i] / P.g.lib_n[i] : 0.0, P.g.lib_max[i]); + rt_log("vk_pipeline: gpl: %s libraries %llu (avg %.2f ms, max %.2f ms; %llu of 25 ms or more, %.0f ms in all)", + kn[i], (unsigned long long)P.g.lib_n[i], P.g.lib_n[i] ? P.g.lib_ms[i] / P.g.lib_n[i] : 0.0, P.g.lib_max[i], + (unsigned long long)P.g.lib_slow[i], P.g.lib_slow_ms[i]); + if (P.g.pr_added) + rt_log("vk_pipeline: gpl: %llu of the pre-rasterization libraries had outputs added for their fragment shaders " + "(avg %.1f ms): another module than the one the preparation built, so the driver's cache has none of them", + (unsigned long long)P.g.pr_added, P.g.pr_added_ms / (double)P.g.pr_added); if (LB.n) rt_log("vk_pipeline: gpl: prebuilt libraries a draw used: pre-rasterization %llu, fragment shader %llu; " "compute pipelines built on the replay thread %llu", (unsigned long long)P.g.pre_used[LIB_PR],