Merge the shader stutter fixes into the combined test line.

Eight commits: a draw that misses its pipeline has the rest of its submission read ahead, the library workers link
the pipelines of skipped draws, a pipeline whose fragment shader reads a target of its own pass is fast-linked like
any other, and diagnostics. The merge was clean. Their before and after matrix had not finished when this was merged.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 11:05:17 -05:00
11 changed files with 504 additions and 41 deletions

View File

@@ -42,8 +42,8 @@ static const char *const known[] = {
"COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT",
"DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLSS_IDLE_FREE", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
"DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES",
"FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES",
"FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST",
"FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES",
"FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_LINK_DRAW_MS", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST",
"HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE",
"HDR_LIVE",
"HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
@@ -115,7 +115,7 @@ static const struct { const char *key, *section; } sections[] = {
{"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"},
{"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"},
{"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"},
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"},
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"},
{"COMPUTE_KEEP", "performance"},
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
{"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
@@ -228,6 +228,10 @@ static const CfgMeta metas[] = {
// shaders and pipelines
{"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "any", "strict|any|fullscreen", RESTART},
{"PIPE_LINK_MS", "6", "0..100", RESTART}, // vk_exec_draw.c link_budget
{"PIPE_LINK_DRAW_MS", "3", "1..50", RESTART}, // vk_exec_draw.c link_draw_ms
{"PIPE_AHEAD", "1", "0|1", RESTART}, // vk_exec.c vk_exec_ahead, vk_pipeline.c vk_pipeline_hint
{"GPL_FEEDBACK", "1", "0|1", RESTART}, // vk_pipeline.c gpl_feedback
{"PIPE_FAST_WORKERS", "1", "0|1", RESTART}, // vk_pipeline.c lib_worker, fast_take
{"FRAME_HOLD_MS", "120", "0..500", RESTART}, // vk_backend.c frame_hold
{"PACE_SMOOTH", "1", "0|1", RESTART}, // vk_backend.c pace_smooth
{"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART},

View File

@@ -2,8 +2,10 @@
// render-thread blocked time per perf kind, event counts) and a 320x180 thumbnail each (gfx_present.c, blitted on the
// GPU by the present thread). A dump goes to <dir>/<YYYYMMDD-HHMMSS>-f<frame>/: frames.tsv and f<frame>.png per
// thumbnail, written by a background thread. Triggers: F8 while the game window has focus, a frame interval over
// NX2NAVI_FLIGHT_SPIKE_MS (100; 0 off; at most one dump per 10 s), and NX2NAVI_FLIGHT_AT=<frame>[,<frame>...]. A dump holds
// the frames up to FLIGHT_POST presents after the trigger, so the aftermath is in it too. tools/flight_view.py reads it.
// NX2NAVI_FLIGHT_SPIKE_MS (100; 0 off; at most one dump per 10 s), NX2NAVI_FLIGHT_AT=<frame>[,<frame>...] and
// NX2NAVI_FLIGHT_EVERY=<n> (every n-th frame: with n under the ring's length the dumps of a run hold every frame of it
// but the last n). A dump holds the frames up to FLIGHT_POST presents after the trigger, so the aftermath is in it
// too. tools/flight_view.py reads it.
//
// Records come from perf_present (prof_win.c) on the render thread; the counts are differences of the NX2NAVI_TRACE=perf
// running totals and of vk_exec_frame. Off, the only cost is the flight_on branch in perf_present and vk_backend.c.
@@ -53,7 +55,7 @@ static uint64_t recorded; // records written so far
static uint64_t spike_ns, t0_ns;
static uint64_t dump_at, trig_frame, trig_interval_us, last_spike_ns;
static const char *trig_why;
static uint64_t at_frames[16];
static uint64_t at_frames[16], every;
static int n_at;
static char base_dir[PLAT_PATH_MAX];
static uint64_t prev_cnt[PERF_CNT_N], prev_xlated;
@@ -79,6 +81,7 @@ void flight_init(void) {
at_frames[n_at++] = f;
p = *end == ',' ? end + 1 : end;
}
every = env_u("NX2NAVI_FLIGHT_EVERY", 0);
const char *d = cfg_get("NX2NAVI_FLIGHT_DIR");
char tmp[PLAT_PATH_MAX];
if (d && *d) snprintf(tmp, sizeof tmp, "%s", d);
@@ -354,6 +357,7 @@ void flight_frame(uint64_t frame, uint64_t now_ns, uint64_t interval_ns, const u
key_was = key;
for (int i = 0; i < n_at; i++)
if (at_frames[i] == frame) trigger(frame, interval_ns, "NX2NAVI_FLIGHT_AT");
if (every && frame && frame % every == 0) trigger(frame, interval_ns, "NX2NAVI_FLIGHT_EVERY");
if (spike_ns && interval_ns > spike_ns && recorded > 30 && (!last_spike_ns || now_ns - last_spike_ns >= FLIGHT_SPIKE_GAP_NS)) {
if (!dump_at) last_spike_ns = now_ns;
trigger(frame, interval_ns, "spike");

View File

@@ -42,6 +42,7 @@ typedef struct { // NVNmappingRequest
_Static_assert(sizeof(nvn_mapping_t) == 0x28, "NVNmappingRequest stride");
const pool_t *nvn_pool_find(uint64_t addr, uint64_t len); // GPU-visible pool holding [addr, addr+len], NULL if none
uint64_t nvn_prog_made_ns(uint64_t serial); // NX2NAVI_PROG_LEAD: when the program was made, 0 unknown
uint64_t nvn_new_serial(void); // next tex_t / prog_t serial, from 1
uint64_t nvn_fence_last(void); // fence value of the newest submit (0 before the first)
int nvn_depth_minus_w(void); // 1 after DeviceSetDepthMode(NEAR_IS_MINUS_W), else 0

View File

@@ -443,6 +443,15 @@ NVN_API(nvnProgramFinalize) {
prog_release(o);
o->magic = 0;
}
// NX2NAVI_PROG_LEAD=1 (diagnostics): when each program was made, by serial (direct-mapped; a serial that lost its
// slot reads 0). vk_exec_draw.c prog_lead compares it with the program's first draw.
#define PROG_MADE_N 16384
static uint64_t prog_made_serial[PROG_MADE_N], prog_made_at[PROG_MADE_N];
uint64_t nvn_prog_made_ns(uint64_t serial) {
uint32_t i = (uint32_t)serial & (PROG_MADE_N - 1);
return __atomic_load_n(&prog_made_serial[i], __ATOMIC_RELAXED) == serial ? __atomic_load_n(&prog_made_at[i], __ATOMIC_RELAXED) : 0;
}
static int prog_set_shaders(prog_t *o, int n, const NVNshaderData *sd) {
if (n < 0 || n > 6 || (n && !sd)) {
rt_log("nvn: ProgramSetShaders with %d stages at %p", n, (const void *)sd);
@@ -452,6 +461,13 @@ static int prog_set_shaders(prog_t *o, int n, const NVNshaderData *sd) {
o->nstage = (uint32_t)n;
if (n) memcpy(o->sd, sd, (size_t)n * sizeof *sd);
o->serial = nvn_new_serial();
static int lead = -1;
if (lead < 0) lead = cfg_get("NX2NAVI_PROG_LEAD") != NULL && strcmp(cfg_get("NX2NAVI_PROG_LEAD"), "0") != 0;
if (lead) {
uint32_t i = (uint32_t)o->serial & (PROG_MADE_N - 1);
__atomic_store_n(&prog_made_at[i], plat_now_ns(), __ATOMIC_RELAXED);
__atomic_store_n(&prog_made_serial[i], o->serial, __ATOMIC_RELAXED);
}
NVN_LOG("ProgramSetShaders %p n=%d data0=%#llx", (void *)o, n, n ? (unsigned long long)sd[0].data : 0ull);
return 1;
}

View File

@@ -714,6 +714,9 @@ VkPipeline vk_pipeline_graphics(const VkPipeState *s, const VkPipeInfo **info);
// wait_ms > 0 waits up to that long for a pending pipeline first. Pipelines built in a run are recorded to
// pipeline-keys.bin next to the pipeline cache and prebuilt at the next boot (NX2NAVI_PIPE_PREBUILD=0: not).
VkPipeline vk_pipeline_graphics_async(const VkPipeState *s, const VkPipeInfo **info, int *pending, uint32_t wait_ms);
// The stage libraries of a draw the replay has not reached yet, queued to the workers when no pipeline is known for its
// state (vk_pipeline.c "stage libraries ahead of their draws"). Returns the libraries queued.
int vk_pipeline_hint(const VkPipeState *s);
// rt_boot calls it unless --shot is given; NX2NAVI_PIPE_SYNC=1 makes it a no-op. Of the probes only --probe-prewarm
// calls it.
void vk_pipeline_async_enable(void);

View File

@@ -1216,11 +1216,125 @@ static void evo_resume(VkExec *ex) {
}
}
// ---- looking ahead for pipelines (NX2NAVI_PIPE_AHEAD, default 1; 0 off)
// A draw whose pipeline is not there yet waits for its fast link or is skipped until a worker has built it, and the
// next draw with a new pipeline is only found when the replay reaches it: an area's first frame, or a turn that brings
// a dozen new materials into view, paid for them one after another (each wait a few milliseconds with the stage
// libraries in the driver's cache, 100 ms and more without), or showed frames with those draws missing while the
// workers that could have built them all at once sat idle.
// The submission is recorded before it is replayed, so what comes after the draw is known. The first draw of a
// submission that finds a shader or pipeline missing has the rest read here (vk_exec_ahead), from the op after it to
// the end of the submission's last command buffer: the scan applies the binds that decide a draw's shaders and its
// shader libraries (program, state objects, vertex attributes, textures and images, render targets, depth clamp) to a
// copy of the replay state and, at each draw, looks its shaders up as the draw will and names the state to
// vk_pipeline_hint, which queues the libraries not built yet. Nothing is recorded and no Vulkan object is touched;
// a shader not in memory is queued for its disk read or translation by the lookup itself. The draws then find their
// libraries built, or building on several workers at once.
// What the scan does not follow: the op stream past a wait for another queue's signal (what follows may be held), a
// change of the texture pool, mesh draws (their pipelines have no libraries), and whatever would take it past
// AHEAD_NS. Render targets are read from their descriptors (format, view format, sample count), which is what the
// replay resolves them to unless a target is left out; a guess that misses costs two libraries at most.
#define AHEAD_NS 3000000ull
#define AHEAD_OPS 400000u
static struct {
int on, live; // on: -1 until read; live: vk_exec_record walks handles[i] now
const uint64_t *handles;
int n, i;
const op_hdr *at[CALL_DEPTH + 1]; // the op being replayed at each call depth
int depth;
uint64_t seq; // the submission scanned last
uint64_t scans, ops, cut, ns, max_ns, logged_scans;
} AH = {.on = -1};
static void ahead_set_rt(VkExec *sx, const p_set_rt *r) {
sx->rt = *r;
int n = r->n < 0 ? 0 : r->n > 8 ? 8 : r->n, samples = 0;
for (int i = 0; i < 8; i++) {
const tex_t *t = i < n ? r->color[i] : NULL;
sx->pipe.color_fmt[i] = !t ? 0 : r->has_cview[i] && r->cview[i].set_format ? (int32_t)r->cview[i].format : t->d.format;
if (t && !samples) samples = t->d.samples > 1 ? t->d.samples : 1;
}
sx->pipe.depth_fmt = r->depth ? r->depth->d.format : 0;
if (r->depth && !samples) samples = r->depth->d.samples > 1 ? r->depth->d.samples : 1;
sx->pipe.samples = (uint8_t)(samples ? samples : 1);
}
void vk_exec_ahead(VkExec *ex) {
if (AH.on < 0) {
AH.on = cfg_bool("NX2NAVI_PIPE_AHEAD");
rt_log("vk: a draw that finds its pipeline missing has the rest of its submission read ahead for the stage "
"libraries the later draws need: %s (NX2NAVI_PIPE_AHEAD)", AH.on ? "on" : "off");
}
if (!AH.on || !AH.live || evo_in || AH.seq == vk_exec_submit_seq) return;
AH.seq = vk_exec_submit_seq;
uint64_t t0 = kern_now_ns();
uint32_t ops = 0;
static VkExec sx; // a copy of the replay state: too large for the stack
sx = *ex;
vk_exec_ahead_begin();
const op_hdr *ret[CALL_DEPTH + 1];
int depth = AH.depth, i = AH.i;
for (int d = 0; d < depth; d++) ret[d] = AH.at[d];
const op_hdr *h = nvn_op_next(AH.at[depth]);
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) break; // the replay reports it when it gets there
if (++ops >= AHEAD_OPS || (!(ops & 63) && kern_now_ns() - t0 > AHEAD_NS)) {
AH.cut++;
break;
}
const void *a = nvn_op_payload(h);
int stop = 0;
switch (h->op) {
case OP_END:
if (depth > 0) h = ret[--depth]; // back in the caller: on from the op after its call, below
else if (++i < AH.n) {
h = nvn_op_first(AH.handles[i]);
continue;
} else stop = 1;
break;
case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue;
case OP_CALL:
if (depth >= CALL_DEPTH) {
stop = 1;
break;
}
ret[depth++] = h;
h = nvn_op_first(((const p_jump *)a)->target);
continue;
case OP_WAIT_EVENT: stop = 1; break;
case OP_SET_TEX_POOL: stop = ((const p_set_pool *)a)->pool != sx.tex_pool; break;
case OP_SET_RT:
if (memcmp(&sx.rt, a, sizeof sx.rt)) ahead_set_rt(&sx, a);
break;
case OP_DEPTH_CLAMP: sx.pipe.depth_clamp = ((const p_flag *)a)->value != 0; break;
case OP_RASTER_DISCARD: sx.pipe.raster_discard = ((const p_flag *)a)->value != 0; break;
case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_TEX: case OP_BIND_IMG:
case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND:
vk_exec_ahead_op(&sx, h->op, a);
break;
default: break;
}
if (stop) break;
h = nvn_op_next(h);
}
uint64_t ns = kern_now_ns() - t0;
AH.scans++, AH.ops += ops, AH.ns += ns;
if (ns > AH.max_ns) AH.max_ns = ns;
if ((rt_trace & RT_TRACE_PERF) && AH.scans - AH.logged_scans >= 100) {
rt_log("perf: looking ahead: %llu scans so far, %.0f ops and %.2f ms each (longest %.2f ms, %llu ended at their "
"limit); %llu draws seen, %llu looked up", (unsigned long long)AH.scans, (double)AH.ops / (double)AH.scans,
(double)AH.ns / 1e6 / (double)AH.scans, (double)AH.max_ns / 1e6, (unsigned long long)AH.cut,
(unsigned long long)vk_exec_ahead_draws, (unsigned long long)vk_exec_ahead_lookups);
AH.logged_scans = AH.scans;
}
}
static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) {
for (;;) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_)
rt_fatal("vk: corrupt command record at %p: op %u size %u", (const void *)h, h->op, h->size);
const void *a = nvn_op_payload(h);
AH.at[depth] = h, AH.depth = depth; // where vk_exec_ahead reads on from
switch (h->op) {
case OP_END: return NULL;
case OP_BEGIN: break;
@@ -1405,7 +1519,9 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
if (n) cont_push(slot, cont_new(handles, n, 0, NULL)), evo_held++, vk_exec_evt('p', NULL, 0);
} else {
evo_queue = evo ? gfx_submit_queue : 0;
AH.handles = handles, AH.n = n, AH.live = 1;
for (int i = 0; i < n; i++) {
AH.i = i;
const op_hdr *stop = walk(ex, nvn_op_first(handles[i]), 0);
if (!stop) continue;
cont_push(slot, cont_new(handles, n, i, stop)); // the rest of this submission waits for its signal
@@ -1413,6 +1529,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
vk_exec_evt('h', NULL, 0);
break;
}
AH.live = 0;
evo_queue = 0;
}
vk_exec_pass_end(ex);

View File

@@ -162,6 +162,14 @@ void vk_exec_noatt_dims(const VkExec *ex, uint32_t *w, uint32_t *h); // pass s
// rendered for the first time) was rendered in the previous presented frame (per-frame targets); 0 otherwise, or when
// the pass has none.
int vk_exec_targets_recent(VkExec *ex, int any);
// Looking ahead (vk_exec.c): called by a draw that found its shader or pipeline missing, before it waits or is skipped.
// The rest of the submission being replayed is read once and the stage libraries of the pipelines its later draws need
// are queued to the workers (vk_pipeline_hint). vk_exec_ahead_op (vk_exec_draw.c) applies one bind or draw op of that
// scan to sx, a copy of the replay state; vk_exec_ahead_begin starts a scan there.
void vk_exec_ahead(VkExec *ex);
void vk_exec_ahead_begin(void);
void vk_exec_ahead_op(VkExec *sx, uint32_t op, const void *payload);
extern uint64_t vk_exec_ahead_draws, vk_exec_ahead_lookups; // draws the scans saw, and those whose shaders they looked up
// The current pass's targets with their formats, extents and the frames they were last rendered in (diagnostics).
void vk_exec_targets_describe(VkExec *ex, char *buf, size_t cap);

View File

@@ -3341,7 +3341,15 @@ static int pipe_skip_mode(void) {
// cold it runs out after a few draws and the rest are skipped as before, so a burst of cold pipelines costs a frame
// this much and no more. Measured on an RX 7900 XT, first walk through a village with no pipeline recorded: 2,349
// draws skipped without the budget, with the libraries warm in both runs.
#define LINK_DRAW_MS 3u
static uint32_t link_draw_ms(void) {
static int ms = -1;
if (ms < 0) {
long long v = cfg_int("NX2NAVI_PIPE_LINK_DRAW_MS");
ms = v < 1 ? 1 : v > 50 ? 50 : (int)v;
}
return (uint32_t)ms;
}
#define LINK_DRAW_MS link_draw_ms()
// How long a full-screen draw waits on for its pipeline once its frame budget is spent and the previous picture
// cannot be held any longer (prepare_draw_ex): the longest optimised build of such a pipeline measured was 194 ms.
#define FULLSCREEN_LAST_MS 500u
@@ -3351,7 +3359,8 @@ static uint64_t *link_budget(void) {
long long ms = cfg_int("NX2NAVI_PIPE_LINK_MS");
per_frame = (uint64_t)(ms < 0 ? 0 : ms > 100 ? 100 : ms) * 1000000ull;
rt_log("vk_exec: draws wait up to %llu ms a frame in all for the fast link of a missing pipeline "
"(NX2NAVI_PIPE_LINK_MS)", (unsigned long long)(per_frame / 1000000));
"(NX2NAVI_PIPE_LINK_MS), %u ms at most each (NX2NAVI_PIPE_LINK_DRAW_MS)",
(unsigned long long)(per_frame / 1000000), LINK_DRAW_MS);
}
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
if (f != frame) frame = f, left = per_frame;
@@ -3457,6 +3466,7 @@ uint32_t vk_exec_pending_draws_take(void) {
static int draw_skipped(const VkExec *ex, int mesh, int why) {
DRS.skipped[mesh][why]++;
if (why == SK_SHADER_PENDING || why == SK_PIPE_PENDING) pending_draws++;
if (why == SK_PIPE_PENDING) perf_count(PERF_CNT_PIPE_SKIP); // skipped draws only: a lookup that then waited is none
if (!(DRS.first[mesh] >> why & 1)) {
DRS.first[mesh] |= 1u << why;
rt_log("vk: first %s draw skipped because: %s (program %llu)", mesh ? "mesh" : "vertex", sk_names[why],
@@ -3489,6 +3499,40 @@ static int skip_prog_wanted(uint64_t serial) {
return 0;
}
// NX2NAVI_PROG_LEAD=1 (diagnostics): how long before its first draw the game made each program
// (nvnProgramSetShaders, nvn_api_obj.c nvn_prog_made_ns): the head start a build begun at that moment would have over
// one begun at the draw, which is where shaders are looked up and pipelines built today. A line for vertex, mesh and
// compute programs (their first dispatch) each, after every 200 programs first used.
#define PROG_LEAD_N 16384
static struct { int on; uint64_t seen[PROG_LEAD_N]; uint64_t n[3][6], logged; } PLD = {-1};
static int prog_lead_on(void) {
if (PLD.on < 0) PLD.on = cfg_get("NX2NAVI_PROG_LEAD") != NULL && strcmp(cfg_get("NX2NAVI_PROG_LEAD"), "0") != 0;
return PLD.on;
}
// mesh: 0 a vertex program's draw, 1 a mesh program's, 2 a compute program's dispatch.
static void prog_lead(const prog_t *p, int mesh) {
uint32_t i = (uint32_t)p->serial & (PROG_LEAD_N - 1);
if (PLD.seen[i] == p->serial) return;
PLD.seen[i] = p->serial;
uint64_t made = nvn_prog_made_ns(p->serial);
if (!made) return;
double ms = (double)(plat_now_ns() - made) / 1e6;
PLD.n[mesh][ms < 17 ? 0 : ms < 100 ? 1 : ms < 1000 ? 2 : ms < 5000 ? 3 : ms < 30000 ? 4 : 5]++;
uint64_t tot = 0;
for (int m = 0; m < 3; m++)
for (int k = 0; k < 6; k++) tot += PLD.n[m][k];
if (tot - PLD.logged < 200) return;
PLD.logged = tot;
static const char *const kind[3] = {"vertex", "mesh", "compute"};
for (int m = 0; m < 3; m++)
rt_log("vk: %s programs were made before their first use by: under 17 ms %llu, 17-100 ms %llu, 0.1-1 s %llu, 1-5 s "
"%llu, 5-30 s %llu, longer %llu (NX2NAVI_PROG_LEAD)", kind[m], (unsigned long long)PLD.n[m][0],
(unsigned long long)PLD.n[m][1], (unsigned long long)PLD.n[m][2], (unsigned long long)PLD.n[m][3],
(unsigned long long)PLD.n[m][4], (unsigned long long)PLD.n[m][5]);
}
// mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing
// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted.
static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) {
@@ -3506,6 +3550,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
VkRect2D sc = vk_exec_scissor(ex);
if (!sc.extent.width || !sc.extent.height) return draw_skipped(ex, mesh, SK_SCISSOR);
if (!ex->prog) return draw_skipped(ex, mesh, SK_NO_PROGRAM);
if (prog_lead_on()) prog_lead(ex->prog, mesh);
if (n_skip_prog && skip_prog_wanted(ex->prog->serial)) return draw_skipped(ex, mesh, SK_DISABLED); // -1 until read
int has_vs, has_fs, vs_wait = 0, fs_wait = 0, memo = mesh ? 0 : shader_memo_on();
int has_ts = 0, ts_wait = 0, vst = mesh ? NVN_STAGE_MESH : NVN_STAGE_VS;
@@ -3533,6 +3578,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
// session. A draw into per-frame targets is skipped until they are ready and comes back next frame.
int oneshot = -1, skip = pipe_skip_mode(); // -1: not looked up
if (vs_wait || fs_wait || ts_wait) {
vk_exec_ahead(ex); // the later draws' shaders and stage libraries are queued while this one waits or is skipped
if (!(oneshot = !vk_exec_targets_recent(ex, skip >= 1))) // skipped until ready
return perf_count(PERF_CNT_SHADER_SKIP), draw_skipped(ex, mesh, SK_SHADER_PENDING);
perf_count(PERF_CNT_ONESHOT_WAIT);
@@ -3595,15 +3641,15 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
info = PM.info, pending = 0;
p = __atomic_load_n(&info->pipeline, __ATOMIC_ACQUIRE);
} else {
if (linking) { // nearly every draw finds its pipeline: the clock is read only for one that did not
p = vk_pipeline_graphics_async(&s, &info, &pending, 0);
if (pending) {
t0 = kern_now_ns();
p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
}
} else {
w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0; // the clock is read only for draws that may wait
p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
// Nearly every draw finds its pipeline, so the first lookup waits for nothing and the clock is read only for a
// draw that missed. A miss first has the rest of the submission read ahead (vk_exec_ahead: the stage libraries
// of the later draws' pipelines build on the workers from here on), inside this draw's wait budget.
p = vk_pipeline_graphics_async(&s, &info, &pending, 0);
if (pending) {
if (linking) t0 = kern_now_ns();
else w0 = t0 ? t0 : wait_ms ? kern_now_ns() : 0;
vk_exec_ahead(ex);
if (wait_ms) p = vk_pipeline_graphics_async(&s, &info, &pending, wait_ms);
}
if (t0 && budget) {
uint64_t spent = kern_now_ns() - t0;
@@ -3641,9 +3687,10 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int
char tg[512];
wait_logs++;
vk_exec_targets_describe(ex, tg, sizeof tg);
rt_log("vk_exec: frame %llu: a draw waited %.1f ms for its pipeline (%s%s): %u vertices, pass %ux%u, %s",
rt_log("vk_exec: frame %llu: a draw waited %.1f ms for its pipeline (%s%s%s): %u vertices, pass %ux%u, %s",
(unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1, waited / 1e6, why,
pending ? ", skipped after all" : "", count, ex->pass_w, ex->pass_h, tg);
s.feedback ? ", feedback loop" : "", pending ? ", skipped after all" : "", count, ex->pass_w, ex->pass_h,
tg);
}
if (pending) return draw_skipped(ex, mesh, SK_PIPE_PENDING); // a background worker is building it: skipped until it is ready
if (!p || !info) return draw_skipped(ex, mesh, SK_PIPE_FAILED);
@@ -3692,6 +3739,89 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) {
return prepare_draw_ex(ex, prim, count, restart, 0);
}
// ---- looking ahead (vk_exec.c vk_exec_ahead): the binds and draws of ops the replay has not reached, applied to sx, a
// copy of the replay state. A draw looks its shaders up as prepare_draw_ex will (one not in memory is queued for its
// disk read or translation by that lookup) and names its state to vk_pipeline_hint. The last draw looked up is kept:
// one with the same program, texture binds and state repeats it, as most draws of a pass do.
uint64_t vk_exec_ahead_draws, vk_exec_ahead_lookups;
static struct { int valid; const prog_t *prog; uint64_t serial; uint32_t tex_serial; VkPipeState pipe; } AHL;
void vk_exec_ahead_begin(void) { AHL.valid = 0; }
static void ahead_draw(VkExec *sx) {
vk_exec_ahead_draws++;
if (!sx->prog) return;
if (AHL.valid && AHL.prog == sx->prog && AHL.serial == sx->prog->serial && AHL.tex_serial == sx->tex_serial &&
!memcmp(&AHL.pipe, &sx->pipe, sizeof AHL.pipe))
return;
AHL.valid = 1, AHL.prog = sx->prog, AHL.serial = sx->prog->serial, AHL.tex_serial = sx->tex_serial, AHL.pipe = sx->pipe;
vk_exec_ahead_lookups++;
int has_vs, has_fs, vs_wait = 0, fs_wait = 0;
const GfxShader *vs = shader_for(sx, NVN_STAGE_VS, &has_vs, &vs_wait, 0);
if (!has_vs) return; // a mesh program, or none: no libraries
const GfxShader *fs = shader_for(sx, NVN_STAGE_FS, &has_fs, &fs_wait, 0);
if (!vs || vs_wait || fs_wait || (has_fs && !fs) || shader_skipped(vs) || shader_skipped(fs)) return;
VkPipeState s = sx->pipe;
s.vs = vs, s.fs = fs, s.ts = NULL;
s.feedback = 0;
s.prim = NVN_DRAW_PRIMITIVE_TRIANGLES;
s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0);
s.y_flip = 1;
s.prim_restart = 0;
vk_pipeline_hint(&s);
}
void vk_exec_ahead_op(VkExec *sx, uint32_t op, const void *payload) {
switch (op) {
case OP_BIND_PROGRAM: {
const p_bind_program *b = payload;
sx->prog = b->prog, sx->prog_stages = b->stages;
break;
}
case OP_BIND_STATE: {
const p_bind_state *b = payload;
blend_st bs;
switch (b->kind) {
case 0:
memcpy(&bs, b->bytes, sizeof bs);
if (bs.target < 8) sx->pipe.blend[bs.target] = bs;
break;
case 1: memcpy(&sx->pipe.color, b->bytes, sizeof sx->pipe.color); break;
case 2: memcpy(&sx->pipe.chmask, b->bytes, sizeof sx->pipe.chmask); break;
case 3: memcpy(&sx->pipe.ms, b->bytes, sizeof sx->pipe.ms); break;
case 4: memcpy(&sx->pipe.poly, b->bytes, sizeof sx->pipe.poly); break;
case 5: memcpy(&sx->pipe.ds, b->bytes, sizeof sx->pipe.ds); break;
}
break;
}
case OP_BIND_VATTR: {
const p_bind_vattr *b = payload;
if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break;
memset(sx->pipe.vattr, 0, sizeof sx->pipe.vattr);
memcpy(&sx->pipe.vattr[b->first], b->st, sizeof(uint32_t) * (size_t)b->n);
break;
}
case OP_BIND_VSTREAM: {
const p_bind_vstream *b = payload;
if (b->first < 0 || b->n < 0 || b->first + b->n > 16) break;
memset(sx->pipe.vstream, 0, sizeof sx->pipe.vstream);
memcpy(&sx->pipe.vstream[b->first], b->st, sizeof(vs_st) * (size_t)b->n);
break;
}
case OP_BIND_TEX: case OP_BIND_IMG: {
const p_bind_handles *b = payload;
int cap = op == OP_BIND_TEX ? 128 : 32;
if (b->stage < 0 || b->stage >= 8 || b->first < 0 || b->n < 0 || b->n > 32 || b->first + b->n > cap) break;
memcpy(op == OP_BIND_TEX ? &sx->tex[b->stage][b->first] : &sx->img[b->stage][b->first], b->h,
sizeof(uint64_t) * (size_t)b->n);
sx->tex_serial++;
break;
}
case OP_DRAW: case OP_DRAW_IDX: case OP_DRAW_IDX_IND: ahead_draw(sx); break;
default: break;
}
}
// ---- per-object motion
// The camera pass (vk_dlaa.c mv_pass) gives everything the camera's motion, so a character that moves with the camera,
// or on its own, gets the wrong one and DLSS throws its history away there. Each solid scene draw whose position reads
@@ -5020,6 +5150,7 @@ static void untranslated_dispatch_fallback(VkExec *ex) {
static void op_dispatch(VkExec *ex, const p_dispatch *d) {
if (d->x <= 0 || d->y <= 0 || d->z <= 0) return;
if (!ex->prog) return warn_once(W_PROGRAM, "a dispatch without a bound program is skipped");
if (prog_lead_on()) prog_lead(ex->prog, 2);
vk_contact_flush(ex);
cb_res(ex);
dummies_init();

View File

@@ -76,7 +76,7 @@ static struct {
int indep; // stage-independent layouts (vk_caps.gpl): vertex set 0, fragment set 1
int push_stage; // GFX_STAGE_* whose set is pushed with independent layouts
int gpl; // draws whose pipeline is building get a fast-linked one (vk_caps.gpl)
struct { uint64_t links, wlinks, dlinks, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g;
struct { uint64_t links, fb_links, wlinks, flinks, dlinks, pr_added, lib_slow[4], ready_n[6], ready_nofast, hints, hint_libs, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double pr_added_ms, lib_slow_ms[4], ready_ms, ready_max, link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g;
int vs_outs_all; // NX2NAVI_VS_OUTS: 0 union of paired fragment inputs, 1 (all) all 32 output locations
int dyn_vi, dyn_clamp, dyn_poly, dyn_blend_en, dyn_blend_eq, dyn_wmask, dyn_a2c;
int f_logic_op, f_depth_clamp, f_fill_non_solid, f_dual_src, f_full_subgroups;
@@ -134,6 +134,7 @@ typedef struct Pipe {
uint8_t link_queued; // a waiting draw queued its fast link to the workers (LibJob p), not yet run
uint8_t used; // compute: a dispatch of this run asked for it (vk_pipeline_compute)
uint8_t pre; // compute: built and kept by the boot prebuild (lb_compute) or bg_main
uint64_t t_miss; // graphics: when a draw first missed it (ns), 0 once it is usable (ready_note)
size_t key_size;
uint8_t key[];
} Pipe;
@@ -2619,8 +2620,8 @@ static Lib *libs[LIB_BUCKETS];
typedef struct { uint32_t kind, topo_class, attr_fmt[32], attr_off[32]; uint8_t attr_bind[32], rate[17], pad[3]; } ViKey;
typedef struct { uint32_t kind, missing; const GfxShader *vs; uint8_t clamp, poly_mode, minus_w, pad; GfxShaderIo fin[32]; } PrKey;
typedef struct { uint32_t kind; uint8_t dsb, samples, a2c, pad; const GfxShader *fs; } FsKey;
typedef struct { uint32_t kind; uint8_t n_color, samples, a2c, logic_op, blend_en, pad[3]; uint32_t color[8], depth, stencil, wmask;
uint8_t blend[8][6]; } FoKey;
typedef struct { uint32_t kind; uint8_t n_color, samples, a2c, logic_op, blend_en, fbl, pad[2]; uint32_t color[8], depth, stencil, wmask;
uint8_t blend[8][6]; } FoKey; // fbl: GKey.fbl, the feedback loop flags of the library (gpl_feedback)
static uint64_t qpc(void) { return plat_now_ns(); }
static double ms_since(uint64_t t0) { return (double)(qpc() - t0) / 1e6; }
@@ -2732,6 +2733,11 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi
ri.colorAttachmentCount = k->n_color, ri.pColorAttachmentFormats = (const VkFormat *)k->color;
ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil;
pi.pColorBlendState = &cb, pi.pMultisampleState = &ms;
// Whether the pipeline may sample an attachment it renders to is part of the fragment output state: the flags
// go on this library alone, and a pipeline linked from it has them (gpl_feedback).
const FoKey *ok = key;
if (ok->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
if (ok->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
}
VkPipelineCreationFeedbackCreateInfo fci;
VkPipelineCreationFeedback fb;
@@ -2814,6 +2820,8 @@ static VkPipeline lib_build(Lib *b, int kind, const VkPipeState *s, const GKey *
plat_mutex_lock(&P.lock);
P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms;
if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms;
if (kind == LIB_PR && ((const PrKey *)(const void *)b->key)->missing) P.g.pr_added++, P.g.pr_added_ms += ms;
if (ms >= 25) P.g.lib_slow[kind]++, P.g.lib_slow_ms[kind] += ms;
}
if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind);
b->pipe = pipe, b->building = 0;
@@ -2897,19 +2905,48 @@ static void lib_keys(const VkPipeState *s, const GKey *k, ViKey *v, PrKey *pk, F
fk->kind = LIB_FS, fk->fs = s->fs, fk->dsb = k->dsb, fk->samples = k->samples, fk->a2c = k->a2c;
memset(ok, 0, sizeof *ok);
ok->kind = LIB_FO, ok->n_color = k->n_color, ok->samples = k->samples, ok->a2c = k->a2c, ok->logic_op = k->logic_op;
ok->blend_en = k->blend_en, ok->depth = k->depth, ok->stencil = k->stencil, ok->wmask = k->wmask;
ok->blend_en = k->blend_en, ok->fbl = k->fbl, ok->depth = k->depth, ok->stencil = k->stencil, ok->wmask = k->wmask;
memcpy(ok->color, k->color, sizeof ok->color), memcpy(ok->blend, k->blend, sizeof ok->blend);
}
// p, which a draw missed, has its first usable pipeline: how long that took is how long its draws were skipped or
// waited. fast 0: it had no fast link and this is the optimised build. Caller holds P.lock.
static void ready_note(Pipe *p, int fast) {
if (!p->t_miss) return;
double ms = (double)(plat_now_ns() - p->t_miss) / 1e6;
p->t_miss = 0;
P.g.ready_n[ms < 4 ? 0 : ms < 8 ? 1 : ms < 17 ? 2 : ms < 50 ? 3 : ms < 150 ? 4 : 5]++;
P.g.ready_ms += ms;
if (ms > P.g.ready_max) P.g.ready_max = ms;
P.g.ready_nofast += !fast;
}
// NX2NAVI_GPL_FEEDBACK (default 1; 0 = as before): a pipeline whose fragment shader samples an attachment of its own pass
// (VkPipeState.feedback: soft particles and water reading the depth buffer, decals, full-screen passes that read the
// target they write) gets a fast link like any other. The feedback loop flags belong to the fragment output state, so
// they are part of that library's key (FoKey.fbl) and of nothing else; the shader libraries are the ones the same
// shaders use without a loop. Before, such a pipeline had no fast link at all and its draws waited for, or were
// skipped during, the whole optimised build. On a first visit to Kokiri Forest (RTX 5090, every stage library in the
// driver's cache, DLSS Frame Generation on, so no frame is held back) full-screen draws of such pipelines waited
// 267 ms at the area's first frame and 410 ms in the middle of a walk; linked, 3 ms.
static int gpl_feedback(void) {
static int on = -1;
if (on < 0) {
on = cfg_bool("NX2NAVI_GPL_FEEDBACK");
if (!on) rt_log("vk_pipeline: gpl: pipelines with a feedback loop get no fast link (NX2NAVI_GPL_FEEDBACK=0)");
}
return on;
}
// A fast-linked pipeline for s from its stage libraries. Caller holds P.lock.
// build 0 (drawing thread): links only when the shader libraries exist (vertex input and fragment output libraries,
// which hold no shader, are built here); *final 0 = not yet, a worker builds them. build 1 (worker): builds the
// missing libraries and links with P.lock released. VK_NULL_HANDLE with *final 1 when GPL cannot express the pipeline
// (shared layout, feedback loop) or a build failed.
// (shared layout; a feedback loop with NX2NAVI_GPL_FEEDBACK=0) or a build failed.
static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc, int build, int *final) {
Layout *l = (Layout *)lc;
*final = 1;
if (!l->set1 || k->fbl) return VK_NULL_HANDLE;
if (!l->set1 || (k->fbl && !gpl_feedback())) return VK_NULL_HANDLE;
uint64_t t0 = qpc();
ViKey vk_;
PrKey pk;
@@ -2942,7 +2979,7 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc
P.g.link_ms += ms;
if (ms > P.g.link_max) P.g.link_max = ms;
}
if (pipe) P.g.links++;
if (pipe) P.g.links++, P.g.fb_links += k->fbl != 0;
else if (!P.g.link_fail++) rt_log("vk_pipeline: gpl: fast link failed; such draws wait or skip as without GPL");
double total = ms_since(t0);
if (!build && total > 5.0 && P.g.slow++ < 16)
@@ -3186,6 +3223,7 @@ static void u_run(LibJob *lj) {
VkPipeline f = gpl_link(&lj->s, &lj->k, lj->l, 1, &final); // releases P.lock while linking
if (f && !p->info.pipeline) {
p->fast = p->info.pipeline = f;
ready_note(p, 1);
job_lower(p->job); // its job now only does the optimised build
} else if (f) {
vkDestroyPipeline(P.device, f, NULL); // the job published one meanwhile
@@ -3201,17 +3239,74 @@ static void u_run(LibJob *lj) {
free(lj);
}
// A worker for waiting draws' jobs only (U): a draw that waits finds one free even while the pool builds pipelines,
// so the drawing thread never builds a library itself.
// NX2NAVI_PIPE_FAST_WORKERS (default 1; 0 = as before): the library workers also take the fast link of a draw-queue job
// (fast_take). A draw that is skipped rather than waiting leaves its pipeline to the pool, where a job is first
// fast-linked and then goes back in the queue for its optimised build. The pool keeps one worker out of optimised
// builds, so with the others inside one (about 100 ms each with the driver's cache warm, 350 ms cold) that single
// worker linked every new pipeline of a burst, one after another at 5 to 10 ms each. On a first visit to Kokiri
// Forest (RTX 5090, libraries in the driver's cache) 180 of the 444 pipelines draws missed took over 50 ms to become
// usable, the longest 1.3 s, and their draws were missing from every frame until then; with the library workers
// linking, 23 did, 19 of them mesh pipelines, which have no fast link.
static int fast_workers(void) {
static int on = -1;
if (on < 0) on = cfg_bool("NX2NAVI_PIPE_FAST_WORKERS");
return on;
}
// The first draw-queue job whose pipeline still lacks its fast link, taken out of the queue; NULL when there is none.
// Caller holds P.lock.
static Job *fast_take(void) {
if (!P.gpl || !fast_workers()) return NULL;
for (Job *j = A.head, *prev = NULL; j; prev = j, j = j->next) {
Pipe *p = j->p;
if (p->info.pipeline || p->fast_tried || p->link_queued || !p->lay || is_mesh(&j->s)) continue;
if (prev) prev->next = j->next;
else A.head = j->next;
if (A.tail == j) A.tail = prev;
j->next = j->prev = NULL;
return j;
}
return NULL;
}
// A worker for stage libraries and fast links only: a waiting draw's jobs (U) first, so a draw that waits finds one
// free even while the pool builds pipelines and the drawing thread never builds a library itself; then the fast link
// of a draw-queue job (fast_take), whose optimised build goes on to the pool.
static void lib_worker(void *arg) {
(void)arg;
plat_mutex_lock(&P.lock);
for (;;) {
while (!U.head) plat_cond_wait(&lib_cv, &P.lock);
LibJob *lj = U.head;
if (!(U.head = lj->next)) U.tail = NULL;
if (lj->b) lj->b->job = NULL;
u_run(lj);
Job *j = NULL;
while (!U.head && !(j = fast_take())) plat_cond_wait(&lib_cv, &P.lock);
if (!j) {
LibJob *lj = U.head;
if (!(U.head = lj->next)) U.tail = NULL;
if (lj->b) lj->b->job = NULL;
u_run(lj);
continue;
}
Pipe *p = j->p;
int final;
p->fast_tried = 1; // no other thread links it meanwhile; a draw that waits for it waits for this link
VkPipeline f = gpl_link(&j->s, &j->k, p->lay, 1, &final); // releases P.lock while building
if (f && !p->info.pipeline) {
p->fast = p->info.pipeline = f, P.g.wlinks++, P.g.flinks++;
ready_note(p, 1);
plat_cond_broadcast(&done_cv);
} else if (f) {
vkDestroyPipeline(P.device, f, NULL);
}
if (p->info.pipeline) { // the optimised build: ahead of the prebuild, behind other draws' misses
j->low = 1, P.g.requeued++;
j->next = A.low;
if (A.low) A.low->prev = j;
else A.low_tail = j;
A.low = j;
} else { // no fast link for this one: its whole build, next in the draw queue
if (!(j->next = A.head)) A.tail = j;
A.head = j;
}
plat_cond_signal(&work_cv);
}
}
@@ -3259,6 +3354,7 @@ static void pipe_worker(void *arg) {
j->p->fast_tried = 1;
if (f && !j->p->info.pipeline) {
j->p->fast = j->p->info.pipeline = f, P.g.wlinks++;
ready_note(j->p, 1);
plat_cond_broadcast(&done_cv);
j->low = 1, j->worker = -1, P.g.requeued++;
j->prev = NULL, j->next = A.low;
@@ -3275,6 +3371,7 @@ static void pipe_worker(void *arg) {
VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off);
plat_mutex_lock(&P.lock);
if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one
ready_note(j->p, 0);
if (pipe && j->p->fast) P.g.swaps++;
j->p->building = 0;
j->p->job = NULL;
@@ -3300,6 +3397,8 @@ static void pipe_worker(void *arg) {
return;
}
static int hint_on(void);
// Starts the pool on first use: a quarter of the logical processors, 2 to 4. Caller holds P.lock.
static int workers_start(void) {
if (A.threads) return 1;
@@ -3310,8 +3409,12 @@ static int workers_start(void) {
if (!t) break;
A.thread[A.threads++] = t; // kept: promote() raises a worker running a prebuild job
}
// two for waiting draws' libraries and links (the vertex and fragment libraries of one draw in parallel)
for (int i = 0; A.threads && U.on && P.gpl && i < 2; i++)
// Workers for stage libraries and fast links alone: two (the vertex and fragment libraries of one waiting draw in
// parallel), and as many as the pool has with the look-ahead, which queues the libraries of a whole submission's
// new pipelines at once (vk_pipeline_hint), or with the fast links of skipped draws (lib_worker). They sleep
// unless a library or a fast link is queued.
int nlib = hint_on() || fast_workers() ? n : 2;
for (int i = 0; A.threads && U.on && P.gpl && i < nlib; i++)
if (plat_thread_new(lib_worker, NULL, "pipe lib worker")) U.lib_workers++;
if (A.threads)
rt_log("vk_pipeline: %d background graphics pipeline workers, %d for waiting draws' libraries", A.threads,
@@ -3355,6 +3458,7 @@ int vk_pipeline_async_on(void) { return A.on; }
static void promote(Pipe *p) {
Job *j = p->job;
if (!j || !j->low) return;
if (j->prebuild && !j->demanded && !p->info.pipeline) p->t_miss = plat_now_ns();
j->low = 0, j->demanded = 1;
if (j->worker < 0) {
if (j->prev) j->prev->next = j->next;
@@ -3366,6 +3470,7 @@ static void promote(Pipe *p) {
else A.head = j;
A.tail = j;
plat_cond_signal(&work_cv);
if (U.lib_workers) plat_cond_signal(&lib_cv); // its fast link (lib_worker)
} else if (A.prio_low[j->worker]) {
A.prio_low[j->worker] = 0;
plat_thread_set_priority(A.thread[j->worker], PLAT_PRIO_NORMAL);
@@ -3425,6 +3530,7 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode
}
j->p = p;
p->building = 1, p->job = j;
if (mode != 2) p->t_miss = plat_now_ns();
if (mode == 2) {
j->prev = A.low_tail;
if (A.low_tail) A.low_tail->next = j;
@@ -3435,6 +3541,7 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode
if (A.tail) A.tail->next = j;
else A.head = j;
A.tail = j;
if (U.lib_workers && fast_workers()) plat_cond_signal(&lib_cv); // its fast link (lib_worker)
}
A.inflight++;
plat_cond_signal(&work_cv);
@@ -3536,6 +3643,42 @@ static void lib_wait(const VkPipeState *s, const GKey *k, Layout *l, double end)
}
}
// ---- stage libraries ahead of their draws (NX2NAVI_PIPE_AHEAD, default 1; 0 off; vk_exec.c "looking ahead")
// The replay names the state of a draw it has not reached yet. When no pipeline is known for it, its pre-rasterization
// and fragment shader libraries are queued to the library workers now, so they build while the draws before it are
// recorded, in parallel with each other and with those of the other draws named: the draw's own lookup then finds both
// and links on its thread in microseconds instead of waiting for them or being skipped while they build. Nothing else
// is entered, no pipeline, no optimised build and no key record: a state the scan guessed wrong costs two libraries
// at most. The feedback loop bits, the topology and the targets do not matter here, as no shader library depends on
// them. Returns the libraries queued.
static int hint_on(void) {
static int on = -1;
if (on < 0) on = cfg_bool("NX2NAVI_PIPE_AHEAD");
return on;
}
int vk_pipeline_hint(const VkPipeState *s) {
if (!A.on || !U.on || !hint_on() || !s->vs || !ensure_init() || !P.gpl || !P.indep || is_mesh(s)) return 0;
if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) return 0;
GKey k;
gkey_fill(s, &k);
if (k.topo_class == 3) return 0;
uint64_t h = key_hash(&k, sizeof k);
int queued = 0;
plat_mutex_lock(&P.lock);
if (!pipe_find(&k, sizeof k, h)) {
Bindings B = {0}, F = {0};
Layout *l = add_stage(&B, s->vs, 0) && (!s->fs || add_stage(&F, s->fs, 0)) ? layout_pair(&B, &F) : NULL;
Lib *b[2];
uint64_t q0 = U.queued, w0 = U.waits;
if (l && l->set1 && workers_start() && lib_queue(s, &k, l, b)) queued = (int)(U.queued - q0);
U.waits = w0, U.queued = q0; // those count waiting draws' jobs
P.g.hints++, P.g.hint_libs += (uint64_t)queued;
}
plat_mutex_unlock(&P.lock);
return queued;
}
static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int async, int *pending, uint32_t wait_ms) {
if (info) *info = NULL;
if (pending) *pending = 0;
@@ -3594,6 +3737,7 @@ static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int as
if (final) p->fast_tried = 1;
if (f && !p->info.pipeline) {
p->fast = p->info.pipeline = f;
ready_note(p, 1);
job_lower(p->job);
plat_cond_broadcast(&done_cv);
} else if (f) {
@@ -3614,7 +3758,6 @@ static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int as
VkPipeline pipe = p ? p->info.pipeline : VK_NULL_HANDLE;
plat_mutex_unlock(&P.lock);
if (wait) {
perf_count(PERF_CNT_PIPE_SKIP);
if (pending) *pending = 1;
return VK_NULL_HANDLE;
}
@@ -3650,6 +3793,7 @@ static void warm_builder(void *arg) {
VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off);
plat_mutex_lock(&P.lock);
if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one
ready_note(j->p, 0);
if (pipe && j->p->fast) P.g.swaps++;
j->p->building = 0;
j->p->job = NULL;
@@ -5749,14 +5893,27 @@ static int cache_save(uint32_t wait_ms) {
static void save_with(uint32_t wait_ms) {
if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log();
if (P.ready && P.gpl) {
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws "
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; "
"%llu of pipelines with a feedback loop; avg %.2f ms, "
"max %.2f ms, %llu failed), %llu draws "
"lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued "
"low after their fast link), %llu drawing-thread links over 5 ms; %llu waiting draws built their libraries "
"(avg %.2f ms, max %.2f ms)",
(unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, P.g.links ? P.g.link_ms / P.g.links : 0.0,
(unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, (unsigned long long)P.g.flinks,
(unsigned long long)P.g.fb_links,
P.g.links ? P.g.link_ms / P.g.links : 0.0,
P.g.link_max, (unsigned long long)P.g.link_fail, (unsigned long long)P.g.defer,
(unsigned long long)P.g.swaps, (unsigned long long)P.g.requeued, (unsigned long long)P.g.slow,
(unsigned long long)P.g.dlinks, P.g.dlinks ? P.g.dlink_ms / P.g.dlinks : 0.0, P.g.dlink_max);
uint64_t rn = P.g.ready_n[0] + P.g.ready_n[1] + P.g.ready_n[2] + P.g.ready_n[3] + P.g.ready_n[4] + P.g.ready_n[5];
rt_log("vk_pipeline: gpl: pipelines a draw missed were usable after under 4 ms %llu, 4-8 ms %llu, 8-17 ms %llu, "
"17-50 ms %llu, 50-150 ms %llu, longer %llu (avg %.1f ms, longest %.0f ms); %llu of them had no fast link",
(unsigned long long)P.g.ready_n[0], (unsigned long long)P.g.ready_n[1], (unsigned long long)P.g.ready_n[2],
(unsigned long long)P.g.ready_n[3], (unsigned long long)P.g.ready_n[4], (unsigned long long)P.g.ready_n[5],
rn ? P.g.ready_ms / (double)rn : 0.0, P.g.ready_max, (unsigned long long)P.g.ready_nofast);
if (P.g.hints)
rt_log("vk_pipeline: gpl: looking ahead named %llu pipeline states not known yet and queued %llu stage libraries "
"for them", (unsigned long long)P.g.hints, (unsigned long long)P.g.hint_libs);
if (U.waits || U.queued)
rt_log("vk_pipeline: gpl: library jobs for waiting draws: %llu queued, %llu built by a worker, %llu by the "
"thread that needed them (no worker free), %llu waits ended at their limit; %llu fast links on the %d "
@@ -5765,8 +5922,13 @@ static void save_with(uint32_t wait_ms) {
(unsigned long long)U.timeouts, (unsigned long long)U.links, U.lib_workers);
static const char *const kn[4] = {"vertex input", "pre-rasterization", "fragment shader", "fragment output"};
for (int i = 0; i < 4; i++)
rt_log("vk_pipeline: gpl: %s libraries %llu (avg %.2f ms, max %.2f ms)", kn[i], (unsigned long long)P.g.lib_n[i],
P.g.lib_n[i] ? P.g.lib_ms[i] / P.g.lib_n[i] : 0.0, P.g.lib_max[i]);
rt_log("vk_pipeline: gpl: %s libraries %llu (avg %.2f ms, max %.2f ms; %llu of 25 ms or more, %.0f ms in all)",
kn[i], (unsigned long long)P.g.lib_n[i], P.g.lib_n[i] ? P.g.lib_ms[i] / P.g.lib_n[i] : 0.0, P.g.lib_max[i],
(unsigned long long)P.g.lib_slow[i], P.g.lib_slow_ms[i]);
if (P.g.pr_added)
rt_log("vk_pipeline: gpl: %llu of the pre-rasterization libraries had outputs added for their fragment shaders "
"(avg %.1f ms): another module than the one the preparation built, so the driver's cache has none of them",
(unsigned long long)P.g.pr_added, P.g.pr_added_ms / (double)P.g.pr_added);
if (LB.n)
rt_log("vk_pipeline: gpl: prebuilt libraries a draw used: pre-rasterization %llu, fragment shader %llu; "
"compute pipelines built on the replay thread %llu", (unsigned long long)P.g.pre_used[LIB_PR],