mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 07:09:57 -04:00
Leave the boot prebuild's huge compute shaders to their first dispatch, so memory no longer climbs to 30 GB at the title screen.
Four play-recorded compute shaders of 0.87 to 4.4 million SPIR-V words (the game's tensor-core upscaler passes, which run through the stand-in here) are recorded and so prebuilt at boot. The driver spent minutes and gigabytes on each, four at once, none finished, and every start began again until RAM ran out and the screen went black. The boot prebuild now creates a compute shader over 512K words only from a pipeline cache (VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT) and leaves it to its first dispatch otherwise. NX2NAVI_LIB_PREBUILD_BIG=<words> sets the limit, 0 the old behaviour. From a tester's fixes: memory flat at 13 GB on the title screen, prebuild done in 463 ms. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -45,7 +45,7 @@ static const char *const known[] = {
|
||||
"FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FREEZE_IDLE", "FREE_CORES",
|
||||
"FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE",
|
||||
"HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
|
||||
"LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
|
||||
"LIB_PREBUILD_BIG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
|
||||
"MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",
|
||||
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
|
||||
"MV_SURVEY",
|
||||
|
||||
@@ -76,6 +76,7 @@ static struct {
|
||||
int vs_outs_all; // NX2NAVI_VS_OUTS: 0 union of paired fragment inputs, 1 (all) all 32 output locations
|
||||
int dyn_vi, dyn_clamp, dyn_poly, dyn_blend_en, dyn_blend_eq, dyn_wmask, dyn_a2c;
|
||||
int f_logic_op, f_depth_clamp, f_fill_non_solid, f_dual_src, f_full_subgroups;
|
||||
int cache_control; // pipelineCreationCacheControl: a create can ask for the pipeline caches only
|
||||
int pv_last; // provokingVertexLast enabled: NVN's default is the last vertex
|
||||
VkShaderStageFlags sg32_stages; // stages that accept requiredSubgroupSize = 32 (0: not available)
|
||||
VkDynamicState dyn[32];
|
||||
@@ -237,6 +238,7 @@ static int ensure_init(void) {
|
||||
p13.maxSubgroupSize, (unsigned)p13.requiredSubgroupSizeStages, vk_caps.sg_size, (unsigned)P.sg32_stages);
|
||||
}
|
||||
P.f_full_subgroups = v13.computeFullSubgroups;
|
||||
P.cache_control = v13.pipelineCreationCacheControl;
|
||||
P.f_logic_op = f2.features.logicOp;
|
||||
P.f_depth_clamp = f2.features.depthClamp;
|
||||
P.f_fill_non_solid = f2.features.fillModeNonSolid;
|
||||
@@ -2933,7 +2935,9 @@ static uint32_t *spv_override(const uint32_t *code, uint64_t words, uint64_t *ou
|
||||
}
|
||||
|
||||
// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised).
|
||||
static VkPipeline compute_create(const GfxShader *cs, const Layout *l) {
|
||||
// cached_only (needs P.cache_control): from P.cache or the driver's own disk cache, never compiled; VK_NULL_HANDLE when
|
||||
// neither holds it (the driver returns VK_PIPELINE_COMPILE_REQUIRED, a success code, with no pipeline).
|
||||
static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cached_only) {
|
||||
uint64_t words;
|
||||
const uint32_t *code = gfx_shader_spirv(cs, &words);
|
||||
uint64_t ow = 0;
|
||||
@@ -2946,6 +2950,7 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l) {
|
||||
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
||||
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
||||
.layout = l->layout, .flags = noopt_flag()};
|
||||
if (cached_only) ci.flags |= VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
|
||||
pin_subgroup32(&ci.stage, &rqc, code, words);
|
||||
// Subgroup operations in a workgroup whose width is a whole number of subgroups need every subgroup full: the
|
||||
// driver may otherwise run partial ones, and a ballot over the culling's candidates then counted differently from
|
||||
@@ -2990,8 +2995,16 @@ static Layout *compute_layout(const GfxShader *cs, Bindings *B, const char **why
|
||||
// file: clamp 0, fill, -w..w), single-sampled, no dual source; the state dynamic on this GPU is 0 in every key.
|
||||
// A draw that meets such a variant in a new pipeline then gets its libraries from libs[] or the driver cache and
|
||||
// fast-links (gpl_link). NX2NAVI_LIB_PREBUILD=0 turns the boot part off, =N stops it after N variants;
|
||||
// NX2NAVI_LIB_PREBUILD_THREADS sets its thread count.
|
||||
// NX2NAVI_LIB_PREBUILD_THREADS sets its thread count, NX2NAVI_LIB_PREBUILD_BIG its compute size limit (LB_BIG_WORDS).
|
||||
#define LB_MAX_THREADS 16
|
||||
// The boot part builds a compute shader over LB_BIG_WORDS SPIR-V words only from the pipeline caches (compute_create
|
||||
// cached_only); one that no cache holds is left to its first dispatch, like any shader outside the warm list. Play
|
||||
// recorded four straight-line compute shaders of 0.87-4.4 million words (one function of up to 888K instructions; the
|
||||
// median of 3,648 compute shaders is 7,351 words, p99 64,844). NVIDIA 621.01 on an RTX 4090 compiled them on four
|
||||
// prebuild threads for minutes at 100% CPU each while the process grew from 12 GB to 28-31 GB, until RAM ran out and the
|
||||
// game stuttered and went black a few minutes after boot. None finished, so no cache held one and every boot did it
|
||||
// again. NX2NAVI_LIB_PREBUILD_BIG sets the limit in words; 0 builds every size.
|
||||
#define LB_BIG_WORDS (512u * 1024)
|
||||
|
||||
static struct {
|
||||
GfxWarmEnt *e;
|
||||
@@ -3005,6 +3018,7 @@ static struct {
|
||||
// warmed: driver cache only; nolib: vertex and fragment variants skipped with the graphics pipeline library off
|
||||
// skipped: variants of the skip list (SK below)
|
||||
uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, skipped, done, cs_draw;
|
||||
uint64_t big_words, deferred; // LB_BIG_WORDS; compute shaders over it left to their dispatch
|
||||
double warm_ms[3];
|
||||
volatile int32_t kept; // libraries entered in libs[] so far
|
||||
int32_t keep_n;
|
||||
@@ -3187,7 +3201,7 @@ VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) {
|
||||
int flight = cs_flight_add(cs);
|
||||
plat_mutex_unlock(&P.lock); // the compile runs without the lock
|
||||
uint64_t p0 = perf_begin();
|
||||
pipe = compute_create(cs, l);
|
||||
pipe = compute_create(cs, l, 0);
|
||||
perf_end(PERF_PIPE, p0);
|
||||
plat_mutex_lock(&P.lock);
|
||||
if (flight) cs_flight_remove(cs);
|
||||
@@ -3247,11 +3261,30 @@ static uint32_t vs_union_peek(const GfxShader *vs, GfxShaderIo fin[32]) {
|
||||
return missing;
|
||||
}
|
||||
|
||||
static void lb_compute(const GfxShader *cs) {
|
||||
// A compute shader over LB.big_words that no pipeline cache holds: its first dispatch builds it (vk_pipeline_compute).
|
||||
static void lb_defer(const uint8_t iid[32], uint64_t words) {
|
||||
plat_mutex_lock(&LB.stat);
|
||||
LB.deferred++;
|
||||
plat_mutex_unlock(&LB.stat);
|
||||
rt_log("vk_pipeline: stage library prebuild: compute shader %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) %s: "
|
||||
"left to its first dispatch (NX2NAVI_LIB_PREBUILD_BIG %llu)", iid[0], iid[1], iid[2], iid[3], iid[4], iid[5],
|
||||
iid[6], iid[7], (unsigned long long)words,
|
||||
P.cache_control ? "not in the pipeline caches" : "too large to build without a cache probe (no pipelineCreationCacheControl)",
|
||||
(unsigned long long)LB.big_words);
|
||||
}
|
||||
|
||||
static void lb_compute(const GfxShader *cs, const uint8_t iid[32]) {
|
||||
struct { const GfxShader *cs; uint32_t compute; } k = {cs, 1};
|
||||
uint64_t h = key_hash(&k, sizeof k);
|
||||
Bindings B = {0};
|
||||
const char *why = NULL;
|
||||
uint64_t words = 0;
|
||||
gfx_shader_spirv(cs, &words);
|
||||
int big = LB.big_words && words > LB.big_words; // only from the caches (LB_BIG_WORDS)
|
||||
if (big && !P.cache_control) {
|
||||
lb_defer(iid, words);
|
||||
return;
|
||||
}
|
||||
lb_lock();
|
||||
int busy = cs_flight_has(cs); // the replay compiles it now (vk_pipeline_compute)
|
||||
Layout *l = busy || pipe_find(&k, sizeof k, h) ? NULL : compute_layout(cs, &B, &why);
|
||||
@@ -3265,7 +3298,7 @@ static void lb_compute(const GfxShader *cs) {
|
||||
// at normal priority: while it is in flight a dispatch of the same shader waits for it (vk_pipeline_compute), and an
|
||||
// idle-priority compile under load held that frame for seconds
|
||||
if (LB.idle) plat_thread_priority(PLAT_PRIO_NORMAL);
|
||||
VkPipeline pipe = compute_create(cs, l);
|
||||
VkPipeline pipe = compute_create(cs, l, big);
|
||||
double ms = now_ms() - t0;
|
||||
lb_lock();
|
||||
if (flight) cs_flight_remove(cs);
|
||||
@@ -3277,7 +3310,8 @@ static void lb_compute(const GfxShader *cs) {
|
||||
have = 1;
|
||||
}
|
||||
lb_unlock();
|
||||
lb_count(2, pipe ? have : 2, ms);
|
||||
if (!pipe && big) lb_defer(iid, words);
|
||||
else lb_count(2, pipe ? have : 2, ms);
|
||||
}
|
||||
|
||||
// The pre-rasterization (vertex) or fragment-shader library of one shader with the LB state.
|
||||
@@ -3355,7 +3389,8 @@ static void lb_report(const char *how) {
|
||||
"%llu there; fragment %llu built (avg %.1f ms), %llu there; compute %llu built (avg %.1f ms), %llu there; driver "
|
||||
"cache only: vertex %llu (avg %.1f ms), fragment %llu (avg %.1f ms), compute %llu (avg %.1f ms); %llu failed, %llu not in the shader cache, "
|
||||
"%llu vertex/fragment skipped (graphics pipeline library off), %llu skipped (the driver crashed on them before, "
|
||||
"shader-prepare-skip.txt); process working set %.0f MB", how, (unsigned long long)LB.done, LB.n,
|
||||
"shader-prepare-skip.txt), %llu large compute shaders left to their first dispatch (NX2NAVI_LIB_PREBUILD_BIG); "
|
||||
"process working set %.0f MB", how, (unsigned long long)LB.done, LB.n,
|
||||
now_ms() - LB.t0, (unsigned long long)LB.built[0], LB.built[0] ? LB.ms[0] / LB.built[0] : 0.0,
|
||||
(unsigned long long)LB.have[0], (unsigned long long)LB.built[1], LB.built[1] ? LB.ms[1] / LB.built[1] : 0.0,
|
||||
(unsigned long long)LB.have[1], (unsigned long long)LB.built[2], LB.built[2] ? LB.ms[2] / LB.built[2] : 0.0,
|
||||
@@ -3363,7 +3398,8 @@ static void lb_report(const char *how) {
|
||||
(unsigned long long)LB.warmed[1], LB.warmed[1] ? LB.warm_ms[1] / LB.warmed[1] : 0.0,
|
||||
(unsigned long long)LB.warmed[2], LB.warmed[2] ? LB.warm_ms[2] / LB.warmed[2] : 0.0,
|
||||
(unsigned long long)(LB.fail[0] + LB.fail[1] + LB.fail[2]),
|
||||
(unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (unsigned long long)LB.skipped, (double)mem_proc);
|
||||
(unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (unsigned long long)LB.skipped,
|
||||
(unsigned long long)LB.deferred, (double)mem_proc);
|
||||
plat_mutex_unlock_shared(&LB.stat);
|
||||
}
|
||||
|
||||
@@ -3389,7 +3425,7 @@ static void lb_worker(void *arg) {
|
||||
LB.noshader++;
|
||||
plat_mutex_unlock(&LB.stat);
|
||||
} else if (gfx_shader_stage(sh) == GFX_STAGE_CS) {
|
||||
lb_compute(sh);
|
||||
lb_compute(sh, LB.e[i].iid);
|
||||
} else if (gfx_shader_stage(sh) == GFX_STAGE_VS || gfx_shader_stage(sh) == GFX_STAGE_FS) {
|
||||
if (P.gpl) lb_library(sh, gfx_shader_stage(sh) == GFX_STAGE_VS, 1);
|
||||
else lb_skip_lib();
|
||||
@@ -3417,7 +3453,9 @@ static void lb_state(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) {
|
||||
static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) {
|
||||
const char *e = cfg_value("NX2NAVI_LIB_PREBUILD"), *ke = cfg_get("NX2NAVI_LIB_PREBUILD_KEEP");
|
||||
const char *te = cfg_get("NX2NAVI_LIB_PREBUILD_THREADS"); // no fixed default: half the CPUs when unset
|
||||
const char *be = cfg_get("NX2NAVI_LIB_PREBUILD_BIG"); // SPIR-V words; LB_BIG_WORDS when unset, 0 no limit
|
||||
if (!A.on || !cfg_bool("NX2NAVI_LIB_PREBUILD")) return;
|
||||
LB.big_words = be && *be ? strtoull(be, NULL, 10) : LB_BIG_WORDS;
|
||||
char dir[PLAT_PATH_MAX * 2];
|
||||
if (cache_dir(dir, sizeof dir)) skip_init(dir);
|
||||
GfxWarmEnt *all = NULL;
|
||||
@@ -3449,10 +3487,14 @@ static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) {
|
||||
if ((LB.th[started] = plat_thread_new(lb_worker, NULL, "lib prebuild"))) started++;
|
||||
plat_add32(&LB.running, started - t);
|
||||
LB.threads = started;
|
||||
char big[128];
|
||||
if (!LB.big_words) snprintf(big, sizeof big, "compute shaders of every size");
|
||||
else snprintf(big, sizeof big, "compute shaders over %llu SPIR-V words %s", (unsigned long long)LB.big_words,
|
||||
P.cache_control ? "only from the pipeline caches" : "left to their first dispatch");
|
||||
rt_log("vk_pipeline: stage library prebuild: %zu compute pipelines and %zu recorded variants' libraries "
|
||||
"(NX2NAVI_LIB_PREBUILD_KEEP %d) on %d idle-priority threads; library state clamp %u, polygon mode %u, depth "
|
||||
"-w..w %u, samples %u", n_cs < n ? n_cs : n, n > n_cs ? n - n_cs : 0, LB.keep_n, started, LB.clamp, LB.poly,
|
||||
LB.minus_w, LB.samples);
|
||||
"-w..w %u, samples %u; %s (NX2NAVI_LIB_PREBUILD_BIG)", n_cs < n ? n_cs : n, n > n_cs ? n - n_cs : 0, LB.keep_n,
|
||||
started, LB.clamp, LB.poly, LB.minus_w, LB.samples, big);
|
||||
}
|
||||
|
||||
static void lb_stop(void) {
|
||||
@@ -3486,7 +3528,7 @@ static void lb_warm_worker(void *arg) {
|
||||
Layout *l = compute_layout(sh, &B, &why);
|
||||
lb_unlock();
|
||||
double t0 = now_ms();
|
||||
VkPipeline pipe = l ? compute_create(sh, l) : VK_NULL_HANDLE;
|
||||
VkPipeline pipe = l ? compute_create(sh, l, 0) : VK_NULL_HANDLE;
|
||||
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
||||
lb_count(2, pipe ? 3 : 2, now_ms() - t0);
|
||||
} else if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) {
|
||||
|
||||
Reference in New Issue
Block a user