diff --git a/docs/README-BuildKit.md b/docs/README-BuildKit.md index 04cb8871..ed3affc6 100644 --- a/docs/README-BuildKit.md +++ b/docs/README-BuildKit.md @@ -144,6 +144,10 @@ beside the game. "Preparing shaders" once while the driver rebuilds them. With a driver that has Vulkan's graphics pipeline library it lasts at most about 20 seconds once the shaders are translated, and the remaining pipelines are built in the background while you play. +- On an NVIDIA card, set **Shader Cache Size** to **Unlimited** in the NVIDIA app (Graphics, Global Settings). The + driver keeps the prepared shaders' libraries in that cache, and a smaller limit drops the oldest as new ones come in + (a 2048 MB cache could not hold them): every start then builds them again in the background, and play stutters, with + some things missing for a few frames, until that is done. The log's `library warm` lines say when this happens. - **Screen** in the menu has Texture filtering (anisotropic, up to 16x, at once), Upscaling (DLSS on NVIDIA RTX cards, FSR 3.1 on any card where `amd_fidelityfx_vk.dll` loads), Frame generation (NVIDIA DLSS on RTX 40 and 50 cards, with a DLSS multiplier of 2x, or 3x and 4x on RTX 50 cards; FSR 3.1 on any card where it loads; it needs DLSS or FSR 3.1 diff --git a/docs/building.md b/docs/building.md index 516edbe0..ce227ab4 100644 --- a/docs/building.md +++ b/docs/building.md @@ -315,6 +315,10 @@ A change to the button prompts (`tools/make_romfs_patches.py --prompts`, `tools/ `.venv-py314\Scripts\python tests\romfs_prompts_test.py`, which runs the tool on a RomFS it makes itself (no game files); a change to the settings passes `nx2navi.exe --probe-cfg`. +A change to what the library warm logs when the driver's cache of stage libraries is cold (`runtime/vk_warm_hint.h`, +`runtime/vk_pipeline.c` `lw_main`) passes `python tests\warm_hint_check.py`, which builds the header alone and checks +the added text by GPU vendor and by why the cache was cold. No game files. + The lifter (`recomp/`) and the CPU headers (`runtime/cpu.h`, `runtime/simd.h`) are checked against Unicorn. Every lifter change passes these, all run with `.venv`: diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 6df619aa..936c55ef 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -98,6 +98,9 @@ # pipe_link_ms = 6 # The graphics driver's cache of shader libraries is checked at every start and, when it lost them (a driver update, # a cache the driver emptied), filled again in the background while the game runs. 0: no check; force: at every start. +# After two cold starts in a row warming stops being tried on that driver; once its cache can hold the libraries (on +# NVIDIA under Windows, Shader Cache Size Unlimited in the NVIDIA app), start once with force, or delete +# shader-warm.txt, to warm again. # lib_warm = 1 # How long (ms) a draw into a target no recent frame rendered waits for a pipeline built for the first time, # then is skipped. The default waits, as such a target (a lookup table, a new area's target) can stay empty if diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 3904265b..4e6f3d97 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -49,6 +49,7 @@ #include "platform/platform.h" #include "tex.h" #include "vk_gpa.h" +#include "vk_warm_hint.h" #define LOG_ONCE(...) do { static volatile int32_t once_; if (!plat_xchg32(&once_, 1)) rt_log(__VA_ARGS__); } while (0) @@ -5115,6 +5116,7 @@ static struct { volatile int32_t running, stop, started; PlatThread *th[LW_MAX_THREADS + 1]; int threads, strikes; + uint32_t vendor; // the GPU's vendor id, for what a cold cache's log line adds (vk_warm_hint.h) double t0; char stamp[PLAT_PATH_MAX * 2 + 32], id[64]; } LW; @@ -5208,7 +5210,8 @@ static void lw_main(void *arg) { if (!resume && same && strikes >= 2 && strcmp(mode, "force")) { rt_log("vk_pipeline: library warm: the prepared libraries are cold again (a sample's median %.0f ms) after two " "finished warms on this driver: it does not keep them, so pipelines compile when a draw first needs " - "them (NX2NAVI_LIB_WARM=force warms anyway)", median); + "them (NX2NAVI_LIB_WARM=force warms anyway).%s", median, + lw_cold_hint(LW.vendor, LW_COLD_GAVE_UP, LW_ON_WINDOWS)); return; } LW.strikes = resume ? strikes : same && !strcmp(was, "done") ? strikes + 1 : 0; @@ -5218,11 +5221,15 @@ static void lw_main(void *arg) { if (resume) rt_log("vk_pipeline: library warm: going on with the warm an earlier run began: %zu prepared variants on %d " "idle threads", LW.n, t); - else + else // cold right after a finished warm on this driver says more than a first cold (vk_warm_hint.h) rt_log("vk_pipeline: library warm: the driver's cache does not have the prepared libraries (a sample's median " "%.0f ms; a driver update or a change of what they are built with): building %zu prepared variants " "into it on %d idle threads while the game runs. Until it is done, a pipeline a draw first needs may " - "compile, which skips that draw for some frames or makes it wait", median, LW.n, t); + "compile, which skips that draw for some frames or makes it wait.%s", median, LW.n, t, + lw_cold_hint(LW.vendor, + median <= LW_COLD_MS ? LW_WARM_FORCED // warm, but lib_warm = force + : same && !strcmp(was, "done") ? LW_COLD_AFTER_WARM : LW_COLD_FIRST, + LW_ON_WINDOWS)); LW.t0 = now_ms(); LW.running = t; int started = 0; @@ -5252,6 +5259,7 @@ static void lw_start(void) { VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties(vk_ctx()->phys, &pp); snprintf(LW.id, sizeof LW.id, "%04x:%04x:%08x", pp.vendorID, pp.deviceID, pp.driverVersion); + LW.vendor = pp.vendorID; LW.th[0] = plat_thread_new(lw_main, NULL, "lib warm check"); } diff --git a/runtime/vk_warm_hint.h b/runtime/vk_warm_hint.h new file mode 100644 index 00000000..22fe3ad9 --- /dev/null +++ b/runtime/vk_warm_hint.h @@ -0,0 +1,55 @@ +// What the library warm's log adds when the driver's cache of stage libraries is cold (vk_pipeline.c lw_main), by GPU +// vendor, by platform and by why the warm found it cold. Own header so tests/warm_hint_check.c checks it without +// Vulkan. +// +// On NVIDIA under Windows the cache has a size limit, Shader Cache Size in the NVIDIA app. A limit smaller than the +// prepared libraries empties the cache as the warm fills it: the driver drops the oldest libraries to make room for +// the newest, and the next start's sample finds them gone. Measured on an RTX 5070 Ti (driver 617.42): the game's cache +// file stood at exactly 2048 MB, a full warm of 42,122 variants was cold again at the next start (a sample's median +// 241 ms), and with Shader Cache Size set to Unlimited the warm finished, the file grew to 2711 MB and later starts +// found the libraries in the cache (median 1.1 ms). Elsewhere there is no NVIDIA app and the driver's cache is set +// otherwise, so the hints that name the setting are for Windows only. +#pragma once + +#include + +enum { + LW_COLD_FIRST, // cold with no finished warm before it on this driver: a driver update, a change of what + // the libraries are built with, or the build's preparation, which writes no shader-warm.txt + LW_COLD_AFTER_WARM, // cold again right after a finished warm on the same driver + LW_COLD_GAVE_UP, // cold after two finished warms in a row: warming is no longer tried on this driver + LW_WARM_FORCED, // lib_warm = force on a cache the sample found warm: nothing lost, no cause to name +}; + +#define LW_VENDOR_NVIDIA 0x10de +#ifdef _WIN32 +#define LW_ON_WINDOWS 1 +#else +#define LW_ON_WINDOWS 0 +#endif + +// A sentence or two for the end of the log line ("" when there is nothing to add), each starting with a space. +// windows: the game runs on Windows (LW_ON_WINDOWS), where NVIDIA's limit is the NVIDIA app's Shader Cache Size. +static inline const char *lw_cold_hint(uint32_t vendor, int why, int windows) { + int nvidia = vendor == LW_VENDOR_NVIDIA && windows; + switch (why) { + case LW_COLD_FIRST: + return nvidia ? " On NVIDIA the driver's Shader Cache Size limit empties it too, when it is smaller than the " + "prepared libraries (a 2048 MB cache dropped the oldest as the warm built the newest): " + "Shader Cache Size Unlimited in the NVIDIA app keeps them." + : ""; + case LW_COLD_AFTER_WARM: + return nvidia ? " A finished warm did not stay in the driver's cache: on NVIDIA that is the Shader Cache Size " + "limit, smaller than the prepared libraries; set Shader Cache Size to Unlimited in the NVIDIA " + "app, or this warm is lost again." + : ""; + case LW_COLD_GAVE_UP: + return nvidia ? " On NVIDIA that is the Shader Cache Size limit: after setting it to Unlimited in the NVIDIA " + "app, start once with lib_warm = force (or delete shader-warm.txt) to warm again." + : " Once the driver's cache can hold them, start once with lib_warm = force (or delete " + "shader-warm.txt) to warm again."; + case LW_WARM_FORCED: + return ""; + } + return ""; +} diff --git a/tests/warm_hint_check.c b/tests/warm_hint_check.c new file mode 100644 index 00000000..677d2328 --- /dev/null +++ b/tests/warm_hint_check.c @@ -0,0 +1,72 @@ +// runtime/vk_warm_hint.h: what the library warm's log adds when the driver's cache of stage libraries is cold +// (runtime/vk_pipeline.c lw_main), by GPU vendor, by platform and by why it is cold. Built and run by +// tests/warm_hint_check.py; prints each failed check and exits 1 on any. +#include +#include + +#include "vk_warm_hint.h" + +static int fails; +#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0) + +enum { NVIDIA = 0x10de, AMD = 0x1002, INTEL = 0x8086 }; +enum { LINUX = 0, WINDOWS = 1 }; + +static int has(const char *s, const char *part) { return s && strstr(s, part) != NULL; } + +int main(void) { + // Windows, cold at a start with no finished warm before it (a driver update, or the build's preparation step, + // which writes no shader-warm.txt): on NVIDIA the driver's Shader Cache Size limit is named as another cause, + // with the setting. + const char *first = lw_cold_hint(NVIDIA, LW_COLD_FIRST, WINDOWS); + CHECK(has(first, "Shader Cache Size") && has(first, "Unlimited"), "NVIDIA first cold: %s", first); + CHECK(first && first[0] == ' ', "a hint starts with a space, as it follows a sentence: \"%s\"", first); + CHECK(!strcmp(lw_cold_hint(AMD, LW_COLD_FIRST, WINDOWS), ""), "AMD first cold: no NVIDIA setting"); + CHECK(!strcmp(lw_cold_hint(INTEL, LW_COLD_FIRST, WINDOWS), ""), "Intel first cold: no NVIDIA setting"); + + // Windows, cold right after a finished warm on the same driver: on NVIDIA that is the cache size limit, said + // plainly. + const char *again = lw_cold_hint(NVIDIA, LW_COLD_AFTER_WARM, WINDOWS); + CHECK(has(again, "Shader Cache Size") && has(again, "Unlimited"), "NVIDIA cold after a warm: %s", again); + CHECK(strcmp(again, first), "cold after a warm says more than a first cold"); + CHECK(!strcmp(lw_cold_hint(AMD, LW_COLD_AFTER_WARM, WINDOWS), ""), "AMD cold after a warm: no NVIDIA setting"); + + // Warming given up after two cold starts in a row: every vendor, on every platform, is told how to warm again + // once the cache can hold the libraries; NVIDIA on Windows also names the setting that lets it. + for (int os = LINUX; os <= WINDOWS; os++) + for (unsigned v = 0; v < 3; v++) { + unsigned vendor = v == 0 ? NVIDIA : v == 1 ? AMD : INTEL; + const char *gave = lw_cold_hint(vendor, LW_COLD_GAVE_UP, os); + CHECK(has(gave, "lib_warm = force") && has(gave, "shader-warm.txt"), "os %d vendor %04x gave up: %s", os, + vendor, gave); + } + CHECK(has(lw_cold_hint(NVIDIA, LW_COLD_GAVE_UP, WINDOWS), "Shader Cache Size"), + "NVIDIA gave up: names the setting"); + CHECK(!has(lw_cold_hint(AMD, LW_COLD_GAVE_UP, WINDOWS), "Shader Cache Size"), "AMD gave up: no NVIDIA setting"); + + // Elsewhere there is no NVIDIA app and no Shader Cache Size setting: the driver's cache is set otherwise there, + // so no case names them. + for (int why = LW_COLD_FIRST; why <= LW_WARM_FORCED; why++) { + const char *h = lw_cold_hint(NVIDIA, why, LINUX); + CHECK(!has(h, "NVIDIA app") && !has(h, "Shader Cache Size"), "Linux NVIDIA case %d: %s", why, h); + } + CHECK(!strcmp(lw_cold_hint(NVIDIA, LW_COLD_FIRST, LINUX), ""), "Linux NVIDIA first cold: nothing to add"); + + // A warm lib_warm = force started on a cache that is not cold: no cause to name, on any vendor or platform. + for (int os = LINUX; os <= WINDOWS; os++) + for (unsigned v = 0; v < 3; v++) { + unsigned vendor = v == 0 ? NVIDIA : v == 1 ? AMD : INTEL; + CHECK(!strcmp(lw_cold_hint(vendor, LW_WARM_FORCED, os), ""), "os %d vendor %04x forced warm: no hint", os, + vendor); + } + + // The game passes the platform it was built for. +#ifdef _WIN32 + CHECK(LW_ON_WINDOWS == 1, "a Windows build says so"); +#else + CHECK(LW_ON_WINDOWS == 0, "a build elsewhere says so"); +#endif + + if (!fails) printf("warm hint: all checks passed\n"); + return fails != 0; +} diff --git a/tests/warm_hint_check.py b/tests/warm_hint_check.py new file mode 100644 index 00000000..142100db --- /dev/null +++ b/tests/warm_hint_check.py @@ -0,0 +1,30 @@ +"""Build and run tests/warm_hint_check.c: runtime/vk_warm_hint.h, what the library warm's log adds when the driver's +cache of stage libraries is cold, by GPU vendor and by why it is cold. + +Usage: warm_hint_check.py +""" +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests' / 'diff')) +from common import MSYS_BIN, compiler # noqa: E402 + + +def main(): + work = ROOT / 'build' / 'test' / 'warm_hint' + work.mkdir(parents=True, exist_ok=True) + exe = work / ('warm_hint_check.exe' if os.name == 'nt' else 'warm_hint_check') + env = dict(os.environ) + if os.name == 'nt' and MSYS_BIN.exists(): + env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '') + # both platforms' text is checked through lw_cold_hint's windows argument; LW_ON_WINDOWS on the one built for + subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'), + str(ROOT / 'tests' / 'warm_hint_check.c'), '-o', str(exe)], env=env) + return subprocess.run([str(exe)], env=env).returncode + + +if __name__ == '__main__': + sys.exit(main())