diff --git a/docs/architecture.md b/docs/architecture.md index 810973dd..a8636cdf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,7 +115,11 @@ keeps per-core data without locks and relies on that, so the host models it (`ru - Pipelines: `VK_EXT_graphics_pipeline_library` where the driver has it: stage libraries compiled ahead, a fast link at a new state, an optimised pipeline replacing it from a background thread. A draw whose pipeline is not ready is skipped when one of its pass's targets was rendered in the previous frame, and waits otherwise. Compute pipelines - are compiled outside the pipeline lock, and one compile serves every thread that needs it. + are compiled outside the pipeline lock, and one compile serves every thread that needs it. The boot builds the + pipeline of every compute shader play recorded, but keeps only those of the shaders dispatched most recently + (`compute-used.bin`, `compute_keep`): the others go into the pipeline cache and are destroyed, and their first + dispatch creates them from it. A kept pipeline holds driver memory whether it is used or not, and the recorded list + grows with the play history (`runtime/vk_pipeline.c`, "compute pipelines the boot keeps"). - The game's tensor-core upscaler is replaced by a decode-and-scale pass (`vk_tonemap.c`); `vk_fsr.c` runs AMD FSR 1 (EASU, RCAS) on the present when the window is larger than the picture. - HDR (`vk_hdr.c`): the game tonemaps its scene into a float target, upscales it, draws the HUD into a second float diff --git a/docs/building.md b/docs/building.md index 22213142..9e280ad4 100644 --- a/docs/building.md +++ b/docs/building.md @@ -303,6 +303,11 @@ Runtime probes, each a mode of `nx2navi.exe` (the list is at the top of `runtime part of `--probe-vkexec` translate shaders of the census below: they look for `build//shaders/jobs.json` of the exe's own version first, then of the other one. `--probe-vkmem` also checks CPU-backed texture uploads and rewrites, GPU-written texture aliases, and memory write watches. +`--probe-prewarm` runs the boot's pipeline stages without the game, from the caches next to the exe (it skips when +there are none): the pipeline cache load, the shader prewarm, the recorded pipelines and the compute pipeline prebuild, +with a `mem: boot stage` log line after each that gives the device memory and the process commit so far. It then asks +for compute pipelines the boot did not keep, as their first dispatch would, and prints what one costs. A change to what +the boot builds or keeps (`runtime/vk_pipeline.c`: `prewarm_run`, `lb_compute`, the `compute-used.bin` record) passes it. Shaders: `.venv-py314\Scripts\python tools\shader_census.py --version --translate` extracts every Maxwell shader to `build\\shaders\` and runs it through the translator and `spirv-val`; failure groups go to `failures.txt`. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 8b7962dc..47cf58d4 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -104,6 +104,11 @@ # threads mostly wait on the graphics driver's lock: 16 on NVIDIA spent about 8 cores spinning in the driver. # lib_prebuild = 1 # lib_prebuild_threads = 2 +# How many compute pipelines the start builds and keeps: those of the compute shaders the game used most recently. +# Every other known one is built into the pipeline cache and released again, and made from the cache when the game +# first uses it (well under a millisecond). all: the start keeps every compute shader the game ever used, as versions up +# to 0.6.1 did; with a long play history that held more than 2 GB of video memory. 0: none are kept. +# compute_keep = 512 # 1: very large compute shaders that no cache holds are built on one background thread after the start. 0: each is # built when the game first uses it, which can hold that frame for a second or two. # lib_prebuild_big_bg = 1 diff --git a/runtime/cfg.c b/runtime/cfg.c index f2b9d989..b8d051c8 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -38,7 +38,7 @@ static const char *const known[] = { "BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_RSTICK_DOWN", "BIND_RSTICK_LEFT", "BIND_RSTICK_RIGHT", "BIND_RSTICK_UP", "BIND_SHAKE", "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", - "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CACHE_DIE", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA", + "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CACHE_DIE", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COMPUTE_KEEP", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLSS_IDLE_FREE", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", @@ -115,6 +115,7 @@ static const struct { const char *key, *section; } sections[] = { {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"}, {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_PREBUILD", "performance"}, + {"COMPUTE_KEEP", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, @@ -220,6 +221,8 @@ static const CfgMeta metas[] = { {"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART}, {"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, {"PIPE_BINARIES", "0", "0|1", RESTART}, {"ONESHOT_WAIT_MS", "30000", "0..60000", RESTART}, {"ONESHOT_BUDGET", "0", "0|1", RESTART}, {"PIPE_BINARIES_MAX_MB", "1024", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART}, {"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART}, + // all|: the compute pipelines the boot keeps, those dispatched most recently (vk_pipeline.c, compute-used.bin) + {"COMPUTE_KEEP", "512", NULL, RESTART}, {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, diff --git a/runtime/host.h b/runtime/host.h index 880acae0..f86f7a6e 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -378,4 +378,5 @@ int probe_gamepad(void); // input.c int probe_movie(void); // movie_probe.c: host VP9 movie::Decoder on one RomFS movie int vk_pipeline_prepare_shaders(void); // vk_pipeline.c: --prepare-shaders (tools/prepare_shaders.py) int vk_pipeline_warm_shaders(void); // vk_pipeline.c: --warm-shaders (tools/build_game.py) +int vk_pipeline_probe_prewarm(void); // vk_pipeline.c: --probe-prewarm int vk_pipeline_spirv_selftest(void); // vk_pipeline.c: --selftest, the SPIR-V rewrites without a device diff --git a/runtime/main.c b/runtime/main.c index 36cb77fb..9f1e6d1d 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -20,6 +20,8 @@ // --probe-dpi the process DPI awareness (per-monitor v2) and the primary monitor's size; no game files // --probe-gfx-abi NVN object layouts against nvn.h's reserved sizes; needs no game files // --probe-nvn, --probe-nvn-tex, --probe-window, --probe-shader, --probe-vkmem, --probe-vkexec graphics probes +// --probe-prewarm the boot's pipeline stages from the caches next to the executable, the memory each leaves and +// what a compute pipeline's first dispatch costs; skips without recorded shaders, runs no game // --probe-astc the compute ASTC decoder against astc_decode_ldr; needs no game files // --probe-zstd host zstd streams (ov_zstd.c) on frames made here; needs no game files // --probe-input gamepad polls on a virtual gamepad, also while SDL's update takes long; needs no game files @@ -495,7 +497,7 @@ static int usage(const char *argv0) { "--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " "--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | " - "--prepare-shaders | --warm-shaders | --boot |--seconds S [--shot PNG]]\n", argv0); + "--prepare-shaders | --warm-shaders | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0); return 2; } @@ -643,6 +645,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--probe-movie")) return probe_movie(); if (!strcmp(mode, "--prepare-shaders")) return vk_pipeline_prepare_shaders(); if (!strcmp(mode, "--warm-shaders")) return vk_pipeline_warm_shaders(); + if (!strcmp(mode, "--probe-prewarm")) return vk_pipeline_probe_prewarm(); if (!strcmp(mode, "--boot")) { cfg.boot = 1; rt_boot(); return 0; } return usage(argv[0]); } diff --git a/runtime/vk.h b/runtime/vk.h index a45c430f..0c330abb 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -342,6 +342,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic int vk_mem_class_of(uint64_t addr); // VK_MCLS_* of the pool at addr (NONE when unbacked) void vk_mem_census(uint64_t frame); // at each present: pool census and traffic, logged at frame 1800 and, // with NX2NAVI_TRACE=perf, every 1800 frames +void vk_mem_stage_log(const char *stage); // one line after a boot stage: device memory, process commit // VRAM in use and its budget, summed over the DEVICE_LOCAL heaps. exact 1: VK_EXT_memory_budget (the whole process), // 0: this allocator's own allocations against the heap sizes. 0 when unknown. int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact); @@ -680,7 +681,8 @@ VkPipeline vk_pipeline_graphics(const VkPipeState *s, const VkPipeInfo **info); // wait_ms > 0 waits up to that long for a pending pipeline first. Pipelines built in a run are recorded to // pipeline-keys.bin next to the pipeline cache and prebuilt at the next boot (NX2NAVI_PIPE_PREBUILD=0: not). VkPipeline vk_pipeline_graphics_async(const VkPipeState *s, const VkPipeInfo **info, int *pending, uint32_t wait_ms); -// rt_boot calls it unless --shot is given; NX2NAVI_PIPE_SYNC=1 makes it a no-op. Probes never call it. +// rt_boot calls it unless --shot is given; NX2NAVI_PIPE_SYNC=1 makes it a no-op. Of the probes only --probe-prewarm +// calls it. void vk_pipeline_async_enable(void); int vk_pipeline_async_on(void); // 1 after vk_pipeline_async_enable: graphics shaders are translated in the background too // Boot prewarm, before the guest runs (rt_boot): loads the shaders pipeline-keys.bin names, translating the ones not diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 284c004e..bc55e15b 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2682,6 +2682,39 @@ static void outside_watch(uint64_t frame) { last = out, seen = 1; } +// One line per boot stage (vk_pipeline.c calls it after the pipeline cache load, the shader prewarm, the recorded +// pipelines and the compute pipeline prebuild): the device-local memory in use, the part of it VMA did not allocate +// (outside_watch's "outside VMA": pipelines and the driver's own, before the swapchain and the upscaler exist), and the +// process's commit and working set. Subtracting two lines gives what a stage left behind: with a long play history the +// boot's compute pipelines alone held 2.3 GiB of device memory on an RTX 5090 (4,066 pipelines kept, 95 of them +// dispatched before the first village), which is what this line was added to place. +void vk_mem_stage_log(const char *stage) { + VkCtx *c = vk_ctx(); + double proc, used, limit; + float ram_used, ram_total, ws; + plat_commit_info(&proc, &used, &limit); + plat_mem_info(&ram_used, &ram_total, &ws); + const double M = 1048576.0; + uint64_t usage = 0, blocks = 0; + int budget = c && c->has_memory_budget; + if (budget) { + VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + VmaBudget vb[VK_MAX_MEMORY_HEAPS] = {0}; + if (c->vma) vmaGetHeapBudgets(c->vma, vb); + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + usage += bp.heapUsage[h], blocks += vb[h].statistics.blockBytes; + } + char dev[96] = "device memory unknown (no VK_EXT_memory_budget)"; + if (budget) + snprintf(dev, sizeof dev, "device memory in use %.0f MiB, outside VMA %.0f MiB", usage / M, + usage > blocks ? (usage - blocks) / M : 0.0); + rt_log("mem: boot stage, after %s: %s; process commit %.0f MB, working set %.0f MB; system commit %.0f of %.0f MB", + stage, dev, proc, (double)ws, used, limit); +} + // With NX2NAVI_TRACE=perf every 600 frames: what holds device-local memory, to see what grows over a long session. // VMA's blocks and allocations on the device-local heaps, texture entries with their own memory, the VRAM mirrors and // the objects waiting for their last submission to retire. diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 6cfc0812..c4a2da3d 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -130,6 +130,8 @@ typedef struct Pipe { VkPipeline fast; // GPL: the fast-linked pipeline (kept until shutdown: recorded draws use it) uint8_t fast_tried; uint8_t link_queued; // a waiting draw queued its fast link to the workers (LibJob p), not yet run + uint8_t used; // compute: a dispatch of this run asked for it (vk_pipeline_compute) + uint8_t pre; // compute: built and kept by the boot prebuild (lb_compute) or bg_main size_t key_size; uint8_t key[]; } Pipe; @@ -221,6 +223,8 @@ static struct { double from_ms, compile_ms; } PB = {.lock = PLAT_MUTEX_INIT}; static __thread int pb_last; // 1: the last pipe_create on this thread made its pipeline from binaries +static __thread int cc_hit; // compute_create's last pipeline on this thread: 1 from pipeline-cache.bin or + // the pipeline binaries, 0 not, -1 the driver did not say static uint64_t pb_hash(uint32_t type, const uint8_t *key, uint32_t n) { uint64_t h = fnv(fnv(FNV0, &type, 4), key, n); @@ -513,7 +517,9 @@ static PlatMutex cache_rw = PLAT_MUTEX_INIT; // but the background builder of large compute shaders) for P.cache itself. A create into it does not hold cache_rw, so // a build of minutes does not keep pipeline-cache.bin from being saved; bg_merge moves its content into P.cache. static __thread VkPipelineCache own_cache; -static volatile uint32_t cache_merges; // bg_merge's merges into P.cache: new content for autosave_once to save +// New content in P.cache that P.stats does not count, for autosave_once to save: bg_merge's merges, and the compiles +// of pipelines built for the cache only (lb_compute keep 0). +static volatile uint32_t cache_merges; static VkResult pipe_create_in(int kind, void *ci, VkPipeline *pipe, VkPipelineCache cache) { int shared = cache && !own_cache; @@ -657,6 +663,7 @@ static void nan_preserve_init(VkCtx *c) { } static void keys_open(void); +static void cu_open(void); static void prebuild_start(void); static int ensure_init(void) { @@ -797,12 +804,15 @@ static int ensure_init(void) { if (P.dyn_blend_eq) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_BLEND_EQUATION_EXT; if (P.dyn_wmask) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_WRITE_MASK_EXT; if (P.dyn_a2c) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_ALPHA_TO_COVERAGE_ENABLE_EXT; + vk_mem_stage_log("device creation"); cache_load(c); + vk_mem_stage_log("the pipeline cache load"); rt_log("vk_pipeline: max push descriptors %u; dynamic vertex input %d, depth clamp %d, polygon mode %d, blend enable %d, " "blend equation %d, write mask %d, alpha to coverage %d; cache %s", P.max_push, P.dyn_vi, P.dyn_clamp, P.dyn_poly, P.dyn_blend_en, P.dyn_blend_eq, P.dyn_wmask, P.dyn_a2c, P.cache_path[0] ? P.cache_path : ""); keys_open(); + cu_open(); plat_xchg32(&P.ready, 1); plat_mutex_unlock(&P.lock); prebuild_start(); @@ -3725,6 +3735,7 @@ static void prewarm_run(void) { __atomic_store_n(&pw_phase, 1, __ATOMIC_RELEASE); gfx_shader_prewarm((const uint8_t(*)[32])ids, u, sh, cpus, procs < 1 ? 1 : procs); double t1 = now_ms(); + vk_mem_stage_log("the shader prewarm"); if (P.indep) { // vertex output unions first: workers may take a job before the queue is complete uint64_t pair = 0, uni = 0, pairs = 0; @@ -3815,6 +3826,7 @@ static void prewarm_run(void) { K.file = NULL; plat_mutex_unlock(&P.lock); fb_log("after the boot prewarm"); + vk_mem_stage_log("the recorded pipelines' prebuild"); vk_pipeline_prewarm_marker_clear(); // the boot's share is done; what the pool builds later runs during play lb_start(pr_hist, fs_hist); } @@ -4032,6 +4044,9 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache if (pipe_create(2, &ci, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); if (pipe) fb_note(FB_COMPUTE, &fb); + cc_hit = !pipe ? -1 : pb_last ? 1 + : !P.feedback || !P.cache || !(fb.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) ? -1 + : (fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0; if (P.feedback >= 2 && (pipe || !cached_only)) rt_log("vk_pipeline: compute %016llx: %llu words, module %016llx, %s, %.1f ms", (unsigned long long)fnv(FNV0, code, words * 4), (unsigned long long)words, (unsigned long long)mm_hash, @@ -4059,12 +4074,14 @@ static Layout *compute_layout(const GfxShader *cs, Bindings *B, const char **why // stays is the driver's own disk cache: on AMD a library built again in a later process takes 0.1 ms instead of // 55 ms (vertex) / 28 ms (fragment). Our pipeline-cache.bin is not written there: // it would grow by about 10 KB per library (~240 MB) and be read into memory at every boot. -// - boot (lb_worker), after the pipeline-key prewarm, on cpus/4 threads at PLAT_PRIO_IDLE (the short P.lock +// - boot (lb_worker), after the pipeline-key prewarm, on two threads (lb_start) at PLAT_PRIO_IDLE (the short P.lock // sections and shader loads at normal priority, so a preempted thread never holds the lock the replay needs): -// compute pipelines of every recorded and prepared compute shader, kept (vk_pipeline_compute builds compute -// synchronously on the replay thread; 163 prepared), and the libraries of the first NX2NAVI_LIB_PREBUILD_KEEP variants -// play recorded (priority 0), kept in libs[] with their shaders. KEEP defaults to 0: a kept library costs 140-460 KB -// of process memory (10,000 kept took 9 GB in run 1) to save a rebuild the driver cache makes 0.1 ms. +// compute pipelines of every recorded and prepared compute shader (vk_pipeline_compute builds compute +// synchronously on the replay thread; 163 prepared): those of the shaders dispatched most recently stay, the others +// are built into the pipeline cache and destroyed (comment at compute-used.bin). Then the libraries of the first +// NX2NAVI_LIB_PREBUILD_KEEP variants play recorded (priority 0), kept in libs[] with their shaders. KEEP defaults to +// 0: a kept library costs 140-460 KB of process memory (10,000 kept took 9 GB in run 1) to save a rebuild the driver +// cache makes 0.1 ms. // Library state: the vertex output union known now (vs_union_peek; the prewarm fills it from pipeline-keys.bin, the // preparation has none), the clamp, polygon mode and depth mode most recorded pipelines use (defaults without a key // file: clamp 0, fill, -w..w), single-sampled, no dual source; the state dynamic on this GPU is 0 in every key. @@ -4084,6 +4101,7 @@ static Layout *compute_layout(const GfxShader *cs, Bindings *B, const char **why static struct { GfxWarmEnt *e; size_t n; + size_t n_keep; // e[0..n_keep): compute shaders whose pipeline stays (cu_keep) volatile int64_t next; volatile int32_t stop, running; int threads; @@ -4241,15 +4259,228 @@ static int cs_flight_has(const GfxShader *cs) { return 0; } +static uint8_t cs_flight_want[CS_FLIGHT]; // a dispatch waits for the slot's pipeline (vk_pipeline_compute) + static int cs_flight_add(const GfxShader *cs) { // 0 when the table is full (the caller compiles anyway) for (int i = 0; i < CS_FLIGHT; i++) - if (!cs_flight[i]) return cs_flight[i] = cs, 1; + if (!cs_flight[i]) return cs_flight[i] = cs, cs_flight_want[i] = 0, 1; return 0; } -static void cs_flight_remove(const GfxShader *cs) { +// A dispatch waits for cs's build: a builder that would only have filled the pipeline cache keeps the pipeline. +static void cs_flight_wanted(const GfxShader *cs) { for (int i = 0; i < CS_FLIGHT; i++) - if (cs_flight[i] == cs) cs_flight[i] = NULL; + if (cs_flight[i] == cs) cs_flight_want[i] = 1; +} + +static int cs_flight_remove(const GfxShader *cs) { // 1 when a dispatch waited for it + int want = 0; + for (int i = 0; i < CS_FLIGHT; i++) + if (cs_flight[i] == cs) cs_flight[i] = NULL, want |= cs_flight_want[i]; + return want; +} + +// ---- compute pipelines the boot keeps (compute-used.bin next to pipeline-keys.bin; NX2NAVI_COMPUTE_KEEP) +// The boot prebuild (lb_compute) used to build the pipeline of every compute shader play ever recorded and keep all of +// them for the process. A pipeline that exists holds driver memory whether or not anything dispatches it, and the list +// only grows with the play history: on an RTX 5090 (NVIDIA 621.01) a history of 4,066 compute shaders held 2.3 GiB of +// device memory and a matching share of the process's commit from the title screen on, while a boot into the first +// village dispatched 95 of them. That is memory an 8 to 12 GB card needs for textures. +// So the boot keeps only the pipelines of the NX2NAVI_COMPUTE_KEEP (default 512) compute shaders dispatched most +// recently, and builds every other one into the pipeline cache and destroys it at once (lb_compute keep 0): the cache +// then holds it, so its first dispatch creates it without compiling, and no pipeline of an unused shader stays. On +// that machine a compute pipeline came from pipeline-cache.bin in 0.25 ms on average at boot, and 256 first dispatches +// of pipelines the boot had not kept (--probe-prewarm, with a game running beside it) took 0.19 ms on average and 3 ms +// at most. Built and destroyed one after the other, they leave nothing behind in the driver either: 372 of them left +// its device memory where it was. NVIDIA's driver does not hand back the device memory of destroyed pipelines, it +// keeps it for the next ones (400 test pipelines, destroyed together: no change), so building all of them first and +// destroying them afterwards would end at the same 2.3 GiB. +// The record: one entry per compute shader a run dispatched (its input id) with the number of the last run that did; +// each run that dispatches anything takes the next number. Kept at boot are the keep_n entries with the highest +// numbers, so a short run does not push out what a long one used, and nothing is forgotten until newer shaders take +// its place. Without a record (the first run that has one) nothing is kept and the run pays the first dispatches once. +// NX2NAVI_COMPUTE_KEEP=all keeps every compute shader of the warm list as before; 0 keeps none. +#define CU_VERSION 1 +typedef struct { char magic[8]; uint32_t version, rec_size, run, pad; } CuHeader; +typedef struct { uint8_t iid[32]; uint32_t run, pad; } CuRec; +static struct { + PlatMutex lock; + int opened, dirty; + char path[PLAT_PATH_MAX * 2 + 32]; + CuRec *rec; + uint8_t *keep; // per record read at boot: among the keep_n dispatched most recently + uint32_t n, cap, n_file; // n_file: the records read at boot (keep has one byte for each) + uint32_t *slot; // open addressing by the id's first bytes: record index + 1, 0 empty + uint32_t slot_cap; + uint32_t run; // this run's number + int quiet; // --probe-prewarm: its dispatches are not the game's, the file stays as it is + int32_t keep_n; // NX2NAVI_COMPUTE_KEEP; -1: all + uint32_t kept; // records flagged in keep +} CU = {.lock = PLAT_MUTEX_INIT}; + +// What the first dispatches of compute shaders cost this run (under P.lock): each creates its pipeline on the replay +// thread. burst: the creations that fell into one window of CF_WINDOW_MS, the worst such window kept, which is what a +// player would feel as one long frame when a new area starts. +#define CF_WINDOW_MS 33.0 +static struct { + uint64_t n, hit, pre_used; // hit: read from pipeline-cache.bin; pre_used: kept from the boot and dispatched + double ms, max_ms, hit_ms; + double win_t0, win_ms, worst_ms; + uint32_t win_n, worst_n; +} CF; + +static uint32_t *cu_slot(const uint8_t iid[32]) { // the id's slot, or the empty one it would take. Caller holds CU.lock. + uint64_t h; + memcpy(&h, iid, 8); + for (uint32_t j = (uint32_t)(h * 0x9e3779b97f4a7c15ull >> 32) & (CU.slot_cap - 1);; j = (j + 1) & (CU.slot_cap - 1)) + if (!CU.slot[j] || !memcmp(CU.rec[CU.slot[j] - 1].iid, iid, 32)) return &CU.slot[j]; +} + +// Room for one more record and a table at most half full. 0 when out of memory. Caller holds CU.lock. +static int cu_room(void) { + if (CU.n == CU.cap) { + uint32_t cap = CU.cap ? CU.cap * 2 : 1024; + CuRec *t = realloc(CU.rec, cap * sizeof *t); + if (!t) return 0; + CU.rec = t, CU.cap = cap; + } + if ((CU.n + 1) * 2 > CU.slot_cap) { + uint32_t cap = CU.slot_cap ? CU.slot_cap * 2 : 4096; + uint32_t *t = calloc(cap, sizeof *t), *old = CU.slot; + if (!t) return 0; + CU.slot = t, CU.slot_cap = cap; + for (uint32_t i = 0; i < CU.n; i++) *cu_slot(CU.rec[i].iid) = i + 1; + free(old); + } + return 1; +} + +static int cu_run_cmp(const void *a, const void *b) { // {run, index} pairs, the highest run first + const uint32_t *x = a, *y = b; + return x[0] != y[0] ? (x[0] < y[0] ? 1 : -1) : (x[1] > y[1]) - (x[1] < y[1]); +} + +// The number of records in a compute-used.bin image (f, size) with its header in *h, or -1 when it is not one. +static long cu_parse(const uint8_t *f, uint64_t size, CuHeader *h) { + if (!f || size < sizeof *h) return -1; + memcpy(h, f, sizeof *h); + if (memcmp(h->magic, "NX2NCUSE", 8) || h->version != CU_VERSION || h->rec_size != sizeof(CuRec)) return -1; + return (long)((size - sizeof *h) / sizeof(CuRec)); +} + +// Reads the record and marks what the boot keeps. Caller holds P.lock (ensure_init), after keys_open. +static void cu_open(void) { + if (!A.on || !K.path[0]) return; + const char *e = cfg_value("NX2NAVI_COMPUTE_KEEP"); + plat_mutex_lock(&CU.lock); + CU.keep_n = !strcmp(e, "all") ? -1 : atoi(e) < 0 ? 0 : atoi(e); + snprintf(CU.path, sizeof CU.path, "%s", K.path); + char *cut = plat_last_sep(CU.path); + snprintf(cut ? cut + 1 : CU.path, sizeof CU.path - (size_t)(cut ? cut + 1 - CU.path : 0), "compute-used.bin"); + uint64_t size = 0; + uint8_t *f = plat_file_read(CU.path, &size); + CuHeader h; + long n = cu_parse(f, size, &h); + if (n >= 0) { + for (long i = 0; i < n && cu_room(); i++) { + CuRec r; + memcpy(&r, f + sizeof h + (size_t)i * sizeof r, sizeof r); + uint32_t *s = cu_slot(r.iid); + if (*s) continue; // an id twice: the first entry stands + CU.rec[CU.n] = r, *s = ++CU.n; + } + CU.run = h.run; + } else if (f) { + rt_log("vk_pipeline: %s has another format or version; it is rewritten", CU.path); + } + free(f); + CU.n_file = CU.n, CU.run++; + uint32_t (*order)[2] = CU.n ? malloc(CU.n * sizeof *order) : NULL; + CU.keep = CU.n ? calloc(CU.n, 1) : NULL; + if (order && CU.keep) { + for (uint32_t i = 0; i < CU.n; i++) order[i][0] = CU.rec[i].run, order[i][1] = i; + qsort(order, CU.n, sizeof *order, cu_run_cmp); + for (uint32_t i = 0; i < CU.n && (CU.keep_n < 0 || i < (uint32_t)CU.keep_n); i++) CU.keep[order[i][1]] = 1, CU.kept++; + } + free(order); + CU.opened = 1; + rt_log("vk_pipeline: %s: %u compute shaders dispatched in earlier runs; the boot keeps the pipelines of %s%u " + "dispatched most recently (NX2NAVI_COMPUTE_KEEP %s)", CU.path, CU.n, + CU.keep_n < 0 ? "every recorded compute shader, among them the " : "the ", CU.kept, e); + plat_mutex_unlock(&CU.lock); +} + +// 1 when the boot keeps the pipeline of the compute shader with this input id. +static int cu_keep(const uint8_t iid[32]) { + plat_mutex_lock(&CU.lock); + int keep = CU.keep_n < 0 || !CU.opened; // without the record (no cache folder): as before, every one + if (!keep && CU.slot_cap) { + uint32_t s = *cu_slot(iid); + keep = s && s <= CU.n_file && CU.keep && CU.keep[s - 1]; + } + plat_mutex_unlock(&CU.lock); + return keep; +} + +// The first dispatch of cs in this run: its entry takes this run's number. The runtime's own shaders (no input id) +// are built at their first use every run and are not recorded. +static void cu_note(const GfxShader *cs) { + static const uint8_t none[32]; + const uint8_t *iid = gfx_shader_input_id(cs); + if (!CU.opened || !memcmp(iid, none, 32)) return; + plat_mutex_lock(&CU.lock); + if (cu_room()) { + uint32_t *s = cu_slot(iid); + if (!*s) { + memset(&CU.rec[CU.n], 0, sizeof *CU.rec); + memcpy(CU.rec[CU.n].iid, iid, 32); + *s = ++CU.n; + } + if (CU.rec[*s - 1].run != CU.run) CU.rec[*s - 1].run = CU.run, CU.dirty = 1; + } + plat_mutex_unlock(&CU.lock); +} + +// Writes the record to path. 1 when written. +static int cu_write(const char *path) { + plat_mutex_lock(&CU.lock); + CuHeader h = {{'N', 'X', '2', 'N', 'C', 'U', 'S', 'E'}, CU_VERSION, sizeof(CuRec), CU.run, 0}; + size_t size = sizeof h + CU.n * sizeof(CuRec); + uint8_t *out = malloc(size); + if (out) memcpy(out, &h, sizeof h), memcpy(out + sizeof h, CU.rec, CU.n * sizeof(CuRec)); + CU.dirty = 0; + plat_mutex_unlock(&CU.lock); + int ok = out && plat_file_write_atomic(path, out, size); + free(out); + return ok; +} + +// Writes the record when a dispatch changed it (the autosave thread and the save at exit). +static void cu_flush(void) { + if (!CU.opened || CU.quiet || !*(volatile int *)&CU.dirty) return; + if (!cu_write(CU.path)) rt_log("vk_pipeline: cannot write %s", CU.path); +} + +// A compute pipeline created at a dispatch took ms. Caller holds P.lock. +static void cf_note(double ms, int hit) { + double now = now_ms(); + CF.n++, CF.ms += ms; + if (ms > CF.max_ms) CF.max_ms = ms; + if (hit == 1) CF.hit++, CF.hit_ms += ms; + if (now - CF.win_t0 > CF_WINDOW_MS) CF.win_t0 = now, CF.win_n = 0, CF.win_ms = 0; + CF.win_n++, CF.win_ms += ms; + if (CF.win_ms > CF.worst_ms) CF.worst_ms = CF.win_ms, CF.worst_n = CF.win_n; +} + +static void cf_log(void) { + plat_mutex_lock_shared(&P.lock); + if (CF.n || CF.pre_used) + rt_log("vk_pipeline: compute pipelines created at their first dispatch: %llu (avg %.2f ms, longest %.2f ms; %llu " + "of them read from the pipeline cache file, avg %.2f ms); the %.0f ms that held the most of it: %u " + "pipelines, %.1f ms; of the compute pipelines kept from the boot, %llu were dispatched", + (unsigned long long)CF.n, CF.n ? CF.ms / CF.n : 0.0, CF.max_ms, (unsigned long long)CF.hit, + CF.hit ? CF.hit_ms / CF.hit : 0.0, CF_WINDOW_MS, CF.worst_n, CF.worst_ms, (unsigned long long)CF.pre_used); + plat_mutex_unlock_shared(&P.lock); } // The background builder of large compute shaders (comment at bg_main). @@ -4275,6 +4506,7 @@ VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { double w0 = !p && cs_flight_has(cs) ? now_ms() : 0; int bg = 0; while (!p && cs_flight_has(cs)) { // another thread is compiling it: its result, not a second compile + cs_flight_wanted(cs); // a build that only fills the pipeline cache keeps its pipeline for this dispatch if (BG.building == cs && BG.th && !plat_xchg32(&BG.raised, 1)) // the background builder: normal priority now plat_thread_set_priority(BG.th, PLAT_PRIO_NORMAL), bg = 1; plat_mutex_unlock(&P.lock); @@ -4299,11 +4531,15 @@ VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { int flight = cs_flight_add(cs); plat_mutex_unlock(&P.lock); // the compile runs without the lock uint64_t p0 = perf_begin(); + double c0 = now_ms(); pipe = compute_create(cs, l, 0); + double cms = now_ms() - c0; + int hit = cc_hit; perf_end(PERF_PIPE, p0); plat_mutex_lock(&P.lock); if (flight) cs_flight_remove(cs); if (!pipe) why = "pipeline creation failed"; + else cf_note(cms, hit); } if ((p = pipe_find(&k, sizeof k, h))) { // another thread added it meanwhile (no flight slot was free) if (pipe) vkDestroyPipeline(P.device, pipe, NULL); @@ -4318,6 +4554,11 @@ VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { p = pipe_add(&k, sizeof k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_COMPUTE, 0); } } + if (p && !p->used) { // the first dispatch of this run: the boot of the next runs keeps this shader's pipeline + p->used = 1; + CF.pre_used += p->pre; + if (p->info.pipeline) cu_note(cs); + } plat_mutex_unlock(&P.lock); if (!p) return VK_NULL_HANDLE; if (info) *info = &p->info; @@ -4448,7 +4689,7 @@ static void bg_merge(VkPipelineCache own, int built) { if (locked) { VkResult r = vkMergePipelineCaches(P.device, P.cache, 1, &own); plat_mutex_unlock(&cache_rw); - if (r == VK_SUCCESS) cache_merges++; + if (r == VK_SUCCESS) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED); else rt_log("vk_pipeline: background compute build: its cache did not merge into %s (%d)", P.cache_path, (int)r); } vkDestroyPipelineCache(P.device, own, NULL); @@ -4534,14 +4775,18 @@ static void bg_main(void *arg) { VkPipeline pipe = compute_create(cs, l, 0); own_cache = VK_NULL_HANDLE; double ms = now_ms() - t0; + int keep = cu_keep(iid); plat_mutex_lock(&P.lock); - if (flight) cs_flight_remove(cs); + if (flight && cs_flight_remove(cs)) keep = 1; // a dispatch waits for it BG.building = NULL; - if (pipe && !pipe_find(&k, sizeof k, h)) { - pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0); + Pipe *e = NULL; + if (pipe && keep && !pipe_find(&k, sizeof k, h) && + (e = pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0))) { + e->pre = 1; P.stats.compute++; - } else if (pipe) { + } else if (pipe) { // only for the pipeline cache (comment at compute-used.bin), or a dispatch built its own vkDestroyPipeline(P.device, pipe, NULL); + if (!own) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED); // new content in P.cache to save } plat_mutex_unlock(&P.lock); if (own) bg_merge(own, pipe != VK_NULL_HANDLE); // after the entry: a dispatch waiting for it does not wait for this @@ -4567,7 +4812,9 @@ static void bg_start(void) { "dispatch", BG.n); } -static void lb_compute(const GfxShader *cs, const uint8_t iid[32]) { +// keep 0: the pipeline is built for the pipeline cache only and destroyed at once, unless a dispatch came to wait for +// it meanwhile (comment at compute-used.bin). +static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep) { struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; @@ -4595,19 +4842,25 @@ static void lb_compute(const GfxShader *cs, const uint8_t iid[32]) { if (LB.idle) plat_thread_priority(PLAT_PRIO_NORMAL); VkPipeline pipe = compute_create(cs, l, big); double ms = now_ms() - t0; + int compiled = pipe && cc_hit != 1, kept = 0; lb_lock(); - if (flight) cs_flight_remove(cs); - if (pipe && !pipe_find(&k, sizeof k, h)) { - pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0); + if (flight && cs_flight_remove(cs)) keep = 1; // a dispatch waits for it + Pipe *e = NULL; + if (pipe && pipe_find(&k, sizeof k, h)) { + vkDestroyPipeline(P.device, pipe, NULL); + have = 1; + } else if (pipe && keep && (e = pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0))) { + e->pre = 1, kept = 1; P.stats.compute++; } else if (pipe) { vkDestroyPipeline(P.device, pipe, NULL); - have = 1; + // P.stats.compute does not count it, so the autosave would not see what its compile added to P.cache + if (compiled) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED); } lb_unlock(); if (!pipe && big && bg_on()) bg_queue(cs, iid, words); else if (!pipe && big) lb_defer(iid, words); - else lb_count(2, pipe ? have : 2, ms); + else lb_count(2, !pipe ? 2 : have ? 1 : kept ? 0 : 3, ms); } // The pre-rasterization (vertex) or fragment-shader library of one shader with the LB state. @@ -4699,7 +4952,8 @@ static void lb_report(const char *how) { plat_mutex_unlock_shared(&LB.stat); } -// Boot: compute pipelines and kept libraries, all entered for the process. +// Boot: compute pipelines (the first LB.n_keep entered for the process, the others only built into the pipeline +// cache) and kept libraries. static void lb_worker(void *arg) { (void)arg; plat_thread_priority(PLAT_PRIO_IDLE); @@ -4721,7 +4975,7 @@ static void lb_worker(void *arg) { LB.noshader++; plat_mutex_unlock(&LB.stat); } else if (gfx_shader_stage(sh) == GFX_STAGE_CS) { - lb_compute(sh, LB.e[i].iid); + lb_compute(sh, LB.e[i].iid, i < LB.n_keep); } else if (gfx_shader_stage(sh) == GFX_STAGE_VS || gfx_shader_stage(sh) == GFX_STAGE_FS) { if (P.gpl) lb_library(sh, gfx_shader_stage(sh) == GFX_STAGE_VS, 1); else lb_skip_lib(); @@ -4733,6 +4987,7 @@ static void lb_worker(void *arg) { if (!plat_dec32(&LB.running)) { lb_report(LB.stop ? "stopped" : "done"); fb_log("after the stage library prebuild"); + vk_mem_stage_log("the compute pipeline prebuild"); bg_start(); } return; @@ -4924,7 +5179,8 @@ static void lw_stop(void) { if (LW.th[i]) plat_thread_join(LW.th[i]), LW.th[i] = NULL; } -// Boot: compute shaders of the warm list (recorded and prepared), then the first KEEP recorded graphics variants. +// Boot: compute shaders of the warm list (recorded and prepared), those whose pipeline stays first, then the first +// KEEP recorded graphics variants. static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { const char *e = cfg_value("NX2NAVI_LIB_PREBUILD"), *ke = cfg_get("NX2NAVI_LIB_PREBUILD_KEEP"); const char *te = cfg_get("NX2NAVI_LIB_PREBUILD_THREADS"); // no fixed default: 2 when unset (1 under 4 CPUs) @@ -4939,13 +5195,20 @@ static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { size_t total = gfx_shader_warm_list(&all), n = 0, n_cs = 0; LB.keep_n = ke && *ke ? atoi(ke) : 0; LB.e = malloc((total + 1) * sizeof *LB.e); + // the compute shaders whose pipeline stays come first: they are the ones the game is about to dispatch + uint8_t *kf = total ? malloc(total) : NULL; + for (size_t i = 0; LB.e && kf && i < total; i++) + if (all[i].stage == GFX_STAGE_CS && (kf[i] = (uint8_t)cu_keep(all[i].iid))) LB.e[n++] = all[i]; + LB.n_keep = n; for (size_t i = 0; LB.e && i < total; i++) - if (all[i].stage == GFX_STAGE_CS) LB.e[n++] = all[i]; + if (all[i].stage == GFX_STAGE_CS && !(kf && kf[i])) LB.e[n++] = all[i]; + free(kf); n_cs = n; for (size_t i = 0; LB.e && i < total && (int32_t)(n - n_cs) < LB.keep_n; i++) if (all[i].stage != GFX_STAGE_CS && all[i].prio == 0) LB.e[n++] = all[i]; free(all); if (atoi(e) > 1 && (size_t)atoi(e) < n) n = (size_t)atoi(e); // a count above 1 caps the list; 1 is plain on + if (LB.n_keep > n) LB.n_keep = n; if (!n) { free(LB.e), LB.e = NULL; return; @@ -4972,10 +5235,12 @@ static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { if (!LB.big_words) snprintf(big, sizeof big, "compute shaders of every size"); else snprintf(big, sizeof big, "compute shaders over %llu SPIR-V words %s", (unsigned long long)LB.big_words, P.cache_control ? "only from the pipeline caches" : "left to their first dispatch"); - rt_log("vk_pipeline: stage library prebuild: %zu compute pipelines and %zu recorded variants' libraries " - "(NX2NAVI_LIB_PREBUILD_KEEP %d) on %d idle-priority threads; library state clamp %u, polygon mode %u, depth " - "-w..w %u, samples %u; %s (NX2NAVI_LIB_PREBUILD_BIG)", n_cs < n ? n_cs : n, n > n_cs ? n - n_cs : 0, LB.keep_n, - started, LB.clamp, LB.poly, LB.minus_w, LB.samples, big); + rt_log("vk_pipeline: stage library prebuild: %zu compute pipelines, of which %zu stay (the shaders dispatched most " + "recently, NX2NAVI_COMPUTE_KEEP) and the others are built into the pipeline cache and destroyed, to be created " + "at their first dispatch; %zu recorded variants' libraries (NX2NAVI_LIB_PREBUILD_KEEP %d); on %d idle-priority " + "threads; library state clamp %u, polygon mode %u, depth -w..w %u, samples %u; %s (NX2NAVI_LIB_PREBUILD_BIG)", + n_cs < n ? n_cs : n, LB.n_keep, n > n_cs ? n - n_cs : 0, LB.keep_n, started, LB.clamp, LB.poly, LB.minus_w, + LB.samples, big); } static void lb_stop(void) { @@ -5121,6 +5386,91 @@ int vk_pipeline_warm_shaders(void) { return 0; } +// nx2navi.exe --probe-prewarm: the boot's shader and pipeline stages without the game, from the caches next to +// nx2navi.exe (pipeline-cache.bin, pipeline-keys.bin, shader-answers.bin, the shader cache): the pipeline cache load, +// the shader prewarm, the recorded pipelines and the compute pipeline prebuild, each followed by vk_mem_stage_log's +// line, so the log tells what every stage leaves in device memory and in the process's commit. Then up to +// PROBE_DISPATCHES compute shaders whose pipeline the boot did not keep are asked for as a dispatch would +// (vk_pipeline_compute), which gives what a first dispatch costs on this machine and how many came from the pipeline +// cache file. Fails when one of those pipelines is not created, when the prebuild kept a pipeline it should not have, +// or when the record of dispatched shaders, written to a file of the probe's own and read back, does not hold the +// probe's dispatches. compute-used.bin itself stays as it is; the pipeline cache is saved as after a boot. +#define PROBE_DISPATCHES 256 +int vk_pipeline_probe_prewarm(void) { + vk_pipeline_async_enable(); + CU.quiet = 1; + vk_pipeline_prewarm(); // brings the device up itself: it has to own the key file before ensure_init runs + if (!P.ready) { + printf("probe-prewarm: SKIP (no Vulkan device)\n"); + return 0; + } + while (__atomic_load_n(&LB.running, __ATOMIC_ACQUIRE) > 0) plat_sleep_ms(50); + if (!LB.n) { + printf("probe-prewarm: SKIP (no recorded compute shaders next to the executable: %s)\n", P.cache_path); + return 0; + } + VkPipeStats st; + vk_pipeline_stats(&st); + plat_mutex_lock_shared(&LB.stat); + uint64_t kept = LB.built[2], cache_only = LB.warmed[2], failed = LB.fail[2], there = LB.have[2]; + plat_mutex_unlock_shared(&LB.stat); + printf("probe-prewarm: %zu compute shaders, %zu to keep: %llu kept, %llu built into the pipeline cache only, %llu " + "there already, %llu failed; %llu graphics pipelines\n", LB.n, LB.n_keep, (unsigned long long)kept, + (unsigned long long)cache_only, (unsigned long long)there, (unsigned long long)failed, + (unsigned long long)st.graphics); + int fails = kept > LB.n_keep; // a dispatch never waits in this mode, so nothing else may have been kept + if (fails) printf("probe-prewarm: FAIL: %llu pipelines kept, %zu expected at most\n", (unsigned long long)kept, LB.n_keep); + uint32_t asked = 0, got = 0; + double t_all = 0, t_max = 0; + for (size_t i = LB.n_keep; i < LB.n && asked < PROBE_DISPATCHES; i++) { + const GfxShader *sh = LB.e[i].stage == GFX_STAGE_CS ? gfx_shader_load_input(LB.e[i].iid, 0) : NULL; + uint64_t words = 0; + if (!sh || gfx_shader_stage(sh) != GFX_STAGE_CS) continue; + gfx_shader_spirv(sh, &words); + if (LB.big_words && words > LB.big_words) continue; // those wait for the background builder + const VkPipeInfo *info = NULL; + double t0 = now_ms(); + VkPipeline p = vk_pipeline_compute(sh, &info); + double ms = now_ms() - t0; + asked++, got += p != VK_NULL_HANDLE; + t_all += ms; + if (ms > t_max) t_max = ms; + } + vk_mem_stage_log("the probe's first dispatches"); + plat_mutex_lock_shared(&P.lock); + uint64_t hit = CF.hit; + plat_mutex_unlock_shared(&P.lock); + printf("probe-prewarm: %u first dispatches of pipelines the boot did not keep: %u created, avg %.2f ms, longest %.2f " + "ms, %llu read from the pipeline cache file\n", asked, got, asked ? t_all / asked : 0.0, t_max, + (unsigned long long)hit); + if (got != asked) fails++, printf("probe-prewarm: FAIL: %u pipelines were not created\n", asked - got); + if (CU.opened) { + char rp[sizeof CU.path + 8]; + snprintf(rp, sizeof rp, "%s.probe", CU.path); + uint64_t rsize = 0; + CuHeader rh = {0}; + uint32_t mine = 0; + int wrote = cu_write(rp); + uint8_t *rf = wrote ? plat_file_read(rp, &rsize) : NULL; + long rn = cu_parse(rf, rsize, &rh); + for (long i = 0; i < rn; i++) { + CuRec r; + memcpy(&r, rf + sizeof rh + (size_t)i * sizeof r, sizeof r); + mine += r.run == CU.run; + } + free(rf); + plat_file_delete(rp); + printf("probe-prewarm: the record of dispatched compute shaders: %ld entries, %u of this run (run %u)\n", rn, mine, + CU.run); + if (!wrote || rn < 0 || rh.run != CU.run || mine != got) + fails++, printf("probe-prewarm: FAIL: the record does not hold the %u shaders dispatched here\n", got); + } + lb_stop(); + vk_pipeline_save(); + printf("probe-prewarm: %s\n", fails ? "FAILED" : "passed"); + return fails ? 1 : 0; +} + // ---- dynamic state void vk_pipeline_set_dynamic(VkCommandBuffer cb, const VkPipeState *s) { @@ -5295,7 +5645,7 @@ static int cache_save(uint32_t wait_ms) { #define SAVE_WAIT_MS 5000 static void save_with(uint32_t wait_ms) { - if (P.ready) keys_flush(), pb_flush(); + if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log(); if (P.ready && P.gpl) { rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws " "lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued " @@ -5341,6 +5691,7 @@ static void autosave_once(void) { last_keys = now; if (*(volatile uint32_t *)&K.pend_n) keys_flush(); pb_flush(); + cu_flush(); } gfx_shader_record_flush(0); if (now - last_ms < 20000) return;