diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 5645d008..6ed2e631 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -66,6 +66,10 @@ # Which draws are dropped instead of waiting for their pipeline: strict (only draws into per-frame passes), any (also # draws into passes whose target the previous frame rendered) or fullscreen (those and full-screen draws). # pipe_skip = fullscreen +# How long (ms) a draw into a target the previous frame did not render waits for a pipeline built for the first time, +# then is skipped. The default waits, as such a target (a lookup table, a new area's target) can stay empty if +# skipped; a lower value caps the stall. +# oneshot_wait_ms = 30000 # Build shader stage libraries in the background ahead of time, and the threads for it (1 to 16; default: a quarter of # the CPU threads). # lib_prebuild = 1 @@ -83,6 +87,12 @@ # pipe_feedback = 1 # pipeline-cache.bin larger than this (MB) starts empty and is rewritten (0: no limit). # pipeline_cache_max_mb = 512 +# 1: where the driver has VK_KHR_pipeline_binary, compiled pipelines are kept in pipeline-binaries.bin under the +# runtime's own keys, so a renamed or updated exe starts without compiling them again (pipeline-cache.bin is then +# not used). Default 0 (pipeline-cache.bin only) while it is being tested. +# pipe_binaries = 0 +# pipeline-binaries.bin larger than this (MB) starts empty and is rewritten (0: no limit). +# pipe_binaries_max_mb = 1024 # Build known pipelines ahead of time. # pipe_prebuild = 1 # 1: the start waits until every known pipeline is built. Default 0: with the graphics pipeline library the start diff --git a/runtime/cfg.c b/runtime/cfg.c index 3bf97fd2..b4c933a5 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -31,7 +31,7 @@ static int save_ok = 1; // the last cfg_save_to wrote (o // Every name the runtime reads (and the patch sets in generated/), without the prefix. A file key outside this list is // logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line. static const char *const known[] = { - "CORES", "TM_DISPATCH", "NVN2_CS_TABLE", "SKIP_SHADERS", "MOVIE_DEBAND", "MOVIE_FPS", "MOVIE_SKIP", "MOVIE_OVERLAY", "MOVIES", "NPLN", "NVN2_PROGRAMS", "NVN_TRAP", "OCARINA_REALTIME", "OCARINA_STEP", "VI_HZ", "VI_VRR", "TIME_DUMP", "SHADER_DUMP", "SHOT_AFTER", "SHOT_DIR", "SHOT_EVERY", // Switch 2 port: movie skip (romfs.c), NVN2 proc traps (nvn_procs.c) + "CORES", "TM_DISPATCH", "NVN2_CS_TABLE", "SKIP_SHADERS", "MOVIE_DEBAND", "MOVIE_FPS", "MOVIE_SKIP", "MOVIE_OVERLAY", "MOVIES", "NPLN", "NVN2_PROGRAMS", "NVN_TRAP", "OCARINA_REALTIME", "ONESHOT_WAIT_MS", "OCARINA_STEP", "VI_HZ", "VI_VRR", "TIME_DUMP", "SHADER_DUMP", "SHOT_AFTER", "SHOT_DIR", "SHOT_EVERY", // Switch 2 port: movie skip (romfs.c), NVN2 proc traps (nvn_procs.c) "AB", "AMIIBO_DIR", "ANISO", "APP_POOL", "ASPECT", "ASPECT_PATCH", "ASSUME_FOCUS", "ASTC", "ASTC_BC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_CHANNELS", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", "BIND_LSTICK_DOWN", "BIND_LSTICK_LEFT", "BIND_LSTICK_RIGHT", "BIND_LSTICK_UP", "BIND_MINUS", @@ -50,7 +50,7 @@ static const char *const known[] = { "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", - "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC", + "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", @@ -197,7 +197,7 @@ static const CfgMeta metas[] = { {"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART}, {"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART}, {"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART}, - {"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART}, + {"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, {"PIPE_BINARIES", "0", "0|1", RESTART}, {"ONESHOT_WAIT_MS", "30000", "0..60000", RESTART}, {"PIPE_BINARIES_MAX_MB", "1024", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART}, {"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART}, // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, diff --git a/runtime/vk.h b/runtime/vk.h index ee749e80..498728b5 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -39,6 +39,7 @@ typedef struct VkCtx { has_hdr_metadata, // VK_EXT_hdr_metadata (vk_hdr.c vk_hdr_metadata) has_mesh_shader, has_vrs, // VK_EXT_mesh_shader with taskShader and meshShader (NVN2 task/mesh stages) has_maximal_reconvergence, // VK_KHR_shader_maximal_reconvergence (vk_pipeline.c spirv_reconverge) + has_pipeline_binary, // VK_KHR_pipeline_binary with VK_KHR_maintenance5 (vk_pipeline.c pipe_create) has_mesh_queries; // meshShaderQueries, kept while NX2NAVI_GPU_PASSES or NX2NAVI_DUMP is set // Feedback loops (maxwell.md 1.9, vk_exec_draw.c feedback_prepare): a draw samples a target of its open pass that // an earlier draw of the pass wrote. NX2NAVI_FEEDBACK=0: VK_FEEDBACK_OFF (sampled in GENERAL without a barrier, the diff --git a/runtime/vk_device.c b/runtime/vk_device.c index a039f0c1..e1398278 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -238,6 +238,8 @@ typedef struct { VkPhysicalDeviceMeshShaderFeaturesEXT mesh; VkPhysicalDeviceFragmentShadingRateFeaturesKHR vrs; VkPhysicalDeviceShaderMaximalReconvergenceFeaturesKHR mrc; + VkPhysicalDeviceMaintenance5FeaturesKHR m5; + VkPhysicalDevicePipelineBinaryFeaturesKHR pbin; } FeatureChain; static int create_device(void) { @@ -280,6 +282,17 @@ static int create_device(void) { // (vk_pipeline.c spirv_reconverge) OPTIONAL_EXT(has_maximal_reconvergence, VK_KHR_SHADER_MAXIMAL_RECONVERGENCE_EXTENSION_NAME, mrc, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MAXIMAL_RECONVERGENCE_FEATURES_KHR); + // Compiled pipelines kept under the runtime's own keys (vk_pipeline.c pipe_create): VK_KHR_pipeline_binary, which + // needs VK_KHR_maintenance5 (core only from Vulkan 1.4; the instance asks for 1.3). NX2NAVI_PIPE_BINARIES=0 leaves + // both off. + if (cfg_bool("NX2NAVI_PIPE_BINARIES") && has_ext(ep, ne, VK_KHR_PIPELINE_BINARY_EXTENSION_NAME) && + has_ext(ep, ne, VK_KHR_MAINTENANCE_5_EXTENSION_NAME)) { + g_vk.has_pipeline_binary = 1; + ext[n_ext++] = VK_KHR_MAINTENANCE_5_EXTENSION_NAME; + ext[n_ext++] = VK_KHR_PIPELINE_BINARY_EXTENSION_NAME; + CHAIN(m5, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_FEATURES_KHR); + CHAIN(pbin, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_BINARY_FEATURES_KHR); + } // Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering int has_fbl = 0, has_lr = 0; if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) && @@ -377,6 +390,7 @@ static int create_device(void) { rt_log("vk: mesh shaders: %s", g_vk.has_mesh_shader ? "VK_EXT_mesh_shader" : "not supported (task/mesh draws are skipped)"); } // vkGetDeviceFaultInfoEXT needs the feature, not just the extension if (g_vk.has_maximal_reconvergence && !c->mrc.shaderMaximalReconvergence) g_vk.has_maximal_reconvergence = 0; + if (g_vk.has_pipeline_binary && !(c->pbin.pipelineBinaries && c->m5.maintenance5)) g_vk.has_pipeline_binary = 0; g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 4cfd8183..dfada1d2 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -43,7 +43,21 @@ #define NVN_STAGE_CS 5 #define NVN_STAGE_TASK 6 // NVN2 #define NVN_STAGE_MESH 7 -#define ONESHOT_WAIT_MS 30000u // prepare_draw: how long a one-shot draw waits for its pipeline before it is skipped +// prepare_draw: how long a one-shot draw waits for its pipeline before it is skipped (NX2NAVI_ONESHOT_WAIT_MS, default +// 30000). Such a draw renders into a target the previous frame did not (a volume, a LUT, a target made new at an area +// change); skipped, the target can stay empty for the session, so the default waits. A lower value trades that risk for +// a frame that never stalls longer than the value on a first-time pipeline. +static uint32_t oneshot_wait_ms(void) { + static int64_t ms = -1; + if (ms < 0) { + ms = strtoll(cfg_value("NX2NAVI_ONESHOT_WAIT_MS"), NULL, 10); + if (ms < 0) ms = 0; + if (ms != 30000) rt_log("vk_exec: a one-shot draw waits at most %lld ms for its pipeline, then is skipped " + "(NX2NAVI_ONESHOT_WAIT_MS)", (long long)ms); + } + return (uint32_t)ms; +} +#define ONESHOT_WAIT_MS oneshot_wait_ms() enum { W_PROGRAM, W_SHADER, W_PIPELINE, W_NO_VS, W_INDEX, W_UB, W_BIND, W_SSBO_ALIGN, W_SSBO_SLOT, W_RESTART, W_INDIRECT, W_MESH, W_COUNT }; diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index c818fb37..be7974f0 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -164,9 +164,401 @@ static void cache_path_init(void) { snprintf(P.cache_path, sizeof P.cache_path, "%s/pipeline-cache.bin", exe); } +// ---- pipeline binaries (VK_KHR_pipeline_binary; NX2NAVI_PIPE_BINARIES, default 0 while it is being tested; 1 on) +// pipeline-cache.bin does not carry compiled pipelines from one executable to another on NVIDIA 621.01: two +// byte-identical exes under different names, every compute module hash-identical, and the second compiled 317 of the +// 323 compute pipelines the first had read from the file (2026-11-09). A pipeline binary made from one exe's pipeline +// creates the same pipeline in another in 0.5 ms instead of 26 ms. So where the driver has the extension, every +// pipeline, stage library and compute pipeline is created through pipe_create: its driver pipeline key +// (vkGetPipelineKeyKHR, computed from the create info) is looked up in pipeline-binaries.bin next to nx2navi.exe and +// the pipeline is made from the stored binaries; a key not there is compiled with its data captured, and its binaries +// are appended to the file (autosave, every 2 s). pipeline-cache.bin is then neither read nor written: the driver's +// own disk cache serves an unchanged exe, and the binaries serve a renamed or updated one. +// Each binary is stored once under its own binary key: on NVIDIA 621.01 nearly every graphics pipeline has, besides +// its own binaries of 20-200 KB, one 2 MiB binary that is byte-identical across pipelines (2,506 copies in a 4.7 GB +// first-version file). +// File: header {"NX2NPBIN", version, the driver's global key size and bytes} (another global key: another driver or +// GPU, so the file starts empty); then records {u32 bytes after this field, u32 type, u32 key size, 32 key bytes, +// payload}: type PB_BLOB, a binary under its binary key, payload its data; type PB_PIPE, a pipeline under its pipeline +// key, payload {u32 count, count x {u32 key size, 32 key bytes}} naming its binaries. Only an index of the records +// stays in memory; a pipeline's records are read from the file when it is created. NX2NAVI_PIPE_BINARIES_MAX_MB +// (default 1024, 0 no limit): a larger file starts empty. +#define PB_VERSION 2 +// 64-bit file offsets: long is 32 bits on Windows, and a file over 2 GB read as cut short. +#ifdef _WIN32 +#define pb_seek _fseeki64 +#define pb_tell _ftelli64 +#else +#define pb_seek fseeko +#define pb_tell ftello +#endif +#define PB_MAX_BIN 8 +#define PB_KEY VK_MAX_PIPELINE_BINARY_KEY_SIZE_KHR +enum { PB_BLOB = 1, PB_PIPE = 2 }; +#define PB_HEAD (4 + 4 + PB_KEY) // a record's bytes after its size field and before its payload +typedef struct { char magic[8]; uint32_t version, gkey_size; uint8_t gkey[PB_KEY]; } PbHeader; +typedef struct { uint64_t h, off; uint32_t size; } PbSlot; // off: file offset of the record's size field +static struct { + int on; + PFN_vkCreatePipelineBinariesKHR create; + PFN_vkDestroyPipelineBinaryKHR destroy; + PFN_vkGetPipelineKeyKHR key; + PFN_vkGetPipelineBinaryDataKHR data; + PFN_vkReleaseCapturedPipelineDataKHR release; + char path[PLAT_PATH_MAX * 2 + 32]; + PlatMutex lock; // the index, the pending records and rf + FILE *rf; // reads records + PbSlot *slot; + uint32_t cap, n, pipes, blobs; + uint8_t *pend; // records not written yet; they follow the file's end in this order + size_t pend_n, pend_cap; + uint64_t file_size; // bytes on disk + int no_capture[3]; // kind (pipe_create) whose create refused the capture flag + uint64_t from_file, load_fail, compiled, captured, capture_fail, blob_new, blob_shared; + double from_ms, compile_ms; +} PB = {.lock = PLAT_MUTEX_INIT}; +static __thread int pb_last; // 1: the last pipe_create on this thread made its pipeline from binaries + +static uint64_t pb_hash(uint32_t type, const uint8_t *key, uint32_t n) { + uint64_t h = fnv(fnv(FNV0, &type, 4), key, n); + return h ? h : 1; +} + +static PbSlot *pb_find(uint64_t h) { // caller holds PB.lock + if (!PB.cap) return NULL; + for (uint32_t j = (uint32_t)h & (PB.cap - 1);; j = (j + 1) & (PB.cap - 1)) { + if (!PB.slot[j].h) return NULL; + if (PB.slot[j].h == h) return &PB.slot[j]; + } +} + +// Enters a record (a hash match is taken as the key: 64 bits). 0 when one with the key is there already or out of +// memory. Caller holds PB.lock. +static int pb_insert(uint64_t h, uint64_t off, uint32_t size) { + if ((PB.n + 1) * 2 > PB.cap) { + uint32_t cap = PB.cap ? PB.cap * 2 : 4096; + PbSlot *t = calloc(cap, sizeof *t); + if (!t) return 0; + for (uint32_t i = 0; i < PB.cap; i++) { + if (!PB.slot[i].h) continue; + uint32_t j = (uint32_t)PB.slot[i].h & (cap - 1); + while (t[j].h) j = (j + 1) & (cap - 1); + t[j] = PB.slot[i]; + } + free(PB.slot); + PB.slot = t, PB.cap = cap; + } + if (pb_find(h)) return 0; + uint32_t j = (uint32_t)h & (PB.cap - 1); + while (PB.slot[j].h) j = (j + 1) & (PB.cap - 1); + PB.slot[j] = (PbSlot){h, off, size}; + PB.n++; + return 1; +} + +static void pb_rewrite(const PbHeader *hd) { // an empty file with this driver's header + FILE *f = fopen(PB.path, "wb"); + if (f) fwrite(hd, sizeof *hd, 1, f), fclose(f); + PB.file_size = f ? sizeof *hd : 0; +} + +// Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache). +static void pb_init(VkCtx *c) { + if (!c->has_pipeline_binary || !P.cache_path[0]) return; +#define PBPROC(field, name) (PB.field = (void *)vk_dpa(c->device, name)) + if (!PBPROC(create, "vkCreatePipelineBinariesKHR") || !PBPROC(destroy, "vkDestroyPipelineBinaryKHR") || + !PBPROC(key, "vkGetPipelineKeyKHR") || !PBPROC(data, "vkGetPipelineBinaryDataKHR") || + !PBPROC(release, "vkReleaseCapturedPipelineDataKHR")) { + rt_log("vk_pipeline: pipeline binaries: the driver lacks a VK_KHR_pipeline_binary function; using pipeline-cache.bin"); + return; + } +#undef PBPROC + PbHeader want = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, PB_VERSION, 0, {0}}; + VkPipelineBinaryKeyKHR gk = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR}; + if (PB.key(c->device, NULL, &gk) != VK_SUCCESS || !gk.keySize || gk.keySize > PB_KEY) { + rt_log("vk_pipeline: pipeline binaries: no global key from the driver; using pipeline-cache.bin"); + return; + } + want.gkey_size = gk.keySize; + memcpy(want.gkey, gk.key, gk.keySize); + snprintf(PB.path, sizeof PB.path, "%s", P.cache_path); + char *cut = plat_last_sep(PB.path); + snprintf(cut ? cut + 1 : PB.path, sizeof PB.path - (size_t)(cut ? cut + 1 - PB.path : 0), "pipeline-binaries.bin"); + uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPE_BINARIES_MAX_MB"), NULL, 10) << 20; + FILE *f = fopen(PB.path, "rb"); + PbHeader hd; + const char *why = NULL; + uint64_t good = 0, len = 0; + if (f) { + pb_seek(f, 0, SEEK_END); + len = (uint64_t)pb_tell(f); + pb_seek(f, 0, SEEK_SET); + if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version"; + else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB"; + else { + good = sizeof hd; + for (;;) { // index: per record its type, key and offset + uint32_t head[3]; + uint8_t key[PB_KEY]; + if (fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || head[0] < PB_HEAD || + (head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || good + 4 + head[0] > len) + break; + if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) { + if (head[1] == PB_PIPE) PB.pipes++; + else PB.blobs++; + } + good += 4 + (uint64_t)head[0]; + if (pb_seek(f, (int64_t)good, SEEK_SET)) break; + } + if (good != len) why = "cut short (a record half written)"; + } + fclose(f); + } + if (why || !f) { // start empty: the index is dropped and the file rewritten + if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0); + free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0; + pb_rewrite(&want); + } else { + PB.file_size = len; + } + PB.rf = fopen(PB.path, "rb"); + PB.on = PB.rf != NULL && PB.file_size >= sizeof want; + rt_log("vk_pipeline: pipeline binaries %s: %s, %u pipelines, %u binaries, %.1f MB (NX2NAVI_PIPE_BINARIES=0 goes back " + "to pipeline-cache.bin)", PB.on ? "on" : "off (cannot open the file)", PB.path, PB.pipes, PB.blobs, + PB.file_size / 1048576.0); +} + +// The payload of the record of (type, key), malloc'd (*size bytes), or NULL. Caller holds PB.lock. +static uint8_t *pb_read(uint32_t type, const uint8_t *key, uint32_t ks, uint32_t *size) { + PbSlot *s = pb_find(pb_hash(type, key, ks)); + if (!s || s->size < PB_HEAD) return NULL; + uint32_t n = s->size - PB_HEAD; + uint8_t *head = malloc(PB_HEAD + (size_t)n), *out = NULL; + if (!head) return NULL; + int ok; + if (s->off >= PB.file_size) { // still pending, at its place in pend + size_t at = (size_t)(s->off - PB.file_size) + 4; + ok = at + s->size <= PB.pend_n; + if (ok) memcpy(head, PB.pend + at, s->size); + } else { + ok = !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1; + } + uint32_t t, k; + if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4); + if (ok && t == type && k == ks && !memcmp(head + 8, key, ks)) { // the key itself, not only its hash + memmove(head, head + PB_HEAD, n); + out = head, *size = n; + } else { + free(head); + } + return out; +} + +// The binaries stored for pipeline key k, created in bins (*n of them). 0 when none are stored or they fail. +static int pb_load(const VkPipelineBinaryKeyKHR *k, VkPipelineBinaryKHR bins[PB_MAX_BIN], uint32_t *n) { + VkPipelineBinaryKeyKHR keys[PB_MAX_BIN]; + VkPipelineBinaryDataKHR data[PB_MAX_BIN]; + uint8_t *blob[PB_MAX_BIN] = {0}; + uint32_t nb = 0, size = 0; + plat_mutex_lock(&PB.lock); + uint8_t *pipe = pb_read(PB_PIPE, k->key, k->keySize, &size); + int ok = pipe && size >= 4; + if (ok) memcpy(&nb, pipe, 4); + if (!nb || nb > PB_MAX_BIN || size < 4 + nb * (4 + PB_KEY)) ok = 0; + for (uint32_t i = 0; ok && i < nb; i++) { + const uint8_t *e = pipe + 4 + i * (4 + PB_KEY); + uint32_t ks, ds = 0; + memcpy(&ks, e, 4); + keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR, .keySize = ks}; + memcpy(keys[i].key, e + 4, PB_KEY); + if (ks > PB_KEY || !(blob[i] = pb_read(PB_BLOB, keys[i].key, ks, &ds))) ok = 0; + else data[i] = (VkPipelineBinaryDataKHR){.dataSize = ds, .pData = blob[i]}; + } + plat_mutex_unlock(&PB.lock); + if (ok) { + VkPipelineBinaryKeysAndDataKHR kd = {.binaryCount = nb, .pPipelineBinaryKeys = keys, .pPipelineBinaryData = data}; + VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pKeysAndDataInfo = &kd}; + VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR, + .pipelineBinaryCount = nb, .pPipelineBinaries = bins}; + ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS; + *n = nb; + } + for (uint32_t i = 0; i < PB_MAX_BIN; i++) free(blob[i]); + free(pipe); + return ok; +} + +// Appends one record to pend and the index. Caller holds PB.lock. +static int pb_put(uint32_t type, const uint8_t *key, uint32_t ks, const void *a, size_t an, const void *b, size_t bn) { + size_t size = PB_HEAD + an + bn; + if (size >= (1u << 31)) return 0; + if (PB.pend_n + 4 + size > PB.pend_cap) { + size_t cap = PB.pend_cap ? PB.pend_cap * 2 : (size_t)4 << 20; + while (cap < PB.pend_n + 4 + size) cap *= 2; + uint8_t *t = realloc(PB.pend, cap); + if (!t) return 0; + PB.pend = t, PB.pend_cap = cap; + } + if (!pb_insert(pb_hash(type, key, ks), PB.file_size + PB.pend_n, (uint32_t)size)) return 0; + uint8_t *o = PB.pend + PB.pend_n; + uint32_t s32 = (uint32_t)size; + memcpy(o, &s32, 4), memcpy(o + 4, &type, 4), memcpy(o + 8, &ks, 4); + memset(o + 12, 0, PB_KEY), memcpy(o + 12, key, ks); + if (an) memcpy(o + 4 + PB_HEAD, a, an); + if (bn) memcpy(o + 4 + PB_HEAD + an, b, bn); + PB.pend_n += 4 + size; + if (type == PB_PIPE) PB.pipes++; + else PB.blobs++; + return 1; +} + +static void pb_flush(void); + +// Stores the binaries of pipeline p (created with its data captured) under pipeline key k, then frees the data. A +// binary whose key is stored already is not stored again. +static void pb_capture(VkPipeline p, const VkPipelineBinaryKeyKHR *k) { + VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pipeline = p}; + VkPipelineBinaryKHR bins[PB_MAX_BIN]; + VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR}; + uint32_t nb = 0; + int ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS && hi.pipelineBinaryCount && + hi.pipelineBinaryCount <= PB_MAX_BIN; + if (ok) { + hi.pPipelineBinaries = bins; + ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS; + nb = ok ? hi.pipelineBinaryCount : 0; + } + VkPipelineBinaryKeyKHR keys[PB_MAX_BIN]; + uint8_t list[4 + PB_MAX_BIN * (4 + PB_KEY)] = {0}; + memcpy(list, &nb, 4); + for (uint32_t i = 0; ok && i < nb; i++) { // each binary's key, and its data unless stored already + VkPipelineBinaryDataInfoKHR di = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_DATA_INFO_KHR, .pipelineBinary = bins[i]}; + keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR}; + size_t ds = 0; + ok = PB.data(P.device, &di, &keys[i], &ds, NULL) == VK_SUCCESS && keys[i].keySize && keys[i].keySize <= PB_KEY; + if (!ok) break; + memcpy(list + 4 + i * (4 + PB_KEY), &keys[i].keySize, 4), memcpy(list + 8 + i * (4 + PB_KEY), keys[i].key, keys[i].keySize); + plat_mutex_lock(&PB.lock); + int have = pb_find(pb_hash(PB_BLOB, keys[i].key, keys[i].keySize)) != NULL; + plat_mutex_unlock(&PB.lock); + if (have) { + PB.blob_shared++; + continue; + } + uint8_t *d = malloc(ds ? ds : 1); + ok = d && PB.data(P.device, &di, &keys[i], &ds, d) == VK_SUCCESS; + if (ok) { + plat_mutex_lock(&PB.lock); + if (pb_put(PB_BLOB, keys[i].key, keys[i].keySize, d, ds, NULL, 0)) PB.blob_new++; + plat_mutex_unlock(&PB.lock); // a 0 from pb_put: another thread stored the same binary meanwhile + } + free(d); + } + for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL); + VkReleaseCapturedPipelineDataInfoKHR ri = {.sType = VK_STRUCTURE_TYPE_RELEASE_CAPTURED_PIPELINE_DATA_INFO_KHR, .pipeline = p}; + PB.release(P.device, &ri, NULL); + plat_mutex_lock(&PB.lock); + if (ok && pb_put(PB_PIPE, k->key, k->keySize, list, 4 + nb * (4 + PB_KEY), NULL, 0)) PB.captured++; + else if (!ok) PB.capture_fail++; + int full = PB.pend_n >= ((size_t)64 << 20); + plat_mutex_unlock(&PB.lock); + if (full) pb_flush(); // a cold boot captures thousands before the autosave thread runs: keep the buffer small +} + +// Appends the pending records (the autosave thread every 2 s, and the save at exit). +static void pb_flush(void) { + if (!PB.on) return; + plat_mutex_lock(&PB.lock); + if (PB.pend_n) { + FILE *f = fopen(PB.path, "ab"); + int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1; + if (f) ok = !fclose(f) && ok; + if (ok) PB.file_size += PB.pend_n; + else rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory", PB.path); + if (ok) PB.pend_n = 0; + } + plat_mutex_unlock(&PB.lock); +} + +static void pb_log(const char *when) { + if (!PB.on) return; + plat_mutex_lock(&PB.lock); + rt_log("vk_pipeline: pipeline binaries %s: %llu made from pipeline-binaries.bin (avg %.2f ms), %llu compiled (avg %.1f " + "ms), %llu captured, %llu could not be stored, %llu stored ones failed; binaries %llu new, %llu shared; %u pipelines and " + "%u binaries in the file, %.1f MB", when, + (unsigned long long)PB.from_file, PB.from_file ? PB.from_ms / PB.from_file : 0.0, + (unsigned long long)PB.compiled, PB.compiled ? PB.compile_ms / PB.compiled : 0.0, + (unsigned long long)PB.captured, (unsigned long long)PB.capture_fail, (unsigned long long)PB.load_fail, + (unsigned long long)PB.blob_new, (unsigned long long)PB.blob_shared, PB.pipes, PB.blobs, + (PB.file_size + PB.pend_n) / 1048576.0); + plat_mutex_unlock(&PB.lock); +} + +// Creates one pipeline: kind 0 a graphics pipeline, 1 a graphics pipeline library, 2 a compute pipeline; ci its +// Vk*PipelineCreateInfo. With binaries on: from the stored binaries of its pipeline key when there are any, else +// compiled with P.cache and its binaries captured. The create info is restored before returning. +typedef struct { VkStructureType sType; const void *pNext; VkPipelineCreateFlags flags; } PipeCiHead; +static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) { + PipeCiHead *hd = ci; + VkResult r; + pb_last = 0; +#define PIPE_CREATE(cache) (kind == 2 ? vkCreateComputePipelines(P.device, cache, 1, ci, NULL, pipe) \ + : vkCreateGraphicsPipelines(P.device, cache, 1, ci, NULL, pipe)) + if (!PB.on) return PIPE_CREATE(P.cache); + VkPipelineCreateInfoKHR kci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_INFO_KHR, .pNext = ci}; + VkPipelineBinaryKeyKHR key = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR}; + int keyed = PB.key(P.device, &kci, &key) == VK_SUCCESS && key.keySize && key.keySize <= PB_KEY; + const void *next = hd->pNext; + if (keyed) { + VkPipelineBinaryKHR bins[PB_MAX_BIN]; + uint32_t nb = 0; + uint64_t t0 = plat_now_ns(); + if (pb_load(&key, bins, &nb)) { + VkPipelineBinaryInfoKHR bi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_INFO_KHR, .pNext = next, + .binaryCount = nb, .pPipelineBinaries = bins}; + hd->pNext = &bi; + r = PIPE_CREATE(VK_NULL_HANDLE); // a pipeline made from binaries takes no pipeline cache + hd->pNext = next; + for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL); + plat_mutex_lock(&PB.lock); + if (r == VK_SUCCESS) PB.from_file++, PB.from_ms += (plat_now_ns() - t0) / 1e6; + else PB.load_fail++; + plat_mutex_unlock(&PB.lock); + if (r == VK_SUCCESS) return pb_last = 1, r; + } + } + int capture = keyed && !PB.no_capture[kind]; + VkPipelineCreateFlags2CreateInfoKHR f2 = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR, .pNext = next, + .flags = (VkPipelineCreateFlags2KHR)hd->flags | VK_PIPELINE_CREATE_2_CAPTURE_DATA_BIT_KHR}; + uint64_t t0 = plat_now_ns(); + if (capture) hd->pNext = &f2; + r = PIPE_CREATE(P.cache); + hd->pNext = next; + if (capture && r < 0) { // the driver refuses the capture for this kind: create without it from now on + PB.no_capture[kind] = 1; + rt_log("vk_pipeline: pipeline binaries: a %s with its data captured failed (%d); such pipelines are not stored", + kind == 2 ? "compute pipeline" : kind ? "pipeline library" : "graphics pipeline", (int)r); + capture = 0; + r = PIPE_CREATE(P.cache); + } +#undef PIPE_CREATE + if (r == VK_SUCCESS && *pipe) { + plat_mutex_lock(&PB.lock); + PB.compiled++, PB.compile_ms += (plat_now_ns() - t0) / 1e6; + plat_mutex_unlock(&PB.lock); + if (capture) pb_capture(*pipe, &key); + } + return r; +} + // Cache data whose header names another device or driver is dropped before Vulkan sees it. static void cache_load(VkCtx *c) { cache_path_init(); + pb_init(c); + if (PB.on) { // the binaries carry compiled pipelines across runs; P.cache stays VK_NULL_HANDLE + rt_log("vk_pipeline: %s is not read or written while pipeline binaries are on", P.cache_path); + return; + } void *data = NULL; size_t size = 0; FILE *f = P.cache_path[0] ? fopen(P.cache_path, "rb") : NULL; @@ -1442,7 +1834,7 @@ static const void *fb_chain(VkPipelineCreationFeedbackCreateInfo *ci, VkPipeline } static void fb_note(int kind, const VkPipelineCreationFeedback *fb) { - if (!P.feedback || !(fb->flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) return; + if (!P.feedback || !P.cache || !(fb->flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) return; int hit = (fb->flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0; double ms = (double)fb->duration / 1e6; plat_mutex_lock(&FB.lock); @@ -1454,7 +1846,8 @@ static void fb_note(int kind, const VkPipelineCreationFeedback *fb) { // One line: per kind, builds that came from pipeline-cache.bin of all builds so far, with the average time of each. static void fb_log(const char *when) { - if (!P.feedback) return; + pb_log(when); + if (!P.feedback || !P.cache) return; static const char *const kn[FB_KINDS] = {"pipelines", "libraries", "compute"}; char line[512]; int o = snprintf(line, sizeof line, "vk_pipeline: pipeline-cache.bin hits %s:", when); @@ -1718,7 +2111,7 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay pi.pNext = fb_chain(&fci, &fb, pi.pNext); uint64_t p0 = perf_begin(); crash_note("a graphics pipeline", s->vs, s->fs, s->ts); - VkResult r = vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe); + VkResult r = pipe_create(0, &pi, &pipe); plat_crash_context(NULL); perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it if (r != VK_SUCCESS) { @@ -1877,7 +2270,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi VkPipelineCreationFeedback fb; pi.pNext = fb_chain(&fci, &fb, pi.pNext); crash_note("a pipeline library", s->vs, s->fs, NULL); - if (vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + if (pipe_create(1, &pi, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); if (pipe && (kind == LIB_PR || kind == LIB_FS)) fb_note(FB_LIBRARY, &fb); // the shader libraries: the others compile nothing out: @@ -3098,13 +3491,13 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache VkPipelineCreationFeedback fb; ci.pNext = fb_chain(&fci, &fb, ci.pNext); crash_note("a compute pipeline", cs, NULL, NULL); - if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + if (pipe_create(2, &ci, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); if (pipe) fb_note(FB_COMPUTE, &fb); if (P.feedback >= 2 && (pipe || !cached_only)) rt_log("vk_pipeline: compute %016llx: %llu words, module %016llx, %s, %.1f ms", (unsigned long long)fnv(FNV0, code, words * 4), (unsigned long long)words, (unsigned long long)mm_hash, - !pipe ? "failed" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled", + !pipe ? "failed" : pb_last ? "from pipeline-binaries.bin" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled", (double)fb.duration / 1e6); vkDestroyShaderModule(P.device, m, NULL); free(over); @@ -4114,7 +4507,7 @@ void vk_pipeline_stats(VkPipeStats *s) { static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache void vk_pipeline_save(void) { - if (P.ready) keys_flush(); + if (P.ready) keys_flush(), pb_flush(); if (P.ready && P.gpl) { rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws " "lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued " @@ -4140,6 +4533,7 @@ void vk_pipeline_save(void) { (unsigned long long)P.g.pre_used[LIB_FS], (unsigned long long)LB.cs_draw); } gfx_shader_record_flush(1); + if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below if (!P.ready || !P.cache || !P.cache_path[0]) return; // Not under P.lock: the driver synchronises P.cache, and holding P.lock through the copy and the file write (about // 60 ms on the 1.4.3 fight) stopped every pipeline lookup of the replay, a freeze at each 20 s autosave. @@ -4170,7 +4564,11 @@ static void autosave_once(void) { static uint64_t last_n, last_ms, last_keys; if (!P.ready) return; uint64_t now = plat_now_ms(); - if (*(volatile uint32_t *)&K.pend_n && now - last_keys >= 2000) last_keys = now, keys_flush(); + if (now - last_keys >= 2000) { + last_keys = now; + if (*(volatile uint32_t *)&K.pend_n) keys_flush(); + pb_flush(); + } gfx_shader_record_flush(0); if (now - last_ms < 20000) return; VkPipeStats s;