// Graphics and compute pipelines and descriptor layouts of the Vulkan replay. The op-stream replay (vk_exec.c) // collects NVN state into a VkPipeState and asks for the pipeline of each draw; this file maps that state to Vulkan, // numbers the descriptor bindings and keeps the caches. // // Binding model. The translator (tools/xlate/Program.cs, a SPIR-V back end) decorates every resource of a // shader with DescriptorSet 0 and numbers the bindings of each shader from 1 in the order it meets them (Accessor // Create*Binding); binding 0 is its support buffer (ResourceManager: SupportBuffer.Binding, a std140 uniform // buffer every shader declares). The numbering is per shader, so the vertex and fragment stages of one program collide. // The scheme here: // - one descriptor set, set 0, per pipeline; no push constants (the translator's SPIR-V uses none); // - binding 0: the support buffer, one uniform buffer shared by all stages of the pipeline; // - vertex stage: its own bindings 1..n unchanged; fragment stage: its bindings b >= 1 move to b + n (the SPIR-V // Binding decorations are rewritten when the module is created); compute: its own bindings; // - descriptor types from the reflection: constant buffers UNIFORM_BUFFER, storage buffers STORAGE_BUFFER, textures // COMBINED_IMAGE_SAMPLER (UNIFORM_TEXEL_BUFFER for buffer textures, SAMPLED_IMAGE for separate ones), images // STORAGE_IMAGE (STORAGE_TEXEL_BUFFER for buffer images), array lengths as descriptor counts; // - set 0 is a push-descriptor set (VK_KHR_push_descriptor) when its descriptors fit maxPushDescriptors (32 on the RX // 9070; a pipeline layout may hold one push set only, which rules out a set per stage); larger ones (a vertex and a // fragment stage near the census maxima of 15 and 27) are ordinary sets the replay allocates (VkPipeInfo.push 0). // A fixed layout per stage combination (a binding per constant buffer, storage buffer, texture // and image slot per stage) would hold about 150 descriptors and could never be pushed, so layouts are made per distinct // binding list and shared by every pipeline with that list; VkPipeInfo.bindings tells the replay which NVN slot feeds // each binding. // // Fragment inputs the vertex stage does not write (stages are translated separately) get // outputs added to the vertex module, as Vulkan rejects such pairs, initialised to (0, 0, 0, 1) as Maxwell reads an // unwritten output (docs/maxwell.md 1.2). The unwritten components of an output the vertex stage does write // start at (0, 0, 0, 1) in the translator (tools/xlate/patches/0006). // // Pipeline key. Everything Vulkan 1.3 can take as dynamic state is dynamic: viewports and scissors with count, depth // bias, blend constants, stencil masks and references, cull mode, front face, topology (within its class), depth and // stencil tests and ops, depth bias enable, rasterizer discard; vertex input (VK_EXT_vertex_input_dynamic_state, else // binding strides only); and, where VK_EXT_extended_dynamic_state3 offers them, depth clamp, polygon mode, blend enables, // equations, write masks and alpha to coverage. The key holds the rest: shaders, target formats, sample count, topology // class, depth clip mode, logic op and what is not dynamic on the GPU. The VkPipelineCache persists to // pipeline-cache.bin next to nx2navi.exe (NX2NAVI_PIPELINE_CACHE overrides the path). // // The vertex format table has a 3-component fallback. The pipeline key holds the program, target formats, blend and // write masks, vertex layout and sample count. The binding conventions are those of the shader translator. // Contains code Copyright (c) 2026 Peiikai, Copyright (c) 2026 jhaizhou (MIT; see THIRD_PARTY_NOTICES.md). #include #include #include #include "cfg.h" #include "gfx.h" #include "host.h" #include "nvn_hmap.h" #include "platform/platform.h" #include "tex.h" #include "vk_gpa.h" #include "vk_warm_hint.h" #define LOG_ONCE(...) do { static volatile int32_t once_; if (!plat_xchg32(&once_, 1)) rt_log(__VA_ARGS__); } while (0) enum { SPV_OP_ENTRY_POINT = 15, SPV_OP_TYPE_INT = 21, SPV_OP_TYPE_FLOAT = 22, SPV_OP_TYPE_VECTOR = 23, SPV_OP_TYPE_POINTER = 32, SPV_OP_CONSTANT = 43, SPV_OP_CONSTANT_COMPOSITE = 44, SPV_OP_CONSTANT_NULL = 46, SPV_OP_FUNCTION = 54, SPV_OP_VARIABLE = 59, SPV_OP_DECORATE = 71, SPV_DEC_BUILTIN = 11, SPV_DEC_LOCATION = 30, SPV_DEC_INDEX = 32, SPV_DEC_BINDING = 33, SPV_DEC_DESCRIPTOR_SET = 34, SPV_SC_OUTPUT = 3 }; #define MAX_BINDINGS 128 #define PIPE_BUCKETS 4096 #define LAYOUT_BUCKETS 256 // ---- device capabilities and the pipeline cache (set up on first use, under P.lock) enum { NP_16 = 1, NP_32 = 2, NP_64 = 4 }; // float widths in P.nan_preserve and spirv_nan_preserve static struct { PlatMutex lock; volatile int32_t ready; VkDevice device; VkPipelineCache cache; uint32_t max_push; int indep; // stage-independent layouts (vk_caps.gpl): vertex set 0, fragment set 1 int push_stage; // GFX_STAGE_* whose set is pushed with independent layouts int gpl; // draws whose pipeline is building get a fast-linked one (vk_caps.gpl) struct { uint64_t links, fb_links, wlinks, flinks, dlinks, pr_added, lib_slow[4], ready_n[6], ready_nofast, hints, hint_libs, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double pr_added_ms, lib_slow_ms[4], ready_ms, ready_max, link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g; int vs_outs_all; // NX2NAVI_VS_OUTS: 0 union of paired fragment inputs, 1 (all) all 32 output locations int dyn_vi, dyn_clamp, dyn_poly, dyn_blend_en, dyn_blend_eq, dyn_wmask, dyn_a2c; int f_logic_op, f_depth_clamp, f_fill_non_solid, f_dual_src, f_full_subgroups; int cache_control; // pipelineCreationCacheControl: a create can ask for the pipeline caches only int feedback; // NX2NAVI_PIPE_FEEDBACK: 1 count pipeline cache hits, 2 also log each compute and graphics build uint64_t cache_loaded; // bytes of pipeline-cache.bin the driver took at boot int pv_last; // provokingVertexLast enabled: NVN's default is the last vertex VkShaderStageFlags sg32_stages; // stages that accept requiredSubgroupSize = 32 (0: not available) int reconverge; // modules with subgroup ops get MaximallyReconvergesKHR (spirv_reconverge) uint32_t nan_preserve; // float widths that get SignedZeroInfNanPreserve (spirv_nan_preserve): NP_16|NP_32|NP_64 int nan_bits; // NaN tests as integer bit tests (spirv_nan_bits) volatile uint32_t nan_bits_tests, nan_bits_selects, nan_bits_modules; // rewritten so far (the first log line) VkDynamicState dyn[32]; uint32_t n_dyn; PFN_vkCmdPushDescriptorSetKHR push; int tpl; // descriptor update templates on (make_tpl) PFN_vkCreateDescriptorUpdateTemplate tpl_create; PFN_vkDestroyDescriptorUpdateTemplate tpl_destroy; PFN_vkUpdateDescriptorSetWithTemplate tpl_update; PFN_vkCmdPushDescriptorSetWithTemplateKHR tpl_push; PFN_vkCmdSetVertexInputEXT set_vi; PFN_vkCmdSetDepthClampEnableEXT set_clamp; PFN_vkCmdSetPolygonModeEXT set_poly; PFN_vkCmdSetColorBlendEnableEXT set_blend_en; PFN_vkCmdSetColorBlendEquationEXT set_blend_eq; PFN_vkCmdSetColorWriteMaskEXT set_wmask; PFN_vkCmdSetAlphaToCoverageEnableEXT set_a2c; char cache_path[PLAT_PATH_MAX * 2]; VkPipeStats stats; } P = {.lock = PLAT_MUTEX_INIT}; typedef struct Layout { struct Layout *next; uint64_t hash; VkDescriptorSetLayout set; VkPipelineLayout layout; int push; uint32_t n; VkDescriptorSetLayout set1; // independent layouts: the fragment set (set 1); VK_NULL_HANDLE otherwise int push1; // set 1 is a push-descriptor set uint32_t n0; // independent layouts: b[0..n0) are set 0, the rest set 1 VkPipelineLayout half[2]; // GPL libraries: {set 0, null}, {null, set 1} (made on first use) VkDescriptorUpdateTemplate tpl[2]; // per set (make_tpl); VK_NULL_HANDLE: written with write structs VkDescriptorSetLayoutBinding b[]; } Layout; typedef struct Pipe { struct Pipe *next; uint64_t hash; VkPipeInfo info; VkPipeBinding *bindings; int building; // a background job owns info.pipeline (under P.lock) struct Job *job; // that job while building const struct Layout *lay; // graphics: the layout it was entered with VkPipeline fast; // GPL: the fast-linked pipeline (kept until shutdown: recorded draws use it) uint8_t fast_tried; uint8_t link_queued; // a waiting draw queued its fast link to the workers (LibJob p), not yet run uint8_t used; // compute: a dispatch of this run asked for it (vk_pipeline_compute) uint8_t pre; // compute: built and kept by the boot prebuild (lb_compute) or bg_main uint64_t t_miss; // graphics: when a draw first missed it (ns), 0 once it is usable (ready_note) size_t key_size; uint8_t key[]; } Pipe; static Layout *layouts[LAYOUT_BUCKETS]; static Pipe *pipes[PIPE_BUCKETS]; static uint64_t fnv(uint64_t h, const void *p, size_t n) { const uint8_t *b = p; for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 0x100000001b3ull; return h; } #define FNV0 0xcbf29ce484222325ull static __thread uint64_t mm_hash; // FNV-1a of the module make_module last created on this thread (PIPE_FEEDBACK 2) // Hash of a pipeline table key, 8 bytes per step: a draw hashes its whole GKey, and byte-wise FNV over it cost more // than the lookup. Only the in-memory table uses it; the key file keeps fnv. static uint64_t key_hash(const void *k, size_t n) { return nvn_hm_hash(k, (uint32_t)n); } // The cache file lives next to nx2navi.exe, so every build dir (the main build and each private fork) has its own: // forks that shared build\v121's cache broke each other's runs (2026-10-22). static void cache_path_init(void) { const char *cfg_path = cfg_get("NX2NAVI_PIPELINE_CACHE"); if (cfg_path && *cfg_path && strlen(cfg_path) < sizeof P.cache_path) { strcpy(P.cache_path, cfg_path); return; } P.cache_path[0] = 0; char exe[PLAT_PATH_MAX * 2]; int n = plat_exe_path(exe, sizeof exe); if (!n || n >= sizeof exe) return; char *cut = plat_last_sep(exe); if (!cut) return; *cut = 0; snprintf(P.cache_path, sizeof P.cache_path, "%s/pipeline-cache.bin", exe); } // ---- pipeline binaries (VK_KHR_pipeline_binary; NX2NAVI_PIPE_BINARIES, default 0 while it is being tested; 1 on) // pipeline-cache.bin does not carry compiled pipelines from one executable to another on NVIDIA 621.01: two // byte-identical exes under different names, every compute module hash-identical, and the second compiled 317 of the // 323 compute pipelines the first had read from the file (2026-11-09). A pipeline binary made from one exe's pipeline // creates the same pipeline in another in 0.5 ms instead of 26 ms. So where the driver has the extension, every // pipeline, stage library and compute pipeline is created through pipe_create: its driver pipeline key // (vkGetPipelineKeyKHR, computed from the create info) is looked up in pipeline-binaries.bin next to nx2navi.exe and // the pipeline is made from the stored binaries; a key not there is compiled with its data captured, and its binaries // are appended to the file (autosave, every 2 s). pipeline-cache.bin is then neither read nor written: the driver's // own disk cache serves an unchanged exe, and the binaries serve a renamed or updated one. // Each binary is stored once under its own binary key: on NVIDIA 621.01 nearly every graphics pipeline has, besides // its own binaries of 20-200 KB, one 2 MiB binary that is byte-identical across pipelines (2,506 copies in a 4.7 GB // first-version file). // File: header {"NX2NPBIN", version, the driver's global key size and bytes} (another global key: another driver or // GPU, so the file starts empty); then records {u32 bytes after this field, u32 type, u32 key size, 32 key bytes, // payload}: type PB_BLOB, a binary under its binary key, payload its data; type PB_PIPE, a pipeline under its pipeline // key, payload {u32 count, count x {u32 key size, 32 key bytes}} naming its binaries. Only an index of the records // stays in memory; a pipeline's records are read from the file when it is created. NX2NAVI_PIPE_BINARIES_MAX_MB // (default 1024, 0 no limit): a larger file starts empty. #define PB_VERSION 2 // 64-bit file offsets: long is 32 bits on Windows, and a file over 2 GB read as cut short. #ifdef _WIN32 #define pb_seek _fseeki64 #define pb_tell _ftelli64 #else #define pb_seek fseeko #define pb_tell ftello #endif #define PB_MAX_BIN 8 #define PB_KEY VK_MAX_PIPELINE_BINARY_KEY_SIZE_KHR enum { PB_BLOB = 1, PB_PIPE = 2 }; #define PB_HEAD (4 + 4 + PB_KEY) // a record's bytes after its size field and before its payload typedef struct { char magic[8]; uint32_t version, gkey_size; uint8_t gkey[PB_KEY]; } PbHeader; typedef struct { uint64_t h, off; uint32_t size; } PbSlot; // off: file offset of the record's size field static struct { int on; PFN_vkCreatePipelineBinariesKHR create; PFN_vkDestroyPipelineBinaryKHR destroy; PFN_vkGetPipelineKeyKHR key; PFN_vkGetPipelineBinaryDataKHR data; PFN_vkReleaseCapturedPipelineDataKHR release; char path[PLAT_PATH_MAX * 2 + 32]; PlatMutex lock; // the index, the pending records and rf FILE *rf; // reads records PbSlot *slot; uint32_t cap, n, pipes, blobs; uint8_t *pend; // records not written yet; they follow the file's end in this order size_t pend_n, pend_cap; uint64_t file_size; // bytes on disk int no_capture[3]; // kind (pipe_create) whose create refused the capture flag uint64_t from_file, load_fail, compiled, captured, capture_fail, blob_new, blob_shared; double from_ms, compile_ms; } PB = {.lock = PLAT_MUTEX_INIT}; static __thread int pb_last; // 1: the last pipe_create on this thread made its pipeline from binaries static __thread int cc_hit; // compute_create's last pipeline on this thread: 1 the pipeline cache held it // (P.cache: read from pipeline-cache.bin, or built into it earlier in this // run) or it was made from pipeline binaries, 0 not, -1 the driver did not say static uint64_t pb_hash(uint32_t type, const uint8_t *key, uint32_t n) { uint64_t h = fnv(fnv(FNV0, &type, 4), key, n); return h ? h : 1; } static PbSlot *pb_find(uint64_t h) { // caller holds PB.lock if (!PB.cap) return NULL; for (uint32_t j = (uint32_t)h & (PB.cap - 1);; j = (j + 1) & (PB.cap - 1)) { if (!PB.slot[j].h) return NULL; if (PB.slot[j].h == h) return &PB.slot[j]; } } // Enters a record (a hash match is taken as the key: 64 bits). 0 when one with the key is there already or out of // memory. Caller holds PB.lock. static int pb_insert(uint64_t h, uint64_t off, uint32_t size) { if ((PB.n + 1) * 2 > PB.cap) { uint32_t cap = PB.cap ? PB.cap * 2 : 4096; PbSlot *t = calloc(cap, sizeof *t); if (!t) return 0; for (uint32_t i = 0; i < PB.cap; i++) { if (!PB.slot[i].h) continue; uint32_t j = (uint32_t)PB.slot[i].h & (cap - 1); while (t[j].h) j = (j + 1) & (cap - 1); t[j] = PB.slot[i]; } free(PB.slot); PB.slot = t, PB.cap = cap; } if (pb_find(h)) return 0; uint32_t j = (uint32_t)h & (PB.cap - 1); while (PB.slot[j].h) j = (j + 1) & (PB.cap - 1); PB.slot[j] = (PbSlot){h, off, size}; PB.n++; return 1; } static void pb_rewrite(const PbHeader *hd) { // an empty file with this driver's header FILE *f = fopen(PB.path, "wb"); if (f) fwrite(hd, sizeof *hd, 1, f), fclose(f); PB.file_size = f ? sizeof *hd : 0; } // Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache). static void pb_init(VkCtx *c) { if (!c->has_pipeline_binary || !P.cache_path[0]) return; #define PBPROC(field, name) (PB.field = (void *)vk_dpa(c->device, name)) if (!PBPROC(create, "vkCreatePipelineBinariesKHR") || !PBPROC(destroy, "vkDestroyPipelineBinaryKHR") || !PBPROC(key, "vkGetPipelineKeyKHR") || !PBPROC(data, "vkGetPipelineBinaryDataKHR") || !PBPROC(release, "vkReleaseCapturedPipelineDataKHR")) { rt_log("vk_pipeline: pipeline binaries: the driver lacks a VK_KHR_pipeline_binary function; using pipeline-cache.bin"); return; } #undef PBPROC PbHeader want = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, PB_VERSION, 0, {0}}; VkPipelineBinaryKeyKHR gk = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR}; if (PB.key(c->device, NULL, &gk) != VK_SUCCESS || !gk.keySize || gk.keySize > PB_KEY) { rt_log("vk_pipeline: pipeline binaries: no global key from the driver; using pipeline-cache.bin"); return; } want.gkey_size = gk.keySize; memcpy(want.gkey, gk.key, gk.keySize); snprintf(PB.path, sizeof PB.path, "%s", P.cache_path); char *cut = plat_last_sep(PB.path); snprintf(cut ? cut + 1 : PB.path, sizeof PB.path - (size_t)(cut ? cut + 1 - PB.path : 0), "pipeline-binaries.bin"); uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPE_BINARIES_MAX_MB"), NULL, 10) << 20; FILE *f = fopen(PB.path, "rb"); PbHeader hd; const char *why = NULL; uint64_t good = 0, len = 0; if (f) { pb_seek(f, 0, SEEK_END); len = (uint64_t)pb_tell(f); pb_seek(f, 0, SEEK_SET); if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version"; else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB"; else { good = sizeof hd; for (;;) { // index: per record its type, key and offset uint32_t head[3]; uint8_t key[PB_KEY]; if (fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || head[0] < PB_HEAD || (head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || good + 4 + head[0] > len) break; if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) { if (head[1] == PB_PIPE) PB.pipes++; else PB.blobs++; } good += 4 + (uint64_t)head[0]; if (pb_seek(f, (int64_t)good, SEEK_SET)) break; } if (good != len) why = "cut short (a record half written)"; } fclose(f); } if (why || !f) { // start empty: the index is dropped and the file rewritten if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0); free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0; pb_rewrite(&want); } else { PB.file_size = len; } PB.rf = fopen(PB.path, "rb"); PB.on = PB.rf != NULL && PB.file_size >= sizeof want; rt_log("vk_pipeline: pipeline binaries %s: %s, %u pipelines, %u binaries, %.1f MB (NX2NAVI_PIPE_BINARIES=0 goes back " "to pipeline-cache.bin)", PB.on ? "on" : "off (cannot open the file)", PB.path, PB.pipes, PB.blobs, PB.file_size / 1048576.0); } // The payload of the record of (type, key), malloc'd (*size bytes), or NULL. Caller holds PB.lock. static uint8_t *pb_read(uint32_t type, const uint8_t *key, uint32_t ks, uint32_t *size) { PbSlot *s = pb_find(pb_hash(type, key, ks)); if (!s || s->size < PB_HEAD) return NULL; uint32_t n = s->size - PB_HEAD; uint8_t *head = malloc(PB_HEAD + (size_t)n), *out = NULL; if (!head) return NULL; int ok; if (s->off >= PB.file_size) { // still pending, at its place in pend size_t at = (size_t)(s->off - PB.file_size) + 4; ok = at + s->size <= PB.pend_n; if (ok) memcpy(head, PB.pend + at, s->size); } else { ok = !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1; } uint32_t t, k; if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4); if (ok && t == type && k == ks && !memcmp(head + 8, key, ks)) { // the key itself, not only its hash memmove(head, head + PB_HEAD, n); out = head, *size = n; } else { free(head); } return out; } // The binaries stored for pipeline key k, created in bins (*n of them). 0 when none are stored or they fail. static int pb_load(const VkPipelineBinaryKeyKHR *k, VkPipelineBinaryKHR bins[PB_MAX_BIN], uint32_t *n) { VkPipelineBinaryKeyKHR keys[PB_MAX_BIN]; VkPipelineBinaryDataKHR data[PB_MAX_BIN]; uint8_t *blob[PB_MAX_BIN] = {0}; uint32_t nb = 0, size = 0; plat_mutex_lock(&PB.lock); uint8_t *pipe = pb_read(PB_PIPE, k->key, k->keySize, &size); int ok = pipe && size >= 4; if (ok) memcpy(&nb, pipe, 4); if (!nb || nb > PB_MAX_BIN || size < 4 + nb * (4 + PB_KEY)) ok = 0; for (uint32_t i = 0; ok && i < nb; i++) { const uint8_t *e = pipe + 4 + i * (4 + PB_KEY); uint32_t ks, ds = 0; memcpy(&ks, e, 4); keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR, .keySize = ks}; memcpy(keys[i].key, e + 4, PB_KEY); if (ks > PB_KEY || !(blob[i] = pb_read(PB_BLOB, keys[i].key, ks, &ds))) ok = 0; else data[i] = (VkPipelineBinaryDataKHR){.dataSize = ds, .pData = blob[i]}; } plat_mutex_unlock(&PB.lock); if (ok) { VkPipelineBinaryKeysAndDataKHR kd = {.binaryCount = nb, .pPipelineBinaryKeys = keys, .pPipelineBinaryData = data}; VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pKeysAndDataInfo = &kd}; VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR, .pipelineBinaryCount = nb, .pPipelineBinaries = bins}; ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS; *n = nb; } for (uint32_t i = 0; i < PB_MAX_BIN; i++) free(blob[i]); free(pipe); return ok; } // Appends one record to pend and the index. Caller holds PB.lock. static int pb_put(uint32_t type, const uint8_t *key, uint32_t ks, const void *a, size_t an, const void *b, size_t bn) { size_t size = PB_HEAD + an + bn; if (size >= (1u << 31)) return 0; if (PB.pend_n + 4 + size > PB.pend_cap) { size_t cap = PB.pend_cap ? PB.pend_cap * 2 : (size_t)4 << 20; while (cap < PB.pend_n + 4 + size) cap *= 2; uint8_t *t = realloc(PB.pend, cap); if (!t) return 0; PB.pend = t, PB.pend_cap = cap; } if (!pb_insert(pb_hash(type, key, ks), PB.file_size + PB.pend_n, (uint32_t)size)) return 0; uint8_t *o = PB.pend + PB.pend_n; uint32_t s32 = (uint32_t)size; memcpy(o, &s32, 4), memcpy(o + 4, &type, 4), memcpy(o + 8, &ks, 4); memset(o + 12, 0, PB_KEY), memcpy(o + 12, key, ks); if (an) memcpy(o + 4 + PB_HEAD, a, an); if (bn) memcpy(o + 4 + PB_HEAD + an, b, bn); PB.pend_n += 4 + size; if (type == PB_PIPE) PB.pipes++; else PB.blobs++; return 1; } static void pb_flush(void); // Stores the binaries of pipeline p (created with its data captured) under pipeline key k, then frees the data. A // binary whose key is stored already is not stored again. static void pb_capture(VkPipeline p, const VkPipelineBinaryKeyKHR *k) { VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pipeline = p}; VkPipelineBinaryKHR bins[PB_MAX_BIN]; VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR}; uint32_t nb = 0; int ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS && hi.pipelineBinaryCount && hi.pipelineBinaryCount <= PB_MAX_BIN; if (ok) { hi.pPipelineBinaries = bins; ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS; nb = ok ? hi.pipelineBinaryCount : 0; } VkPipelineBinaryKeyKHR keys[PB_MAX_BIN]; uint8_t list[4 + PB_MAX_BIN * (4 + PB_KEY)] = {0}; memcpy(list, &nb, 4); for (uint32_t i = 0; ok && i < nb; i++) { // each binary's key, and its data unless stored already VkPipelineBinaryDataInfoKHR di = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_DATA_INFO_KHR, .pipelineBinary = bins[i]}; keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR}; size_t ds = 0; ok = PB.data(P.device, &di, &keys[i], &ds, NULL) == VK_SUCCESS && keys[i].keySize && keys[i].keySize <= PB_KEY; if (!ok) break; memcpy(list + 4 + i * (4 + PB_KEY), &keys[i].keySize, 4), memcpy(list + 8 + i * (4 + PB_KEY), keys[i].key, keys[i].keySize); plat_mutex_lock(&PB.lock); int have = pb_find(pb_hash(PB_BLOB, keys[i].key, keys[i].keySize)) != NULL; plat_mutex_unlock(&PB.lock); if (have) { PB.blob_shared++; continue; } uint8_t *d = malloc(ds ? ds : 1); ok = d && PB.data(P.device, &di, &keys[i], &ds, d) == VK_SUCCESS; if (ok) { plat_mutex_lock(&PB.lock); if (pb_put(PB_BLOB, keys[i].key, keys[i].keySize, d, ds, NULL, 0)) PB.blob_new++; plat_mutex_unlock(&PB.lock); // a 0 from pb_put: another thread stored the same binary meanwhile } free(d); } for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL); VkReleaseCapturedPipelineDataInfoKHR ri = {.sType = VK_STRUCTURE_TYPE_RELEASE_CAPTURED_PIPELINE_DATA_INFO_KHR, .pipeline = p}; PB.release(P.device, &ri, NULL); plat_mutex_lock(&PB.lock); if (ok && pb_put(PB_PIPE, k->key, k->keySize, list, 4 + nb * (4 + PB_KEY), NULL, 0)) PB.captured++; else if (!ok) PB.capture_fail++; int full = PB.pend_n >= ((size_t)64 << 20); plat_mutex_unlock(&PB.lock); if (full) pb_flush(); // a cold boot captures thousands before the autosave thread runs: keep the buffer small } // Appends the pending records (the autosave thread every 2 s, and the save at exit). static void pb_flush(void) { if (!PB.on) return; plat_mutex_lock(&PB.lock); if (PB.pend_n) { FILE *f = fopen(PB.path, "ab"); int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1; if (f) ok = !fclose(f) && ok; if (ok) PB.file_size += PB.pend_n; else rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory", PB.path); if (ok) PB.pend_n = 0; } plat_mutex_unlock(&PB.lock); } static void pb_log(const char *when) { if (!PB.on) return; plat_mutex_lock(&PB.lock); rt_log("vk_pipeline: pipeline binaries %s: %llu made from pipeline-binaries.bin (avg %.2f ms), %llu compiled (avg %.1f " "ms), %llu captured, %llu could not be stored, %llu stored ones failed; binaries %llu new, %llu shared; %u pipelines and " "%u binaries in the file, %.1f MB", when, (unsigned long long)PB.from_file, PB.from_file ? PB.from_ms / PB.from_file : 0.0, (unsigned long long)PB.compiled, PB.compiled ? PB.compile_ms / PB.compiled : 0.0, (unsigned long long)PB.captured, (unsigned long long)PB.capture_fail, (unsigned long long)PB.load_fail, (unsigned long long)PB.blob_new, (unsigned long long)PB.blob_shared, PB.pipes, PB.blobs, (PB.file_size + PB.pend_n) / 1048576.0); plat_mutex_unlock(&PB.lock); } // Creates one pipeline: kind 0 a graphics pipeline, 1 a graphics pipeline library, 2 a compute pipeline; ci its // Vk*PipelineCreateInfo. With binaries on: from the stored binaries of its pipeline key when there are any, else // compiled with P.cache and its binaries captured. The create info is restored before returning. typedef struct { VkStructureType sType; const void *pNext; VkPipelineCreateFlags flags; } PipeCiHead; // Builds into P.cache against reads of its data. Every create with P.cache holds this shared (pipe_create_in); the // copy of the data in cache_save holds it exclusively and only takes it when no build is running. The Vulkan rules // allow reading a cache created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT while pipelines are // built into it, but Intel's Windows driver (101.8991 on an Arc 140V) crashed inside vkGetPipelineCacheData during the // copy, reading through a null entry, with compute and library builds running beside the autosave. static PlatMutex cache_rw = PLAT_MUTEX_INIT; // A pipeline cache of this thread's own that its creates build into in place of P.cache; VK_NULL_HANDLE (every thread // but the background builder of large compute shaders) for P.cache itself. A create into it does not hold cache_rw, so // a build of minutes does not keep pipeline-cache.bin from being saved; bg_merge moves its content into P.cache. static __thread VkPipelineCache own_cache; // New content in P.cache that P.stats does not count, for autosave_once to save: bg_merge's merges, and the compiles // of pipelines built for the cache only (lb_compute keep 0). static volatile uint32_t cache_merges; static VkResult pipe_create_in(int kind, void *ci, VkPipeline *pipe, VkPipelineCache cache) { int shared = cache && !own_cache; if (cache && own_cache) cache = own_cache; if (shared) plat_mutex_lock_shared(&cache_rw); VkResult r = kind == 2 ? vkCreateComputePipelines(P.device, cache, 1, ci, NULL, pipe) : vkCreateGraphicsPipelines(P.device, cache, 1, ci, NULL, pipe); if (shared) plat_mutex_unlock_shared(&cache_rw); return r; } static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) { PipeCiHead *hd = ci; VkResult r; pb_last = 0; #define PIPE_CREATE(cache) pipe_create_in(kind, ci, pipe, cache) if (!PB.on) return PIPE_CREATE(P.cache); VkPipelineCreateInfoKHR kci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_INFO_KHR, .pNext = ci}; VkPipelineBinaryKeyKHR key = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR}; int keyed = PB.key(P.device, &kci, &key) == VK_SUCCESS && key.keySize && key.keySize <= PB_KEY; const void *next = hd->pNext; if (keyed) { VkPipelineBinaryKHR bins[PB_MAX_BIN]; uint32_t nb = 0; uint64_t t0 = plat_now_ns(); if (pb_load(&key, bins, &nb)) { VkPipelineBinaryInfoKHR bi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_INFO_KHR, .pNext = next, .binaryCount = nb, .pPipelineBinaries = bins}; hd->pNext = &bi; r = PIPE_CREATE(VK_NULL_HANDLE); // a pipeline made from binaries takes no pipeline cache hd->pNext = next; for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL); plat_mutex_lock(&PB.lock); if (r == VK_SUCCESS) PB.from_file++, PB.from_ms += (plat_now_ns() - t0) / 1e6; else PB.load_fail++; plat_mutex_unlock(&PB.lock); if (r == VK_SUCCESS) return pb_last = 1, r; } } int capture = keyed && !PB.no_capture[kind]; VkPipelineCreateFlags2CreateInfoKHR f2 = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR, .pNext = next, .flags = (VkPipelineCreateFlags2KHR)hd->flags | VK_PIPELINE_CREATE_2_CAPTURE_DATA_BIT_KHR}; uint64_t t0 = plat_now_ns(); if (capture) hd->pNext = &f2; r = PIPE_CREATE(P.cache); hd->pNext = next; if (capture && r < 0) { // the driver refuses the capture for this kind: create without it from now on PB.no_capture[kind] = 1; rt_log("vk_pipeline: pipeline binaries: a %s with its data captured failed (%d); such pipelines are not stored", kind == 2 ? "compute pipeline" : kind ? "pipeline library" : "graphics pipeline", (int)r); capture = 0; r = PIPE_CREATE(P.cache); } #undef PIPE_CREATE if (r == VK_SUCCESS && *pipe) { plat_mutex_lock(&PB.lock); PB.compiled++, PB.compile_ms += (plat_now_ns() - t0) / 1e6; plat_mutex_unlock(&PB.lock); if (capture) pb_capture(*pipe, &key); } return r; } // Cache data whose header names another device or driver is dropped before Vulkan sees it. static void cache_load(VkCtx *c) { cache_path_init(); pb_init(c); if (PB.on) { // the binaries carry compiled pipelines across runs; P.cache stays VK_NULL_HANDLE rt_log("vk_pipeline: %s is not read or written while pipeline binaries are on", P.cache_path); return; } void *data = NULL; size_t size = 0; FILE *f = P.cache_path[0] ? fopen(P.cache_path, "rb") : NULL; if (f) { fseek(f, 0, SEEK_END); long len = ftell(f); fseek(f, 0, SEEK_SET); if (len > (long)sizeof(VkPipelineCacheHeaderVersionOne) && (data = malloc((size_t)len)) && fread(data, 1, (size_t)len, f) == (size_t)len) size = (size_t)len; fclose(f); VkPipelineCacheHeaderVersionOne h; if (size) memcpy(&h, data, sizeof h); if (size && (h.headerVersion != VK_PIPELINE_CACHE_HEADER_VERSION_ONE || h.vendorID != c->props.vendorID || h.deviceID != c->props.deviceID || memcmp(h.pipelineCacheUUID, c->props.pipelineCacheUUID, VK_UUID_SIZE))) { rt_log("vk_pipeline: %s was written by another device or driver; starting empty", P.cache_path); size = 0; } // NX2NAVI_PIPELINE_CACHE_MAX_MB (default 512; 0 no limit): a larger file starts the run empty, and the run's // saves replace it. The file only grows: entries of earlier translators and builds stay, and on NVIDIA // (621.01) an entry another executable wrote is neither used nor written again, so a renamed or updated exe // compiles while the file keeps its size (2026-11-09: 650 MB, 15-18 % of the pipelines read from it). The // whole file sits in process memory twice at the boot (this read and the driver's copy) and is copied out // at every save, which a 16 GB machine near its commit limit cannot spare. uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_MAX_MB"), NULL, 10) << 20; if (size && cap && size > cap) { rt_log("vk_pipeline: %s holds %.0f MB, over NX2NAVI_PIPELINE_CACHE_MAX_MB %llu: starting empty; this run " "rewrites it", P.cache_path, size / 1048576.0, (unsigned long long)(cap >> 20)); size = 0; } } VkPipelineCacheCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, .initialDataSize = size, .pInitialData = size ? data : NULL}; if (vkCreatePipelineCache(c->device, &ci, NULL, &P.cache) != VK_SUCCESS) { ci.initialDataSize = 0, ci.pInitialData = NULL; if (vkCreatePipelineCache(c->device, &ci, NULL, &P.cache) != VK_SUCCESS) P.cache = VK_NULL_HANDLE; } free(data); // What the driver kept of the file: a driver may drop initial data it does not accept without failing the create, // which leaves every build of the run compiling while the cache file looks valid. size_t kept = 0; if (P.cache && vkGetPipelineCacheData(c->device, P.cache, &kept, NULL) != VK_SUCCESS) kept = 0; P.cache_loaded = kept; if (size) rt_log("vk_pipeline: %s: %.1f MB read, the driver holds %.1f MB of it", P.cache_path, size / 1048576.0, kept / 1048576.0); } // The float widths spirv_nan_preserve declares on this device (P.nan_preserve), from NX2NAVI_VK_NAN_PRESERVE: 0 off, // 1 on every driver; unset, on every driver except NVIDIA's own, on which the game draws right without it. Only widths // the driver can preserve (VkPhysicalDeviceFloatControlsProperties) are declared, and nothing without 32-bit floats. static void nan_preserve_init(VkCtx *c) { VkPhysicalDeviceDriverProperties drv = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; VkPhysicalDeviceFloatControlsProperties fc = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES, .pNext = &drv}; VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &fc}; vkGetPhysicalDeviceProperties2(c->phys, &p2); const char *e = cfg_get("NX2NAVI_VK_NAN_PRESERVE"); // no fixed default: it depends on the driver, see above int set = e && *e, want = set ? strcmp(e, "0") != 0 : drv.driverID != VK_DRIVER_ID_NVIDIA_PROPRIETARY; uint32_t can = (fc.shaderSignedZeroInfNanPreserveFloat16 ? NP_16 : 0) | (fc.shaderSignedZeroInfNanPreserveFloat32 ? NP_32 : 0) | (fc.shaderSignedZeroInfNanPreserveFloat64 ? NP_64 : 0); P.nan_preserve = want && (can & NP_32) ? can : 0; // NX2NAVI_VK_NAN_BITS: 1 every driver, 0 none; unset: AMD's driver for RDNA 1 and 2 (spirv_nan_bits) const char *nb = cfg_get("NX2NAVI_VK_NAN_BITS"); P.nan_bits = nb && *nb ? strcmp(nb, "0") != 0 : vk_caps.amd_sc; if (P.nan_bits) rt_log("vk_pipeline: game shaders test for NaN on the bits of the number, not as a float (%s)", nb && *nb ? "NX2NAVI_VK_NAN_BITS=1" : "AMD's driver for RDNA 1 and 2"); if (P.nan_preserve) rt_log("vk_pipeline: game shaders keep NaN, infinity and signed zero (SignedZeroInfNanPreserve for %s32%s-bit " "floats%s)", can & NP_16 ? "16, " : "", can & NP_64 ? ", 64" : "", set ? ", NX2NAVI_VK_NAN_PRESERVE=1" : ""); else rt_log("vk_pipeline: game shaders without SignedZeroInfNanPreserve (%s)", !want ? (set ? "NX2NAVI_VK_NAN_PRESERVE=0" : "NVIDIA's driver") : "the driver cannot keep them in 32-bit floats"); } static void keys_open(void); static void cu_open(void); static void prebuild_start(void); static int ensure_init(void) { if (P.ready) return 1; if (!vk_init()) return 0; plat_mutex_lock(&P.lock); if (P.ready) { plat_mutex_unlock(&P.lock); return 1; } VkCtx *c = vk_ctx(); P.device = c->device; VkPhysicalDevicePushDescriptorPropertiesKHR pd = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR}; VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &pd}; vkGetPhysicalDeviceProperties2(c->phys, &p2); P.max_push = pd.maxPushDescriptors; P.indep = vk_caps.gpl; P.gpl = vk_caps.gpl; rt_log(P.gpl ? "vk_pipeline: graphics pipeline library on: a draw whose pipeline is building gets a fast-linked one, " "the optimised build replaces it when done" : "vk_pipeline: graphics pipeline library off: draws wait for or skip pipelines that are building"); const char *ps = cfg_get("NX2NAVI_PUSH_STAGE"), *vo = cfg_get("NX2NAVI_VS_OUTS"); P.push_stage = ps && !strcmp(ps, "fs") ? GFX_STAGE_FS : GFX_STAGE_VS; P.vs_outs_all = vo && !strcmp(vo, "all"); if (P.indep) rt_log("vk_pipeline: stage-independent layouts: vertex set 0, fragment set 1, %s set pushed; vertex outputs: %s", P.push_stage == GFX_STAGE_VS ? "vertex" : "fragment", P.vs_outs_all ? "all 32 locations" : "union of paired fragment inputs"); // vk_device.c enables every feature the GPU reports in this chain, so the query tells what is enabled. VkPhysicalDeviceExtendedDynamicState3FeaturesEXT e3 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_3_FEATURES_EXT}; VkPhysicalDeviceProvokingVertexFeaturesEXT pvf = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_FEATURES_EXT, .pNext = c->has_eds3 ? &e3 : NULL}; VkPhysicalDeviceVulkan13Features v13 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES, .pNext = c->has_provoking_vertex ? (void *)&pvf : (c->has_eds3 ? (void *)&e3 : NULL)}; VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &v13}; vkGetPhysicalDeviceFeatures2(c->phys, &f2); P.pv_last = c->has_provoking_vertex && pvf.provokingVertexLast; if (!P.pv_last) rt_log("vk_pipeline: provokingVertexLast unavailable: flat outputs come from the first vertex (NVN: last)"); // Maxwell warps have 32 lanes; the translator assumes it (maxwell.md 3.4). Pin 32 where the driver allows it. if (v13.subgroupSizeControl) { VkPhysicalDeviceVulkan13Properties p13 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES}; VkPhysicalDeviceProperties2 q2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &p13}; vkGetPhysicalDeviceProperties2(c->phys, &q2); P.sg32_stages = vk_caps.sg_pin_stages; // vk_caps.c: 32 unless NX2NAVI_VK_SUBGROUP; 0 without the pin rt_log("vk_pipeline: subgroup size %u-%u, requiredSubgroupSizeStages %#x, pinning %u on %#x", p13.minSubgroupSize, p13.maxSubgroupSize, (unsigned)p13.requiredSubgroupSizeStages, vk_caps.sg_size, (unsigned)P.sg32_stages); } // A warp reconverges where the guest compiler put it back together; Vulkan guarantees that only with maximal // reconvergence (spirv_reconverge). NX2NAVI_VK_RECONVERGE=0 leaves the modules as translated. P.reconverge = c->has_maximal_reconvergence && rt_knob("VK_RECONVERGE", "1") == 1; rt_log("vk_pipeline: maximal reconvergence for shaders with subgroup ops: %s", P.reconverge ? "on" : c->has_maximal_reconvergence ? "off (NX2NAVI_VK_RECONVERGE=0)" : "not supported by the driver"); nan_preserve_init(c); P.f_full_subgroups = v13.computeFullSubgroups; P.cache_control = v13.pipelineCreationCacheControl; // Pipeline creation feedback is core in Vulkan 1.3; the driver reports per pipeline whether P.cache held it. P.feedback = c->props.apiVersion >= VK_API_VERSION_1_3 ? atoi(cfg_value("NX2NAVI_PIPE_FEEDBACK")) : 0; P.f_logic_op = f2.features.logicOp; P.f_depth_clamp = f2.features.depthClamp; P.f_fill_non_solid = f2.features.fillModeNonSolid; P.f_dual_src = f2.features.dualSrcBlend; #define PROC(field, name) (P.field = (void *)vk_dpa(c->device, name)) PROC(push, "vkCmdPushDescriptorSetKHR"); // Descriptor update templates (core 1.1; the push variant comes with VK_KHR_push_descriptor on 1.1): the driver // walks a layout prepared when the layout is made instead of parsing write structs every draw. // NX2NAVI_DESC_TEMPLATE=0 keeps the write structs. if (!cfg_bool("NX2NAVI_DESC_TEMPLATE")) { rt_log("vk_pipeline: descriptor update templates off (NX2NAVI_DESC_TEMPLATE=0)"); } else if (c->props.apiVersion >= VK_API_VERSION_1_1 && PROC(tpl_create, "vkCreateDescriptorUpdateTemplate") && PROC(tpl_destroy, "vkDestroyDescriptorUpdateTemplate") && PROC(tpl_update, "vkUpdateDescriptorSetWithTemplate")) { P.tpl = 1; if (!PROC(tpl_push, "vkCmdPushDescriptorSetWithTemplateKHR")) rt_log("vk_pipeline: vkCmdPushDescriptorSetWithTemplateKHR missing: push sets are written with write structs"); } else { rt_log("vk_pipeline: descriptor update templates unavailable: sets are written with write structs"); } // NX2NAVI_VK_DYN_VERTEX=0/1 forces dynamic vertex input off or on. Default off on two drivers: // - Intel's frozen Windows driver for its older integrated GPUs (UHD 630, Ice Lake: Vulkan 1.3.215 and older), which // draws nothing with it. // - AMD's driver for RDNA 3 and newer (vk_caps.amd_llpc), which compiles a pipeline with dynamic vertex input again // at every boot: neither pipeline-cache.bin nor the driver's own disk cache gives it back. Measured on an // RX 7900 XT (26.10.1) with NX2NAVI_PIPE_FEEDBACK=2, booting one save twice from an empty cache: the second boot // compiled all 622 recorded non-mesh pipelines again at 83 ms each, with the module and state hashes of the first // boot (the same with NX2NAVI_GPL=0, so it is not the pipeline libraries or their layouts), while the mesh // pipelines, which have no vertex input, came from the file. With the vertex layout in the pipeline the second // boot built 787 pipelines in 170 ms and the third 898 of 898 from the file in 233 ms, with no build left for the // scene. A player's file of 3,979 records used the whole boot budget (PREWARM_PIPE_MS, 20 s) at every boot; with // the vertex layout in the pipeline its third boot built 4,273 pipelines in 0.9 s. The price is more pipelines, // one per vertex layout (787 records against 692 for that boot); records written with dynamic vertex input still // prebuild, each for the layout of the draw that recorded it. The RDNA 1 and 2 branch (vk_caps.amd_sc) is not // measured and keeps dynamic vertex input. const char *dv = cfg_get("NX2NAVI_VK_DYN_VERTEX"); // no fixed default: it depends on the GPU, see above int intel_legacy = c->props.vendorID == 0x8086 && c->props.apiVersion <= VK_MAKE_API_VERSION(0, 1, 3, 215); int dv_on = dv && *dv ? strcmp(dv, "0") != 0 : !intel_legacy && !vk_caps.amd_llpc; P.dyn_vi = dv_on && c->has_vertex_input_dynamic_state && PROC(set_vi, "vkCmdSetVertexInputEXT"); if (!dv_on && c->has_vertex_input_dynamic_state) rt_log("vk_pipeline: dynamic vertex input off (%s)", dv && *dv ? "NX2NAVI_VK_DYN_VERTEX=0" : intel_legacy ? "Intel's older Windows driver draws nothing with it" : "AMD's driver compiles pipelines that have it again at every boot"); if (c->has_eds3) { P.dyn_clamp = e3.extendedDynamicState3DepthClampEnable && P.f_depth_clamp && PROC(set_clamp, "vkCmdSetDepthClampEnableEXT"); P.dyn_poly = e3.extendedDynamicState3PolygonMode && PROC(set_poly, "vkCmdSetPolygonModeEXT"); P.dyn_blend_en = e3.extendedDynamicState3ColorBlendEnable && PROC(set_blend_en, "vkCmdSetColorBlendEnableEXT"); P.dyn_blend_eq = e3.extendedDynamicState3ColorBlendEquation && PROC(set_blend_eq, "vkCmdSetColorBlendEquationEXT"); P.dyn_wmask = e3.extendedDynamicState3ColorWriteMask && PROC(set_wmask, "vkCmdSetColorWriteMaskEXT"); P.dyn_a2c = e3.extendedDynamicState3AlphaToCoverageEnable && PROC(set_a2c, "vkCmdSetAlphaToCoverageEnableEXT"); } // NX2NAVI_VK_DYN_A2C=0/1 forces dynamic alpha to coverage off or on. Default off on AMD's Windows driver branch for // RDNA 2 and older (vk_caps.amd_sc), which applies alpha to coverage while the dynamic state says off: on an RX 590 // (24.9.1) NX2navi, which never turns it on, drew Link and the NPCs with depth but no colour (black silhouettes) and // dithered other surfaces into checkerboards (issue #67, RX 470/570/580). Taken out of the dynamic state, it goes // into the pipeline key and the characters draw. With the texture pool marks it stays out of the dynamic state // whatever the setting: a marked fragment shader gets early fragment tests only in a pipeline that has alpha to // coverage off (spirv_pool_mark), so the pipeline has to know. const char *da = cfg_get("NX2NAVI_VK_DYN_A2C"); // no fixed default: it depends on the driver, see above int marks = vk_pool_marks_enabled(); int da_on = !marks && (da && *da ? strcmp(da, "0") != 0 : !vk_caps.amd_sc); if (!da_on && P.dyn_a2c) { P.dyn_a2c = 0; rt_log("vk_pipeline: dynamic alpha to coverage off (%s)", marks ? "the texture pool marks, NX2NAVI_TEX_DISUSE" : da && *da ? "NX2NAVI_VK_DYN_A2C=0" : "AMD's driver for RDNA 2 and older"); } #undef PROC static const VkDynamicState core[] = { VK_DYNAMIC_STATE_VIEWPORT_WITH_COUNT, VK_DYNAMIC_STATE_SCISSOR_WITH_COUNT, VK_DYNAMIC_STATE_DEPTH_BIAS, VK_DYNAMIC_STATE_BLEND_CONSTANTS, VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK, VK_DYNAMIC_STATE_STENCIL_WRITE_MASK, VK_DYNAMIC_STATE_STENCIL_REFERENCE, VK_DYNAMIC_STATE_CULL_MODE, VK_DYNAMIC_STATE_FRONT_FACE, VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY, VK_DYNAMIC_STATE_DEPTH_TEST_ENABLE, VK_DYNAMIC_STATE_DEPTH_WRITE_ENABLE, VK_DYNAMIC_STATE_DEPTH_COMPARE_OP, VK_DYNAMIC_STATE_STENCIL_TEST_ENABLE, VK_DYNAMIC_STATE_STENCIL_OP, VK_DYNAMIC_STATE_DEPTH_BIAS_ENABLE, VK_DYNAMIC_STATE_RASTERIZER_DISCARD_ENABLE, VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE}; P.n_dyn = 0; for (size_t i = 0; i < sizeof core / sizeof *core; i++) P.dyn[P.n_dyn++] = core[i]; P.dyn[P.n_dyn++] = P.dyn_vi ? VK_DYNAMIC_STATE_VERTEX_INPUT_EXT : VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE; if (P.dyn_clamp) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_DEPTH_CLAMP_ENABLE_EXT; if (P.dyn_poly) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_POLYGON_MODE_EXT; if (P.dyn_blend_en) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_BLEND_ENABLE_EXT; if (P.dyn_blend_eq) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_BLEND_EQUATION_EXT; if (P.dyn_wmask) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_WRITE_MASK_EXT; if (P.dyn_a2c) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_ALPHA_TO_COVERAGE_ENABLE_EXT; vk_mem_stage_log("device creation"); cache_load(c); vk_mem_stage_log("the pipeline cache load"); rt_log("vk_pipeline: max push descriptors %u; dynamic vertex input %d, depth clamp %d, polygon mode %d, blend enable %d, " "blend equation %d, write mask %d, alpha to coverage %d; cache %s", P.max_push, P.dyn_vi, P.dyn_clamp, P.dyn_poly, P.dyn_blend_en, P.dyn_blend_eq, P.dyn_wmask, P.dyn_a2c, P.cache_path[0] ? P.cache_path : ""); keys_open(); cu_open(); plat_xchg32(&P.ready, 1); plat_mutex_unlock(&P.lock); prebuild_start(); return 1; } // ---- NVN -> Vulkan mappings static VkBlendFactor blend_factor(unsigned f) { switch (f) { case 1: return VK_BLEND_FACTOR_ZERO; case 2: return VK_BLEND_FACTOR_ONE; case 3: return VK_BLEND_FACTOR_SRC_COLOR; case 4: return VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR; case 5: return VK_BLEND_FACTOR_SRC_ALPHA; case 6: return VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA; case 7: return VK_BLEND_FACTOR_DST_ALPHA; case 8: return VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA; case 9: return VK_BLEND_FACTOR_DST_COLOR; case 10: return VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR; case 11: return VK_BLEND_FACTOR_SRC_ALPHA_SATURATE; case 0x10: return VK_BLEND_FACTOR_SRC1_COLOR; case 0x11: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR; case 0x12: return VK_BLEND_FACTOR_SRC1_ALPHA; case 0x13: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA; case 0x61: return VK_BLEND_FACTOR_CONSTANT_COLOR; case 0x62: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR; case 0x63: return VK_BLEND_FACTOR_CONSTANT_ALPHA; case 0x64: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA; default: LOG_ONCE("vk_pipeline: unknown NVN blend factor %#x, using ONE", f); return VK_BLEND_FACTOR_ONE; } } static VkBlendOp blend_op(unsigned e) { switch (e) { case 2: return VK_BLEND_OP_SUBTRACT; case 3: return VK_BLEND_OP_REVERSE_SUBTRACT; case 4: return VK_BLEND_OP_MIN; case 5: return VK_BLEND_OP_MAX; default: return VK_BLEND_OP_ADD; } } // NVN_DEPTH_FUNC_* / NVN_STENCIL_FUNC_* are NEVER = 1 .. ALWAYS = 8 in VkCompareOp order. static VkCompareOp compare_op(unsigned f) { return f >= 1 && f <= 8 ? (VkCompareOp)(f - 1) : VK_COMPARE_OP_ALWAYS; } // NVN_STENCIL_OP_* are KEEP = 1 .. DECR_WRAP = 8 in VkStencilOp order. static VkStencilOp stencil_op(unsigned o) { return o >= 1 && o <= 8 ? (VkStencilOp)(o - 1) : VK_STENCIL_OP_KEEP; } // NVN_POLYGON_MODE_POINT/LINE/FILL = 0/1/2, VkPolygonMode FILL/LINE/POINT = 0/1/2. static VkPolygonMode polygon_mode(unsigned m) { if (!P.f_fill_non_solid || m > 2) return VK_POLYGON_MODE_FILL; return (VkPolygonMode)(2 - m); } VkPrimitiveTopology vk_pipeline_topology(int prim) { switch (prim) { case NVN_DRAW_PRIMITIVE_POINTS: return VK_PRIMITIVE_TOPOLOGY_POINT_LIST; case NVN_DRAW_PRIMITIVE_LINES: return VK_PRIMITIVE_TOPOLOGY_LINE_LIST; case NVN_DRAW_PRIMITIVE_LINE_LOOP: case NVN_DRAW_PRIMITIVE_LINE_STRIP: return VK_PRIMITIVE_TOPOLOGY_LINE_STRIP; case NVN_DRAW_PRIMITIVE_TRIANGLE_STRIP: case NVN_DRAW_PRIMITIVE_QUAD_STRIP: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; // same triangles, same coverage case NVN_DRAW_PRIMITIVE_TRIANGLE_FAN: case NVN_DRAW_PRIMITIVE_POLYGON: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN; case NVN_DRAW_PRIMITIVE_LINES_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY; case NVN_DRAW_PRIMITIVE_LINE_STRIP_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY; case NVN_DRAW_PRIMITIVE_TRIANGLES_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY; case NVN_DRAW_PRIMITIVE_TRIANGLE_STRIP_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY; case NVN_DRAW_PRIMITIVE_PATCHES: return VK_PRIMITIVE_TOPOLOGY_PATCH_LIST; default: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; // TRIANGLES; QUADS after the replay's index rewrite } } // Topology class (a pipeline's static topology only fixes the class): 0 point, 1 line, 2 triangle, 3 patch. static int topology_class(VkPrimitiveTopology t) { switch (t) { case VK_PRIMITIVE_TOPOLOGY_POINT_LIST: return 0; case VK_PRIMITIVE_TOPOLOGY_LINE_LIST: case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP: case VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY: case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY: return 1; case VK_PRIMITIVE_TOPOLOGY_PATCH_LIST: return 3; default: return 2; } } int vk_pipeline_tex_type(int target) { switch (target) { case NVN_TEXTURE_TARGET_1D: return GFX_TEX_1D; case NVN_TEXTURE_TARGET_3D: return GFX_TEX_3D; case NVN_TEXTURE_TARGET_1D_ARRAY: return GFX_TEX_1D_ARRAY; case NVN_TEXTURE_TARGET_2D_ARRAY: return GFX_TEX_2D_ARRAY; case NVN_TEXTURE_TARGET_2D_MULTISAMPLE: return GFX_TEX_2D_MS; case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY: return GFX_TEX_2D_MS_ARRAY; case NVN_TEXTURE_TARGET_CUBEMAP: return GFX_TEX_CUBE; case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: return GFX_TEX_CUBE_ARRAY; case NVN_TEXTURE_TARGET_BUFFER: return GFX_TEX_BUFFER; default: return GFX_TEX_2D; // 2D, RECTANGLE } } // Numeric type of a colour format as a shader sees it: GFX_TYPE_*. static int format_type(VkFormat f) { switch (f) { case VK_FORMAT_R8_UINT: case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8B8_UINT: case VK_FORMAT_R8G8B8A8_UINT: case VK_FORMAT_R16_UINT: case VK_FORMAT_R16G16_UINT: case VK_FORMAT_R16G16B16_UINT: case VK_FORMAT_R16G16B16A16_UINT: case VK_FORMAT_R32_UINT: case VK_FORMAT_R32G32_UINT: case VK_FORMAT_R32G32B32_UINT: case VK_FORMAT_R32G32B32A32_UINT: case VK_FORMAT_A2B10G10R10_UINT_PACK32: case VK_FORMAT_B8G8R8A8_UINT: case VK_FORMAT_S8_UINT: return GFX_TYPE_UINT; case VK_FORMAT_R8_SINT: case VK_FORMAT_R8G8_SINT: case VK_FORMAT_R8G8B8_SINT: case VK_FORMAT_R8G8B8A8_SINT: case VK_FORMAT_R16_SINT: case VK_FORMAT_R16G16_SINT: case VK_FORMAT_R16G16B16_SINT: case VK_FORMAT_R16G16B16A16_SINT: case VK_FORMAT_R32_SINT: case VK_FORMAT_R32G32_SINT: case VK_FORMAT_R32G32B32_SINT: case VK_FORMAT_R32G32B32A32_SINT: case VK_FORMAT_A2B10G10R10_SINT_PACK32: case VK_FORMAT_B8G8R8A8_SINT: return GFX_TYPE_SINT; default: return GFX_TYPE_FLOAT; } } // Optimal-tiling (o = 1) or buffer (o = 0) features of a core format, cached. static VkFormatFeatureFlags format_features(VkFormat f, int optimal) { static VkFormatFeatureFlags known[2][256]; static uint8_t have[2][256]; VkCtx *c = vk_ctx(); if (!c) return 0; if ((unsigned)f < 256 && have[optimal][f]) return known[optimal][f]; VkFormatProperties p; vkGetPhysicalDeviceFormatProperties(c->phys, f, &p); VkFormatFeatureFlags r = optimal ? p.optimalTilingFeatures : p.bufferFeatures; if ((unsigned)f < 256) known[optimal][f] = r, have[optimal][f] = 1; return r; } // Vertex attribute formats by NVNformat. typedef struct { VkFormat vk; uint8_t type; } VtxFmt; #define F(v) {VK_FORMAT_##v, GFX_TYPE_FLOAT} #define U(v) {VK_FORMAT_##v, GFX_TYPE_UINT} #define I(v) {VK_FORMAT_##v, GFX_TYPE_SINT} static const VtxFmt vtx_formats[0x9B] = { [0x01] = F(R8_UNORM), [0x02] = F(R8_SNORM), [0x03] = U(R8_UINT), [0x04] = I(R8_SINT), [0x05] = F(R16_SFLOAT), [0x06] = F(R16_UNORM), [0x07] = F(R16_SNORM), [0x08] = U(R16_UINT), [0x09] = I(R16_SINT), [0x0A] = F(R32_SFLOAT), [0x0B] = U(R32_UINT), [0x0C] = I(R32_SINT), [0x0D] = F(R8G8_UNORM), [0x0E] = F(R8G8_SNORM), [0x0F] = U(R8G8_UINT), [0x10] = I(R8G8_SINT), [0x11] = F(R16G16_SFLOAT), [0x12] = F(R16G16_UNORM), [0x13] = F(R16G16_SNORM), [0x14] = U(R16G16_UINT), [0x15] = I(R16G16_SINT), [0x16] = F(R32G32_SFLOAT), [0x17] = U(R32G32_UINT), [0x18] = I(R32G32_SINT), [0x19] = F(R8G8B8_UNORM), [0x1A] = F(R8G8B8_SNORM), [0x1B] = U(R8G8B8_UINT), [0x1C] = I(R8G8B8_SINT), [0x1D] = F(R16G16B16_SFLOAT), [0x1E] = F(R16G16B16_UNORM), [0x1F] = F(R16G16B16_SNORM), [0x20] = U(R16G16B16_UINT), [0x21] = I(R16G16B16_SINT), [0x22] = F(R32G32B32_SFLOAT), [0x23] = U(R32G32B32_UINT), [0x24] = I(R32G32B32_SINT), [0x25] = F(R8G8B8A8_UNORM), [0x26] = F(R8G8B8A8_SNORM), [0x27] = U(R8G8B8A8_UINT), [0x28] = I(R8G8B8A8_SINT), [0x29] = F(R16G16B16A16_SFLOAT), [0x2A] = F(R16G16B16A16_UNORM), [0x2B] = F(R16G16B16A16_SNORM), [0x2C] = U(R16G16B16A16_UINT), [0x2D] = I(R16G16B16A16_SINT), [0x2E] = F(R32G32B32A32_SFLOAT), [0x2F] = U(R32G32B32A32_UINT), [0x30] = I(R32G32B32A32_SINT), [0x3D] = F(A2B10G10R10_UNORM_PACK32), [0x3E] = U(A2B10G10R10_UINT_PACK32), [0x3F] = F(B10G11R11_UFLOAT_PACK32), // *_UI2F / *_I2F: integer data read as float; the 8- and 16-bit ones are Vulkan's SCALED formats [0x51] = F(R8_USCALED), [0x52] = F(R8_SSCALED), [0x53] = F(R16_USCALED), [0x54] = F(R16_SSCALED), [0x57] = F(R8G8_USCALED), [0x58] = F(R8G8_SSCALED), [0x59] = F(R16G16_USCALED), [0x5A] = F(R16G16_SSCALED), [0x5D] = F(R8G8B8_USCALED), [0x5E] = F(R8G8B8_SSCALED), [0x5F] = F(R16G16B16_USCALED), [0x60] = F(R16G16B16_SSCALED), [0x63] = F(R8G8B8A8_USCALED), [0x64] = F(R8G8B8A8_SSCALED), [0x65] = F(R16G16B16A16_USCALED), [0x66] = F(R16G16B16A16_SSCALED), // 32-bit ones have no SCALED format: read as integers (values are not converted; logged when used) [0x55] = U(R32_UINT), [0x56] = I(R32_SINT), [0x5B] = U(R32G32_UINT), [0x5C] = I(R32G32_SINT), [0x61] = U(R32G32B32_UINT), [0x62] = I(R32G32B32_SINT), [0x67] = U(R32G32B32A32_UINT), [0x68] = I(R32G32B32A32_SINT), [0x69] = F(A2B10G10R10_SNORM_PACK32), [0x6A] = I(A2B10G10R10_SINT_PACK32), [0x6B] = F(A2B10G10R10_USCALED_PACK32), [0x6C] = F(A2B10G10R10_SSCALED_PACK32), [0x9A] = F(B8G8R8A8_UNORM), }; #undef F #undef U #undef I // 3-component 8/16-bit formats are not vertex formats on every GPU: read 4 components (LAS vertex_format_supported). static VkFormat widen(VkFormat f) { switch (f) { case VK_FORMAT_R8G8B8_UNORM: return VK_FORMAT_R8G8B8A8_UNORM; case VK_FORMAT_R8G8B8_SNORM: return VK_FORMAT_R8G8B8A8_SNORM; case VK_FORMAT_R8G8B8_UINT: return VK_FORMAT_R8G8B8A8_UINT; case VK_FORMAT_R8G8B8_SINT: return VK_FORMAT_R8G8B8A8_SINT; case VK_FORMAT_R8G8B8_USCALED: return VK_FORMAT_R8G8B8A8_USCALED; case VK_FORMAT_R8G8B8_SSCALED: return VK_FORMAT_R8G8B8A8_SSCALED; case VK_FORMAT_R16G16B16_SFLOAT: return VK_FORMAT_R16G16B16A16_SFLOAT; case VK_FORMAT_R16G16B16_UNORM: return VK_FORMAT_R16G16B16A16_UNORM; case VK_FORMAT_R16G16B16_SNORM: return VK_FORMAT_R16G16B16A16_SNORM; case VK_FORMAT_R16G16B16_UINT: return VK_FORMAT_R16G16B16A16_UINT; case VK_FORMAT_R16G16B16_SINT: return VK_FORMAT_R16G16B16A16_SINT; case VK_FORMAT_R16G16B16_USCALED: return VK_FORMAT_R16G16B16A16_USCALED; case VK_FORMAT_R16G16B16_SSCALED: return VK_FORMAT_R16G16B16A16_SSCALED; default: return VK_FORMAT_UNDEFINED; } } VkFormat vk_vertex_format(int nvn_format, int *type) { if (type) *type = GFX_TYPE_FLOAT; if (nvn_format <= 0 || nvn_format >= (int)(sizeof vtx_formats / sizeof *vtx_formats)) return VK_FORMAT_UNDEFINED; const VtxFmt *v = &vtx_formats[nvn_format]; if (v->vk == VK_FORMAT_UNDEFINED) return VK_FORMAT_UNDEFINED; if (type) *type = v->type; if ((nvn_format >= 0x55 && nvn_format <= 0x68) && v->type != GFX_TYPE_FLOAT) LOG_ONCE("vk_pipeline: 32-bit *_UI2F/*_I2F vertex formats are read as integers (first: %#x)", nvn_format); VkFormat f = v->vk; if (vk_ctx() && !(format_features(f, 0) & VK_FORMAT_FEATURE_VERTEX_BUFFER_BIT)) { VkFormat w = widen(f); if (w == VK_FORMAT_UNDEFINED || !(format_features(w, 0) & VK_FORMAT_FEATURE_VERTEX_BUFFER_BIT)) { LOG_ONCE("vk_pipeline: vertex format %d (NVN %#x) unsupported", (int)f, nvn_format); return VK_FORMAT_UNDEFINED; } LOG_ONCE("vk_pipeline: vertex format %d unsupported, read as 4 components (first: NVN %#x)", (int)f, nvn_format); f = w; } return f; } // Attachment format of an NVN format (tex_vk_format: the format the images are created with). static VkFormat target_format(int nvn_format, int *depth, int *stencil) { TexVkFormat t; if (depth) *depth = 0; if (stencil) *stencil = 0; if (!nvn_format || !tex_vk_format(nvn_format, &t)) return VK_FORMAT_UNDEFINED; if (depth) *depth = t.depth; if (stencil) *stencil = t.stencil; return t.vk; } void vk_pipe_state_defaults(VkPipeState *s) { memset(s, 0, sizeof *s); s->samples = 1; s->prim = NVN_DRAW_PRIMITIVE_TRIANGLES; for (int i = 0; i < 8; i++) s->blend[i] = (blend_st){(uint64_t)i, NVN_BLEND_FUNC_ONE, NVN_BLEND_FUNC_ZERO, NVN_BLEND_FUNC_ONE, NVN_BLEND_FUNC_ZERO, NVN_BLEND_EQUATION_ADD, NVN_BLEND_EQUATION_ADD}; s->color = (color_st){0, NVN_LOGIC_OP_COPY, NVN_ALPHA_FUNC_ALWAYS}; s->chmask.mask = 0xffffffffu; s->ms.enable = 1; s->poly = (poly_st){NVN_FACE_NONE, NVN_FRONT_FACE_CCW, NVN_POLYGON_MODE_FILL, 0}; s->ds = (ds_st){0, 1, NVN_DEPTH_FUNC_LESS, 0, NVN_STENCIL_FUNC_ALWAYS, NVN_STENCIL_FUNC_ALWAYS, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP}; } void vk_pipeline_answers(const VkPipeState *s, GfxShaderAnswers *vs, GfxShaderAnswers *fs) { if (vs) for (int i = 0; i < 16; i++) { va_st a; memcpy(&a, &s->vattr[i], sizeof a); int type = GFX_TYPE_FLOAT; if (a.fmt) vk_vertex_format((int)a.fmt, &type); vs->attr_type[i] = (uint8_t)type; } if (fs) for (int i = 0; i < 8; i++) { VkFormat f = target_format(s->color_fmt[i], NULL, NULL); fs->frag_out_type[i] = (uint8_t)(f == VK_FORMAT_UNDEFINED ? GFX_TYPE_FLOAT : format_type(f)); } } // ---- vertex input (pipeline state, or vkCmdSetVertexInputEXT) typedef VkVtxInput VtxInput; // An attribute the game left without a format is a fixed one on Maxwell, a single 32-bit component of 0 (an unused // attribute is bound as IsFixed 1x32 float): it reads (0, 0, 0, 1), as Vulkan expands a // one-component format. NX2NAVI's foliage shaders scale wind sway by 1 - colour.w; shapes without a vertex colour // (Brightbloom stem and leaves) must not sway, or they bend away from the parts that have one (the bulb). static VkFormat zero_format(int type) { return type == GFX_TYPE_UINT ? VK_FORMAT_R32_UINT : type == GFX_TYPE_SINT ? VK_FORMAT_R32_SINT : VK_FORMAT_R32_SFLOAT; } // Attributes for the Locations the vertex shader reads; one without a usable NVN attribute reads VK_PIPE_ZERO_BINDING. static void vertex_input(const VkPipeState *s, VtxInput *v) { GfxShaderIo io[32]; uint32_t used = gfx_shader_inputs(s->vs, io), streams = 0; int zero = 0; memset(v, 0, sizeof *v); for (int l = 0; l < 32; l++) { if (!(used >> l & 1)) continue; VkVertexInputAttributeDescription2EXT *a = &v->a[v->n_attr++]; *a = (VkVertexInputAttributeDescription2EXT){.sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_ATTRIBUTE_DESCRIPTION_2_EXT, .location = (uint32_t)l}; va_st va = {0}; if (l < 16) memcpy(&va, &s->vattr[l], sizeof va); VkFormat f = va.fmt ? vk_vertex_format((int)va.fmt, NULL) : VK_FORMAT_UNDEFINED; if (f != VK_FORMAT_UNDEFINED && va.stream < 16) { a->binding = va.stream, a->format = f, a->offset = va.offset; streams |= 1u << va.stream; } else { a->binding = VK_PIPE_ZERO_BINDING, a->format = zero_format(io[l].type); zero = 1; } } for (uint32_t b = 0; b < 16; b++) { if (!(streams >> b & 1)) continue; uint32_t div = s->vstream[b].divisor; if (div > 1) LOG_ONCE("vk_pipeline: instance divisor %u treated as 1 (no VK_EXT_vertex_attribute_divisor)", div); v->b[v->n_bind++] = (VkVertexInputBindingDescription2EXT){ .sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT, .binding = b, .stride = s->vstream[b].stride, .inputRate = div ? VK_VERTEX_INPUT_RATE_INSTANCE : VK_VERTEX_INPUT_RATE_VERTEX, .divisor = 1}; } if (zero) v->b[v->n_bind++] = (VkVertexInputBindingDescription2EXT){ .sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT, .binding = VK_PIPE_ZERO_BINDING, .stride = 0, .inputRate = VK_VERTEX_INPUT_RATE_VERTEX, .divisor = 1}; } int vk_pipeline_vertex_input_dynamic(void) { return ensure_init() && P.dyn_vi; } PFN_vkCmdPushDescriptorSetKHR vk_pipeline_push_fn(void) { return ensure_init() ? P.push : NULL; } PFN_vkCmdPushDescriptorSetWithTemplateKHR vk_pipeline_push_tpl_fn(void) { return ensure_init() && P.tpl ? P.tpl_push : NULL; } PFN_vkUpdateDescriptorSetWithTemplate vk_pipeline_update_tpl_fn(void) { return ensure_init() && P.tpl ? P.tpl_update : NULL; } int vk_pipeline_set_indep(int on) { if (!ensure_init()) return 0; plat_mutex_lock(&P.lock); int was = P.indep; P.indep = on && vk_caps.gpl; plat_mutex_unlock(&P.lock); return was; } // ---- SPIR-V rewrites // Binding decorations b >= 1 become b + off (the fragment stage's share of set 0). static uint32_t *spirv_rebind(const uint32_t *w, uint64_t n, uint32_t off) { uint32_t *o = malloc(n * 4); if (!o) return NULL; memcpy(o, w, n * 4); for (uint64_t i = 5; i < n;) { uint32_t op = o[i] & 0xffff, len = o[i] >> 16; if (!len || i + len > n || op == SPV_OP_FUNCTION) break; if (op == SPV_OP_DECORATE && len >= 4 && o[i + 2] == SPV_DEC_BINDING && o[i + 3] >= 1) o[i + 3] += off; i += len; } return o; } // DescriptorSet decorations become set (the fragment stage's resources move to set 1 with independent layouts). static uint32_t *spirv_set(const uint32_t *w, uint64_t n, uint32_t set) { uint32_t *o = malloc(n * 4); if (!o) return NULL; memcpy(o, w, n * 4); for (uint64_t i = 5; i < n;) { uint32_t op = o[i] & 0xffff, len = o[i] >> 16; if (!len || i + len > n || op == SPV_OP_FUNCTION) break; if (op == SPV_OP_DECORATE && len >= 4 && o[i + 2] == SPV_DEC_DESCRIPTOR_SET) o[i + 3] = set; i += len; } return o; } // Dual-source blending (docs/maxwell.md 5.6): SRC1 blend factors read the fragment shader's colour output 1, // which Vulkan takes from Location 0, Index 1. The Output variable at Location 1 moves there, as the translator declares it // with DualSourceBlendEnable (CodeGen/Spirv/Declarations.cs); the pipeline is keyed by GKey.dsb, so only pipelines // that blend with SRC1 get this module. NULL when the module has no Output at Location 1. static uint32_t *spirv_dual_source(const uint32_t *w, uint64_t n, uint64_t *out_n) { uint32_t outs[64]; int nout = 0; uint64_t at = 0; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (!len || i + len > n) return NULL; if (op == SPV_OP_VARIABLE && len >= 4 && w[i + 3] == SPV_SC_OUTPUT && nout < 64) outs[nout++] = w[i + 2]; i += len; } for (uint64_t i = 5; i < n && !at;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (op == SPV_OP_DECORATE && len == 4 && w[i + 2] == SPV_DEC_LOCATION && w[i + 3] == 1) for (int k = 0; k < nout; k++) if (outs[k] == w[i + 1]) at = i; i += len; } if (!at) return NULL; uint32_t *o = malloc((n + 4) * 4); if (!o) return NULL; memcpy(o, w, (at + 4) * 4); o[at + 3] = 0; o[at + 4] = 4u << 16 | SPV_OP_DECORATE, o[at + 5] = w[at + 1], o[at + 6] = SPV_DEC_INDEX, o[at + 7] = 1; memcpy(o + at + 8, w + at + 4, (n - at - 4) * 4); *out_n = n + 4; return o; } // Adds an Output variable for each Location of `missing` (type and width from io) to a module, declared in the entry // point interface and initialised as Maxwell reads an unwritten output: (0, 0, 0, 1), the w as the bits of 1.0f whatever // the type (attribute memory is untyped); fewer than 4 components are all zero. Existing scalar, vector and pointer types // are reused. NULL when the module does not have the expected layout. static uint32_t *spirv_add_outputs(const uint32_t *w, uint64_t n, uint32_t missing, const GfxShaderIo io[32], uint64_t *out_n) { uint64_t ep_at = 0, ann_end = 0, fn_at = 0; uint32_t bound = w[3], scalar[3] = {0, 0, 0}; // GFX_TYPE_FLOAT, SINT, UINT struct { uint32_t id, comp, count; } vec[64]; struct { uint32_t id, type; } ptr[64]; int nvec = 0, nptr = 0; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (!len || i + len > n) return NULL; const uint32_t *a = w + i + 1; if (op == SPV_OP_ENTRY_POINT && !ep_at) ep_at = i; else if (op == SPV_OP_DECORATE || op == 72 || op == 73 || op == 74 || op == 75 || op == 332 || op == 5632 || op == 5633) ann_end = i + len; else if (op == SPV_OP_TYPE_FLOAT && len >= 3 && a[1] == 32) scalar[GFX_TYPE_FLOAT] = a[0]; else if (op == SPV_OP_TYPE_INT && len >= 4 && a[1] == 32) scalar[a[2] ? GFX_TYPE_SINT : GFX_TYPE_UINT] = a[0]; else if (op == SPV_OP_TYPE_VECTOR && len >= 4 && nvec < 64) vec[nvec].id = a[0], vec[nvec].comp = a[1], vec[nvec++].count = a[2]; else if (op == SPV_OP_TYPE_POINTER && len >= 4 && a[1] == SPV_SC_OUTPUT && nptr < 64) ptr[nptr].id = a[0], ptr[nptr++].type = a[2]; else if (op == SPV_OP_FUNCTION) { fn_at = i; break; } i += len; } if (!ep_at || !ann_end || !fn_at || ann_end < ep_at || fn_at < ann_end) return NULL; uint32_t types[32 * 40], decs[32 * 4], vars[32], zero[3] = {0, 0, 0}, one[3] = {0, 0, 0}; int nt = 0, nd = 0, nv = 0; for (int l = 0; l < 32; l++) { if (!(missing >> l & 1)) continue; int t = io[l].type <= GFX_TYPE_UINT ? io[l].type : GFX_TYPE_FLOAT; uint32_t comps = io[l].comps ? io[l].comps : 4; if (!scalar[t]) { scalar[t] = bound++; if (t == GFX_TYPE_FLOAT) types[nt++] = 3u << 16 | SPV_OP_TYPE_FLOAT, types[nt++] = scalar[t], types[nt++] = 32; else types[nt++] = 4u << 16 | SPV_OP_TYPE_INT, types[nt++] = scalar[t], types[nt++] = 32, types[nt++] = t == GFX_TYPE_SINT; } uint32_t type = scalar[t]; if (comps > 1) { int k = 0; while (k < nvec && !(vec[k].comp == type && vec[k].count == comps)) k++; if (k == nvec) { if (nvec == 64) return NULL; vec[nvec].id = bound++, vec[nvec].comp = type, vec[nvec++].count = comps; types[nt++] = 4u << 16 | SPV_OP_TYPE_VECTOR, types[nt++] = vec[k].id, types[nt++] = type, types[nt++] = comps; } type = vec[k].id; } int k = 0; while (k < nptr && ptr[k].type != type) k++; if (k == nptr) { if (nptr == 64) return NULL; ptr[nptr].id = bound++, ptr[nptr++].type = type; types[nt++] = 4u << 16 | SPV_OP_TYPE_POINTER, types[nt++] = ptr[k].id, types[nt++] = SPV_SC_OUTPUT, types[nt++] = type; } uint32_t init = bound++, var = bound++; if (comps == 4) { if (!zero[t]) { zero[t] = bound++, one[t] = bound++; types[nt++] = 4u << 16 | SPV_OP_CONSTANT, types[nt++] = scalar[t], types[nt++] = zero[t], types[nt++] = 0; types[nt++] = 4u << 16 | SPV_OP_CONSTANT, types[nt++] = scalar[t], types[nt++] = one[t], types[nt++] = 0x3f800000; } types[nt++] = 7u << 16 | SPV_OP_CONSTANT_COMPOSITE, types[nt++] = type, types[nt++] = init; types[nt++] = zero[t], types[nt++] = zero[t], types[nt++] = zero[t], types[nt++] = one[t]; } else { types[nt++] = 3u << 16 | SPV_OP_CONSTANT_NULL, types[nt++] = type, types[nt++] = init; } types[nt++] = 5u << 16 | SPV_OP_VARIABLE, types[nt++] = ptr[k].id, types[nt++] = var, types[nt++] = SPV_SC_OUTPUT, types[nt++] = init; decs[nd++] = 4u << 16 | SPV_OP_DECORATE, decs[nd++] = var, decs[nd++] = SPV_DEC_LOCATION, decs[nd++] = (uint32_t)l; vars[nv++] = var; } uint64_t total = n + (uint64_t)(nt + nd + nv); uint32_t *o = malloc(total * 4), *p = o; if (!o) return NULL; uint32_t ep_len = w[ep_at] >> 16; memcpy(p, w, 5 * 4), p[3] = bound, p += 5; memcpy(p, w + 5, (ep_at - 5) * 4), p += ep_at - 5; *p++ = (ep_len + (uint32_t)nv) << 16 | SPV_OP_ENTRY_POINT; memcpy(p, w + ep_at + 1, (ep_len - 1) * 4), p += ep_len - 1; memcpy(p, vars, (size_t)nv * 4), p += nv; memcpy(p, w + ep_at + ep_len, (ann_end - ep_at - ep_len) * 4), p += ann_end - ep_at - ep_len; memcpy(p, decs, (size_t)nd * 4), p += nd; memcpy(p, w + ann_end, (fn_at - ann_end) * 4), p += fn_at - ann_end; memcpy(p, types, (size_t)nt * 4), p += nt; memcpy(p, w + fn_at, (n - fn_at) * 4), p += n - fn_at; *out_n = (uint64_t)(p - o); return o; } // Two fixes for what the translator declares but a shader does not use (NULL: nothing to change): // - the fragment shader interlock capability and extension, which the translator adds to every fragment shader when the host // reports interlock support (SpirvGenerator.cs) and vk_device.c does not enable; kept when the shader interlocks; // - the CullDistance[1] member of gl_PerVertex next to ClipDistance[8] (Declarations.cs CreatePerVertexStructType): // 9 distances exceed maxCombinedClipAndCullDistances (8 on the RX 9070). The translator never accesses CullDistance; the // member goes when it is the struct's last and no access chain selects it. static uint32_t *spirv_fixup(const uint32_t *w, uint64_t n, uint64_t *out_n) { enum { OP_EXTENSION = 10, OP_CAPABILITY = 17, OP_MEMBER_NAME = 6, OP_EXECUTION_MODE = 16, OP_TYPE_ARRAY = 28, OP_TYPE_STRUCT = 30, OP_CONSTANT = 43, OP_MEMBER_DECORATE = 72, OP_ACCESS_CHAIN = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66, OP_BEGIN_INTERLOCK = 5364, OP_END_INTERLOCK = 5365, CAP_SAMPLE_INTERLOCK = 5363, CAP_PIXEL_INTERLOCK = 5378, CAP_RATE_INTERLOCK = 5372, BUILTIN_CLIP = 3, BUILTIN_CULL = 4 }; if (n < 5) return NULL; uint32_t bound = w[3] < (1u << 22) ? w[3] : (1u << 22); uint32_t *val = calloc(bound, sizeof *val), *elem = calloc(bound, sizeof *elem), *len = calloc(bound, sizeof *len), *pointee = calloc(bound, sizeof *pointee), *var_type = calloc(bound, sizeof *var_type); uint32_t *o = NULL; int interlock_used = 0, interlock_declared = 0, cull_ok = 0; uint32_t cull_struct = 0, cull_member = 0, clip_member = ~0u, members = 0, member_types[8] = {0}; if (!val || !elem || !len || !pointee || !var_type) goto out; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) goto out; const uint32_t *a = w + i + 1; if (op == OP_BEGIN_INTERLOCK || op == OP_END_INTERLOCK) interlock_used = 1; else if (op == OP_EXECUTION_MODE && l >= 3 && a[1] >= 5366 && a[1] <= 5371) interlock_used = 1; else if (op == OP_CAPABILITY && l >= 2 && (a[0] == CAP_PIXEL_INTERLOCK || a[0] == CAP_SAMPLE_INTERLOCK || a[0] == CAP_RATE_INTERLOCK)) interlock_declared = 1; else if (op == OP_CONSTANT && l == 4 && a[1] < bound) val[a[1]] = a[2]; else if (op == OP_TYPE_ARRAY && l >= 4 && a[0] < bound) elem[a[0]] = a[1], len[a[0]] = a[2] < bound ? val[a[2]] : 0; else if (op == SPV_OP_TYPE_POINTER && l >= 4 && a[0] < bound) pointee[a[0]] = a[2]; else if (op == SPV_OP_VARIABLE && l >= 4 && a[1] < bound) var_type[a[1]] = a[0] < bound ? pointee[a[0]] : 0; else if (op == OP_MEMBER_DECORATE && l >= 5 && a[2] == SPV_DEC_BUILTIN && a[3] == BUILTIN_CULL) cull_struct = a[0], cull_member = a[1]; else if (op == OP_MEMBER_DECORATE && l >= 5 && a[2] == SPV_DEC_BUILTIN && a[3] == BUILTIN_CLIP) clip_member = a[1]; else if (op == OP_TYPE_STRUCT && cull_struct && a[0] == cull_struct && l >= 2) { members = l - 2; for (uint32_t m = 0; m < members && m < 8; m++) member_types[m] = a[1 + m]; } else if ((op == OP_ACCESS_CHAIN || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && cull_struct && a[2] < bound) { uint32_t t = var_type[a[2]], k = 3; if (t < bound && t && elem[t] == cull_struct) k = 4; // arrayed per-vertex block if ((t == cull_struct || k == 4) && k + 1 < l && a[k] < bound && val[a[k]] == cull_member) cull_struct = 0; } i += l; } VkCtx *c = vk_ctx(); uint32_t limit = c ? c->props.limits.maxCombinedClipAndCullDistances : 8; if (cull_struct && members && members <= 8 && cull_member == members - 1 && clip_member < members) { uint32_t clip_t = member_types[clip_member], cull_t = member_types[cull_member]; uint32_t clip_n = clip_t < bound ? len[clip_t] : 0, cull_n = cull_t < bound ? len[cull_t] : 0; cull_ok = clip_n + cull_n > limit && clip_n <= limit; } int strip = interlock_declared && !interlock_used; if (!strip && !cull_ok) goto out; o = malloc(n * 4); if (!o) goto out; uint64_t on = 5; memcpy(o, w, 5 * 4); for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; const uint32_t *a = w + i + 1; int drop = 0; if (strip && op == OP_CAPABILITY && (a[0] == CAP_PIXEL_INTERLOCK || a[0] == CAP_SAMPLE_INTERLOCK || a[0] == CAP_RATE_INTERLOCK)) drop = 1; if (strip && op == OP_EXTENSION && l >= 2 && !strncmp((const char *)a, "SPV_EXT_fragment_shader_interlock", (l - 1) * 4)) drop = 1; if (cull_ok && (op == OP_MEMBER_DECORATE || op == OP_MEMBER_NAME) && l >= 3 && a[0] == cull_struct && a[1] == cull_member) drop = 1; if (!drop) { memcpy(o + on, w + i, l * 4); if (cull_ok && op == OP_TYPE_STRUCT && a[0] == cull_struct) o[on] = (l - 1) << 16 | op; // last member gone on += op == OP_TYPE_STRUCT && cull_ok && a[0] == cull_struct ? l - 1 : l; } i += l; } *out_n = on; out: free(val), free(elem), free(len), free(pointee), free(var_type); return o; } // True when the module declares a GroupNonUniform* or SubgroupBallot-style capability (lane masks assume 32 lanes). static int uses_subgroup_ops(const uint32_t *w, uint64_t words) { for (uint64_t i = 5; i < words;) { uint32_t op = w[i] & 0xffff, n = w[i] >> 16; if (!n || i + n > words) return 0; if (op != 17) return 0; // OpCapability; capabilities come first, stop at the first other instruction uint32_t cap = w[i + 1]; if ((cap >= 61 && cap <= 68) || cap == 4423 || cap == 4431) return 1; // NonUniform*, SubgroupBallotKHR, SubgroupVoteKHR i += n; } return 0; } // Guest code runs warp-wide operations (ballot, shuffle, vote) where the guest compiler had the warp reconverged, for // example right after a loop that each lane leaves at a different iteration. Vulkan only promises the same lanes at the // same point with the MaximallyReconvergesKHR execution mode. Without it a driver may run the lanes that left the loop // early through the operations on their own, with a ballot or scan that sees only some of the lanes: a mesh program // that hands out output slots from one could give two primitives the same slot. This adds the extension and the mode // for each entry point of a module that declares subgroup capabilities. NULL: nothing to change (or no memory). static uint32_t *spirv_reconverge(const uint32_t *w, uint64_t n, uint64_t *out_n) { enum { OP_EXTENSION = 10, OP_EXECUTION_MODE = 16, OP_CAPABILITY = 17, MODE_MAXIMALLY_RECONVERGES = 6023 }; static const char ext[] = "SPV_KHR_maximal_reconvergence"; const uint32_t ext_words = (uint32_t)(sizeof ext + 3) / 4; // the string with its terminating zero, padded if (n < 5 || !uses_subgroup_ops(w, n)) return NULL; uint64_t caps_end = 0, eps_end = 0; uint32_t eps[16], neps = 0; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) return NULL; if (op == OP_CAPABILITY) caps_end = i + l; else if (op == SPV_OP_ENTRY_POINT && l >= 3) { if (neps == 16) return NULL; eps[neps++] = w[i + 2], eps_end = i + l; } else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == MODE_MAXIMALLY_RECONVERGES) return NULL; else if (op == SPV_OP_FUNCTION) break; i += l; } if (!caps_end || !neps || eps_end < caps_end) return NULL; uint64_t on = n + 1 + ext_words + 3ull * neps; uint32_t *o = malloc(on * 4); if (!o) return NULL; uint64_t p = 0; memcpy(o, w, caps_end * 4), p = caps_end; o[p++] = (1 + ext_words) << 16 | OP_EXTENSION; memset(o + p, 0, ext_words * 4); memcpy(o + p, ext, sizeof ext), p += ext_words; memcpy(o + p, w + caps_end, (eps_end - caps_end) * 4), p += eps_end - caps_end; for (uint32_t k = 0; k < neps; k++) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = eps[k], o[p++] = MODE_MAXIMALLY_RECONVERGES; memcpy(o + p, w + eps_end, (n - eps_end) * 4), p += n - eps_end; *out_n = p; return o; } // Maxwell computes in IEEE 754, and the game's shaders rely on it: its compiler writes an unordered comparison as // `x > c || isnan(x)`, a normalisation of a zero-length vector gives an infinity that a later comparison sorts out, and // integer masks such as 0xffffffff travel through float moves and selects. The translator's own guards are isnan // tests as well (fragment outputs, min, max and clamp). Without the SignedZeroInfNanPreserve execution mode, SPIR-V // lets the driver assume that no float is NaN or infinite, and AMD's compilers act on it: on an RX 9070 XT a select // between 1 and the bits 0xffffffff, converted to float as a signed integer, gave 2^32 instead of -1 and blew the // shadow casters up into huge triangles (issue #156); RX 6000 cards drew Navi, and Link's skin next to her, with black // and white blocks (#41, #110); and AMD GPUs drew Navi's sparkles but not her orb, whose light sits at her own centre. // NVIDIA's driver keeps the IEEE results without the mode. This adds the capability, the extension (the translator // writes SPIR-V 1.3) and the mode for each entry point: 32-bit floats always, 16-bit and 64-bit ones where the module // declares Float16 or Float64 and `widths` (NP_*) allows them. NULL: nothing to change (the module has the mode, or no // 32-bit width is allowed, or no memory). static uint32_t *spirv_nan_preserve(const uint32_t *w, uint64_t n, uint32_t widths, uint64_t *out_n) { enum { OP_EXTENSION = 10, OP_EXECUTION_MODE = 16, OP_CAPABILITY = 17, CAP_FLOAT16 = 9, CAP_FLOAT64 = 10, CAP_PRESERVE = 4466, MODE_PRESERVE = 4461 }; static const char ext[] = "SPV_KHR_float_controls"; const uint32_t ext_words = (uint32_t)(sizeof ext + 3) / 4; // the string with its terminating zero, padded if (n < 5 || !(widths & NP_32)) return NULL; uint64_t caps_end = 0, eps_end = 0; uint32_t eps[16], neps = 0, use = NP_32; int has_cap = 0, has_ext = 0; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) return NULL; if (op == OP_CAPABILITY && l >= 2) { caps_end = i + l; if (w[i + 1] == CAP_FLOAT16) use |= NP_16; else if (w[i + 1] == CAP_FLOAT64) use |= NP_64; else if (w[i + 1] == CAP_PRESERVE) has_cap = 1; } else if (op == OP_EXTENSION && (l - 1) * 4 >= sizeof ext && !memcmp(w + i + 1, ext, sizeof ext)) { has_ext = 1; } else if (op == SPV_OP_ENTRY_POINT && l >= 3) { if (neps == 16) return NULL; eps[neps++] = w[i + 2], eps_end = i + l; } else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == MODE_PRESERVE) { return NULL; } else if (op == SPV_OP_FUNCTION) { break; } i += l; } if (!caps_end || !neps || eps_end < caps_end) return NULL; use &= widths; uint32_t nw = 0, width[3]; if (use & NP_16) width[nw++] = 16; width[nw++] = 32; if (use & NP_64) width[nw++] = 64; uint64_t on = n + (has_cap ? 0 : 2) + (has_ext ? 0 : 1 + ext_words) + 4ull * nw * neps; uint32_t *o = malloc(on * 4); if (!o) return NULL; uint64_t p = 0; memcpy(o, w, caps_end * 4), p = caps_end; if (!has_cap) o[p++] = 2u << 16 | OP_CAPABILITY, o[p++] = CAP_PRESERVE; if (!has_ext) { o[p++] = (1 + ext_words) << 16 | OP_EXTENSION; memset(o + p, 0, ext_words * 4); memcpy(o + p, ext, sizeof ext), p += ext_words; } memcpy(o + p, w + caps_end, (eps_end - caps_end) * 4), p += eps_end - caps_end; for (uint32_t k = 0; k < neps; k++) for (uint32_t j = 0; j < nw; j++) o[p++] = 4u << 16 | OP_EXECUTION_MODE, o[p++] = eps[k], o[p++] = MODE_PRESERVE, o[p++] = width[j]; memcpy(o + p, w + eps_end, (n - eps_end) * 4), p += n - eps_end; *out_n = p; return o; } // NaN tests that no compiler can drop (P.nan_bits). The game's shaders and the translator's own guards test for NaN with // OpIsNan: 48,717 of an install's 54,959 translated modules, 26 tests each on average (the game's unordered // comparisons, and the guard that keeps NaN out of every float colour output, patch 0027, so that one NaN texel does not // spread through bloom and blur into blocks of black and white). SignedZeroInfNanPreserve asks the driver to keep NaN, // but whether a compiler that reasons about floats keeps an isnan through every optimisation is up to it. The reports // of black and white blocks, on Navi, on Link next to her and on bright surfaces seen from inside a house, come from // RX 5000 and 6000 cards (#41, #110, #117), whose Windows driver compiles with "AMD proprietary shader compiler" // (vk_caps.amd_sc); RDNA 3 and 4 (LLPC) draw the same scenes clean. It is not known whether that compiler keeps the // mode: on it each float32 test becomes integer arithmetic on the bits, which means the same under any float // assumption: // isnan(x) -> (bitcast(x) & 0x7fffffff) > 0x7f800000 // for scalars and vectors of 32-bit floats; tests of 16- and 64-bit floats stay. A float32 select with a constant // operand whose bits are a NaN (an integer mask such as 0xffffffff that the translator types as float) becomes a // select of the bits (issue #156: on an RX 9070 XT without the mode such a select between 1 and -1 gave 2^32). // NULL: nothing to change (or no memory); *tests and *selects count what changed. static uint32_t *spirv_nan_bits(const uint32_t *w, uint64_t n, uint64_t *out_n, uint32_t *tests, uint32_t *selects) { enum { OP_UNDEF = 1, OP_TYPE_BOOL = 20, OP_SPEC_CONSTANT = 50, OP_SPEC_CONSTANT_COMPOSITE = 51, OP_STORE = 62, OP_BITCAST = 124, OP_IS_NAN = 156, OP_SELECT = 169, OP_UGREATER = 172, OP_BITWISE_AND = 199 }; *tests = *selects = 0; if (n < 5 || w[3] >= (1u << 22)) return NULL; uint32_t bound = w[3]; uint32_t *rtype = calloc(bound, sizeof *rtype), *vec_comp = calloc(bound, sizeof *vec_comp); uint8_t *f32 = calloc(bound, 1), *vec_n = calloc(bound, 1), *nan_const = calloc(bound, 1); uint32_t *o = NULL; if (!rtype || !vec_comp || !f32 || !vec_n || !nan_const) goto out; uint32_t tu = 0, uvec[5] = {0}; // uint32 and its vectors by component count uint64_t fn_at = 0; int vec_used[5] = {0}; // Ids that are not results of the instruction's second word, inside functions: first assignment wins anyway, as // a definition comes before its uses (OpPhi operands aside, which this never reads). for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) goto out; const uint32_t *a = w + i + 1; if (!fn_at) { if (op == SPV_OP_FUNCTION) fn_at = i; else if (op == SPV_OP_TYPE_FLOAT && l >= 3 && a[0] < bound && a[1] == 32) f32[a[0]] = 1; else if (op == SPV_OP_TYPE_INT && l >= 4 && a[1] == 32 && a[2] == 0 && !tu) tu = a[0]; else if (op == SPV_OP_TYPE_VECTOR && l >= 4 && a[0] < bound && a[2] >= 2 && a[2] <= 4) { vec_comp[a[0]] = a[1], vec_n[a[0]] = (uint8_t)a[2]; if (tu && a[1] == tu && !uvec[a[2]]) uvec[a[2]] = a[0]; } else if ((op == SPV_OP_CONSTANT || op == SPV_OP_CONSTANT_COMPOSITE || op == SPV_OP_CONSTANT_NULL || op == OP_UNDEF || op == SPV_OP_VARIABLE || op == OP_SPEC_CONSTANT || op == OP_SPEC_CONSTANT_COMPOSITE) && l >= 3 && a[1] < bound) { rtype[a[1]] = a[0]; if (op == SPV_OP_CONSTANT && l == 4 && a[0] < bound && f32[a[0]] && (a[2] & 0x7fffffffu) > 0x7f800000u) nan_const[a[1]] = 1; } } if (fn_at) { int no_result = op == OP_STORE || op == 63 || op == 64 || op == 8 || op == 317 || (op >= 246 && op <= 255) || op == 99 || op == 224 || op == 225 || op == 228 || op == 56 || op == 218 || op == 219 || op == 5294 || op == 5295 || op == 4416 || op == 5380 || op == 5364 || op == 5365; if (!no_result && op != SPV_OP_FUNCTION && l >= 3 && a[1] < bound && !rtype[a[1]]) rtype[a[1]] = a[0]; if (op == OP_IS_NAN && l == 4 && a[2] < bound) { uint32_t t = rtype[a[2]]; if (t < bound && f32[t]) ++*tests; else if (t < bound && vec_n[t] && vec_comp[t] < bound && f32[vec_comp[t]]) ++*tests, vec_used[vec_n[t]] = 1; } else if (op == OP_SELECT && l == 6 && a[0] < bound && f32[a[0]] && a[3] < bound && a[4] < bound && (nan_const[a[3]] || nan_const[a[4]])) { ++*selects; } } i += l; } if (!fn_at || (!*tests && !*selects)) goto out; // the declarations go right before the first function: uint32 and its vectors where missing, then the constants uint32_t next = bound, decl[64], nd = 0; if (!tu) tu = next++, decl[nd++] = 4u << 16 | SPV_OP_TYPE_INT, decl[nd++] = tu, decl[nd++] = 32, decl[nd++] = 0; uint32_t c_mag = next++, c_inf = next++, cv_mag[5] = {0}, cv_inf[5] = {0}; for (int k = 2; k <= 4; k++) if (vec_used[k] && !uvec[k]) uvec[k] = next++, decl[nd++] = 4u << 16 | SPV_OP_TYPE_VECTOR, decl[nd++] = uvec[k], decl[nd++] = tu, decl[nd++] = (uint32_t)k; decl[nd++] = 4u << 16 | SPV_OP_CONSTANT, decl[nd++] = tu, decl[nd++] = c_mag, decl[nd++] = 0x7fffffffu; decl[nd++] = 4u << 16 | SPV_OP_CONSTANT, decl[nd++] = tu, decl[nd++] = c_inf, decl[nd++] = 0x7f800000u; for (int k = 2; k <= 4; k++) { if (!vec_used[k]) continue; cv_mag[k] = next++, cv_inf[k] = next++; decl[nd++] = (uint32_t)(3 + k) << 16 | SPV_OP_CONSTANT_COMPOSITE, decl[nd++] = uvec[k], decl[nd++] = cv_mag[k]; for (int j = 0; j < k; j++) decl[nd++] = c_mag; decl[nd++] = (uint32_t)(3 + k) << 16 | SPV_OP_CONSTANT_COMPOSITE, decl[nd++] = uvec[k], decl[nd++] = cv_inf[k]; for (int j = 0; j < k; j++) decl[nd++] = c_inf; } uint64_t cap = n + nd + 10ull * *tests + 12ull * *selects; o = malloc(cap * 4); if (!o) goto out; memcpy(o, w, fn_at * 4); uint64_t p = fn_at; memcpy(o + p, decl, nd * 4), p += nd; for (uint64_t i = fn_at; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; const uint32_t *a = w + i + 1; uint32_t t = op == OP_IS_NAN && l == 4 && a[2] < bound ? rtype[a[2]] : 0; int k = t && t < bound ? (f32[t] ? 1 : vec_n[t] && vec_comp[t] < bound && f32[vec_comp[t]] ? vec_n[t] : 0) : 0; if (k) { uint32_t ut = k == 1 ? tu : uvec[k], mag = k == 1 ? c_mag : cv_mag[k], inf = k == 1 ? c_inf : cv_inf[k]; uint32_t bits = next++, low = next++; o[p++] = 4u << 16 | OP_BITCAST, o[p++] = ut, o[p++] = bits, o[p++] = a[2]; o[p++] = 5u << 16 | OP_BITWISE_AND, o[p++] = ut, o[p++] = low, o[p++] = bits, o[p++] = mag; o[p++] = 5u << 16 | OP_UGREATER, o[p++] = a[0], o[p++] = a[1], o[p++] = low, o[p++] = inf; } else if (op == OP_SELECT && l == 6 && a[0] < bound && f32[a[0]] && a[3] < bound && a[4] < bound && (nan_const[a[3]] || nan_const[a[4]])) { uint32_t x = next++, y = next++, s = next++; o[p++] = 4u << 16 | OP_BITCAST, o[p++] = tu, o[p++] = x, o[p++] = a[3]; o[p++] = 4u << 16 | OP_BITCAST, o[p++] = tu, o[p++] = y, o[p++] = a[4]; o[p++] = 6u << 16 | OP_SELECT, o[p++] = tu, o[p++] = s, o[p++] = a[2], o[p++] = x, o[p++] = y; o[p++] = 4u << 16 | OP_BITCAST, o[p++] = a[0], o[p++] = a[1], o[p++] = s; } else { memcpy(o + p, w + i, l * 4), p += l; } i += l; } o[3] = next; *out_n = p; out: free(rtype), free(vec_comp), free(f32), free(vec_n), free(nan_const); return o; } static int spirv_mark_list_test(const char *list); // texture pool marks, below // --selftest (main.c): spirv_nan_preserve on a small fragment module, without a device. Returns the number of failures. int vk_pipeline_spirv_selftest(void) { enum { CAP = 2u << 16 | 17, MODE = 4u << 16 | 16 }; static const uint32_t head[] = {0x07230203, 0x00010300, 0, 5, 0, CAP, 1, CAP, 10}; // Shader, Float64 static const uint32_t body[] = { 3u << 16 | 14, 0, 1, // OpMemoryModel Logical GLSL450 5u << 16 | 15, 4, 1, 0x6e69616d, 0, // OpEntryPoint Fragment %1 "main" 3u << 16 | 16, 1, 7, // OpExecutionMode %1 OriginUpperLeft 2u << 16 | 19, 2, // %2 = OpTypeVoid 3u << 16 | 33, 3, 2, // %3 = OpTypeFunction %2 5u << 16 | 54, 2, 1, 0, 3, // %1 = OpFunction %2 None %3 2u << 16 | 248, 4, 1u << 16 | 253, 1u << 16 | 56}; // OpLabel, OpReturn, OpFunctionEnd enum { NH = sizeof head / 4, NB = sizeof body / 4, EP_END = 8 }; // EP_END: body words up to the entry point's end uint32_t in[NH + NB], want[NH + NB + 32]; memcpy(in, head, sizeof head), memcpy(in + NH, body, sizeof body); int failures = 0; // widths: what the device allows; f64: the module declares Float64; w64: a 64-bit mode is expected for (int t = 0; t < 3; t++) { uint32_t widths = t == 0 ? NP_16 | NP_32 | NP_64 : NP_32, n = NH + NB, p = 0; int f64 = t != 2, w64 = t == 0; if (!f64) in[NH - 1] = 0; // Matrix in place of Float64 memcpy(want, in, NH * 4), p = NH; want[p++] = CAP, want[p++] = 4466; want[p++] = 7u << 16 | 10; memset(want + p, 0, 24), memcpy(want + p, "SPV_KHR_float_controls", 23), p += 6; memcpy(want + p, body, EP_END * 4), p += EP_END; want[p++] = MODE, want[p++] = 1, want[p++] = 4461, want[p++] = 32; if (w64) want[p++] = MODE, want[p++] = 1, want[p++] = 4461, want[p++] = 64; memcpy(want + p, body + EP_END, (NB - EP_END) * 4), p += NB - EP_END; uint64_t on = 0, again_n = 0; uint32_t *o = spirv_nan_preserve(in, n, widths, &on); uint32_t *again = o ? spirv_nan_preserve(o, on, widths, &again_n) : NULL; if (!o || on != p || memcmp(o, want, p * 4) || again) { printf("spirv: SignedZeroInfNanPreserve rewrite, case %d: wrong output\n", t); failures++; } free(o), free(again); } if (spirv_nan_preserve(in, NH + NB, NP_16 | NP_64, &(uint64_t){0})) { printf("spirv: SignedZeroInfNanPreserve rewrite: a module changed without the 32-bit width\n"); failures++; } printf("spirv: SignedZeroInfNanPreserve rewrite: %s\n", failures ? "FAIL" : "ok"); { // spirv_nan_bits: a scalar and a vec2 isnan, a select on a NaN constant; no uint32 type in the module static const uint32_t m[] = { 0x07230203, 0x00010300, 0, 20, 0, 2u << 16 | 17, 1, // OpCapability Shader 3u << 16 | 14, 0, 1, // OpMemoryModel Logical GLSL450 5u << 16 | 15, 5, 1, 0x6e69616d, 0, // OpEntryPoint GLCompute %1 "main" 2u << 16 | 19, 2, // %2 void 3u << 16 | 33, 3, 2, // %3 fn 3u << 16 | 22, 4, 32, // %4 float 2u << 16 | 20, 5, // %5 bool 4u << 16 | 23, 6, 4, 2, // %6 vec2 4u << 16 | 23, 7, 5, 2, // %7 bvec2 4u << 16 | 43, 4, 8, 0xffffffffu, // %8 = NaN bits 4u << 16 | 43, 4, 9, 0x3f800000u, // %9 = 1.0 5u << 16 | 44, 6, 10, 8, 9, // %10 = vec2(%8, %9) 3u << 16 | 41, 5, 11, // %11 = OpConstantTrue 5u << 16 | 54, 2, 1, 0, 3, // %1 = OpFunction 2u << 16 | 248, 12, // OpLabel 4u << 16 | 156, 5, 13, 9, // %13 = OpIsNan bool %9 4u << 16 | 156, 7, 14, 10, // %14 = OpIsNan bvec2 %10 6u << 16 | 169, 4, 15, 11, 9, 8, // %15 = OpSelect float %11 %9 %8 1u << 16 | 253, 1u << 16 | 56}; // OpReturn, OpFunctionEnd uint64_t on = 0; uint32_t tests = 0, sels = 0, isnan_left = 0, ugt = 0, int32 = 0, bitcasts = 0; uint32_t *o = spirv_nan_bits(m, sizeof m / 4, &on, &tests, &sels); for (uint64_t i = 5; o && i < on;) { uint32_t op = o[i] & 0xffff, l = o[i] >> 16; if (!l) break; isnan_left += op == 156, ugt += op == 172, int32 += op == 21 && o[i + 2] == 32, bitcasts += op == 124; i += l; } int bad = !o || tests != 2 || sels != 1 || isnan_left || ugt != 2 || int32 != 1 || bitcasts != 5 || o[3] <= 20; printf("spirv: integer NaN tests rewrite: %s\n", bad ? "FAIL" : "ok"); failures += bad; free(o); } const char *list = cfg_get("NX2NAVI_SPV_MARK_TEST"); if (list && *list) failures += spirv_mark_list_test(list); return failures; } // Chain {size 32} into a stage create info when the stage uses subgroup ops and the driver allows the pin. static void pin_subgroup32(VkPipelineShaderStageCreateInfo *st, VkPipelineShaderStageRequiredSubgroupSizeCreateInfo *rq, const uint32_t *w, uint64_t words) { if (!(P.sg32_stages & st->stage) || !w || !uses_subgroup_ops(w, words)) return; *rq = (VkPipelineShaderStageRequiredSubgroupSizeCreateInfo){ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO, .requiredSubgroupSize = vk_caps.sg_size}; st->pNext = rq; } // The x of a compute module's LocalSize (OpExecutionMode 17), 0 when it has none. static uint32_t local_size_x(const uint32_t *w, uint64_t words) { for (uint64_t i = 5; w && i < words;) { uint32_t op = w[i] & 0xffff, n = w[i] >> 16; if (!n || i + n > words) return 0; if (op == 16 && n >= 6 && w[i + 2] == 17) return w[i + 3]; if (op == 54) return 0; // OpFunction: the execution modes are all before it i += n; } return 0; } // Pool arrays (the translator's bindless texture and sampler arrays of VK_POOL_TEX / VK_POOL_SAMP elements, found by // their types) move to set 2, at the binding of their view kind (vk.h VK_POOL_KINDS) or VK_POOL_KINDS for samplers; // the kinds used are noted in vk_pool_kinds_used so the replay fills them. NULL: nothing to move (or no memory). static uint32_t *spirv_pool_rebind(const uint32_t *w, uint64_t n) { enum { OP_TYPE_IMAGE = 25, OP_TYPE_SAMPLER = 26, OP_TYPE_ARRAY_ = 28 }; uint32_t bound = n >= 5 && w[3] < (1u << 22) ? w[3] : 0; if (!bound) return NULL; int32_t *kind = malloc(bound * sizeof *kind); // per id: image kind, 100 sampler, -1 other (types, then variables) uint32_t *cval = calloc(bound, sizeof *cval); if (!kind || !cval) return free(kind), free(cval), NULL; memset(kind, 0xff, bound * sizeof *kind); // Pool variables and their binding decorations are module declarations, all before the first function. int any = 0; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (!len || i + len > n || op == SPV_OP_FUNCTION) break; if (op == OP_TYPE_IMAGE && len >= 9 && w[i + 1] < bound && w[i + 7] == 1) { // sampled image uint32_t dim = w[i + 3], depth = w[i + 4], arrayed = w[i + 5], ms = w[i + 6]; if (dim <= 3 && !ms) kind[w[i + 1]] = (int32_t)((depth == 1 ? 8 : 0) + dim * 2 + (arrayed ? 1 : 0)); } else if (op == OP_TYPE_SAMPLER && len >= 2 && w[i + 1] < bound) { kind[w[i + 1]] = 100; } else if (op == SPV_OP_CONSTANT && len >= 4 && w[i + 2] < bound) { cval[w[i + 2]] = w[i + 3]; } else if (op == OP_TYPE_ARRAY_ && len >= 4 && w[i + 1] < bound && w[i + 2] < bound && w[i + 3] < bound) { int32_t k = kind[w[i + 2]]; uint32_t want = k == 100 ? VK_POOL_SAMP : VK_POOL_TEX; kind[w[i + 1]] = k >= 0 && cval[w[i + 3]] == want ? k + 1000 : -1; // + 1000: a pool array } else if (op == SPV_OP_TYPE_POINTER && len >= 4 && w[i + 1] < bound && w[i + 3] < bound) { kind[w[i + 1]] = kind[w[i + 3]] >= 1000 ? kind[w[i + 3]] : -1; } else if (op == SPV_OP_VARIABLE && len >= 4 && w[i + 1] < bound && w[i + 2] < bound) { kind[w[i + 2]] = kind[w[i + 1]] >= 1000 ? kind[w[i + 1]] + 1000 : -1; // + 2000: a pool variable if (kind[w[i + 2]] >= 2000) any = 1; } i += len; } uint32_t *o = NULL; if (any && (o = malloc(n * 4))) { memcpy(o, w, n * 4); uint32_t used = 0; for (uint64_t i = 5; i < n;) { uint32_t op = o[i] & 0xffff, len = o[i] >> 16; if (!len || i + len > n || op == SPV_OP_FUNCTION) break; if (op == SPV_OP_DECORATE && len >= 4 && o[i + 1] < bound && kind[o[i + 1]] >= 2000) { int32_t k = kind[o[i + 1]] - 2000; if (o[i + 2] == SPV_DEC_DESCRIPTOR_SET) o[i + 3] = 2; else if (o[i + 2] == SPV_DEC_BINDING) o[i + 3] = k == 100 ? VK_POOL_KINDS : (uint32_t)k; if (k != 100) used |= 1u << k; } i += len; } __atomic_fetch_or(&vk_pool_kinds_used, used, __ATOMIC_RELAXED); } free(kind), free(cval); return o; } // ---- texture pool marks (NX2NAVI_TEX_DISUSE, default on; vk_texture.c "disuse" uses them) // Ocarina samples nearly every texture through the bindless pool: a shader reads a handle from memory and indexes the // pool array with it, so the CPU never learns which of the registered textures a frame samples, and every registered // texture counts as in use (vk_texture.c idle). The game keeps the textures of the areas it has loaded registered, so // none of them could go, and their memory, which the driver charges to the system's commit, grew to 6 GiB in a session. // This rewrite gives each module that indexes the texture pool a storage buffer at set 2, binding VK_POOL_MARK_BINDING // (a uint per pool slot, vk_exec_draw.c "pool marks") and, after every access chain into a pool texture array, a store // of 1 into the slot it indexes. The stores of one warp to one slot are one memory transaction. The replay copies the // buffer out and clears it every few submissions; slots marked mean textures sampled. A module that indexes the pool // and cannot be marked (SPIR-V before 1.3 or after 1.6, a stage whose stores the device does not allow, no memory) is // counted in vk_pool_marks_missed, which keeps vk_texture.c from evicting by disuse: an unmarked shader could sample a // texture the marks call unused. make_module logs the first few by stage, version, size, hash and reason. volatile int32_t vk_pool_marks_missed; static int marks_on = -1, marks_vs_ok, marks_fs_ok; int vk_pool_marks_enabled(void) { if (marks_on < 0) { marks_on = rt_knob("TEX_DISUSE", "1") >= 1; // 2: marks and stand-ins, nothing freed (vk_texture.c "disuse") VkCtx *c = vk_ctx(); if (c) { VkPhysicalDeviceFeatures f; vkGetPhysicalDeviceFeatures(c->phys, &f); // vk_device.c enables every feature the GPU reports marks_vs_ok = f.vertexPipelineStoresAndAtomics, marks_fs_ok = f.fragmentStoresAndAtomics; } } return marks_on; } // A fragment shader that stores to memory runs before the depth test unless it declares EarlyFragmentTests: the GPU // must run it for fragments the test then rejects, as the stores are visible. The marks would so take the early depth // rejection from every marked fragment shader and make overdraw cost full shading. With early_ok, a fragment shader // whose result is the same under early tests gets EarlyFragmentTests along with its marks: one that discards, writes // its depth, sample mask or stencil reference, or already writes memory does not, as early tests would change what it // does. The caller passes early_ok only for a pipeline without alpha to coverage, which also drops samples after the // shader (ensure_init keeps alpha to coverage out of the dynamic state while marks are on, so a pipeline knows). // *status: 1 marked, 0 no pool texture access to mark, -1 one that could not be marked (*why says what stopped it). // NULL unless marked. static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, uint64_t *out_n, int *status, const char **why) { enum { OP_TYPE_RUNTIME_ARRAY = 29, OP_TYPE_STRUCT_ = 30, OP_ACCESS_CHAIN_ = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66, OP_STORE_ = 62, OP_MEMBER_DECORATE_ = 72, OP_DECORATION_GROUP = 73, OP_GROUP_DECORATE = 74, OP_GROUP_MEMBER_DECORATE = 75, OP_DECORATE_ID = 332, OP_DECORATE_STRING = 5632, OP_MEMBER_DECORATE_STRING = 5633, OP_EXECUTION_MODE = 16, OP_EXECUTION_MODE_ID = 331, OP_IMAGE_WRITE = 99, OP_ATOMIC_FIRST = 227, OP_ATOMIC_LAST = 242, OP_KILL = 252, OP_TERMINATE_INVOCATION = 4416, OP_DEMOTE_TO_HELPER = 5380, EM_EARLY_FRAGMENT_TESTS = 9, BI_SAMPLE_MASK = 20, BI_FRAG_DEPTH = 22, BI_FRAG_STENCIL_REF = 5014, SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35 }; *status = 0, *why = NULL; if (n < 5) return NULL; uint32_t bound = w[3]; if (!bound || bound > (1u << 22)) return NULL; uint32_t *bind = calloc(bound, sizeof *bind); // per id: its Binding + 1 when its DescriptorSet is 2, else 0 uint8_t *set2 = calloc(bound, 1); if (!bind || !set2) return free(bind), free(set2), *status = -1, *why = "no memory", NULL; uint64_t ann_end = 0, fn_start = 0, em_end = 0; // em_end: past the entry points and execution modes uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0; int vertex_stage = 0, fragment_stage = 0, entries = 0, early = early_ok; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) return free(bind), free(set2), *status = -1, *why = "an instruction past the module's end", NULL; if (op == SPV_OP_FUNCTION) { fn_start = i; break; } if (op == SPV_OP_ENTRY_POINT || op == OP_EXECUTION_MODE || op == OP_EXECUTION_MODE_ID) em_end = i + l; if (op == SPV_OP_ENTRY_POINT && l >= 3) { entries++; if (w[i + 1] <= 3) vertex_stage = 1; // Vertex, TessellationControl, TessellationEvaluation, Geometry else if (w[i + 1] == 4) fragment_stage = 1, fs_entry = w[i + 2]; } else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == EM_EARLY_FRAGMENT_TESTS) { early = 0; // has them already } else if (op == SPV_OP_DECORATE && l >= 4 && w[i + 1] < bound) { if (w[i + 2] == SPV_DEC_DESCRIPTOR_SET && w[i + 3] == 2) set2[w[i + 1]] = 1; else if (w[i + 2] == SPV_DEC_BINDING) bind[w[i + 1]] = w[i + 3] + 1; else if (w[i + 2] == SPV_DEC_BUILTIN && (w[i + 3] == BI_SAMPLE_MASK || w[i + 3] == BI_FRAG_DEPTH || w[i + 3] == BI_FRAG_STENCIL_REF)) early = 0; } else if (op == SPV_OP_VARIABLE && l >= 4 && w[i + 3] == SC_STORAGE_BUFFER) { early = 0; // the guest's own storage buffers: its stores would change under early tests } else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 0) { uint_t = w[i + 1]; } else if (op == SPV_OP_CONSTANT && l == 4 && uint_t && w[i + 1] == uint_t) { if (w[i + 3] == 0 && !c0) c0 = w[i + 2]; if (w[i + 3] == 1 && !c1) c1 = w[i + 2]; } else if (op == SPV_OP_TYPE_POINTER && l == 4 && w[i + 2] == SC_STORAGE_BUFFER && uint_t && w[i + 3] == uint_t) { ptr_u = w[i + 1]; } if (op == SPV_OP_DECORATE || op == OP_MEMBER_DECORATE_ || op == OP_DECORATION_GROUP || op == OP_GROUP_DECORATE || op == OP_GROUP_MEMBER_DECORATE || op == OP_DECORATE_ID || op == OP_DECORATE_STRING || op == OP_MEMBER_DECORATE_STRING) ann_end = i + l; i += l; } // pool texture variables: set 2, a binding below VK_POOL_KINDS (spirv_pool_rebind); one already at the marks // binding means the module was marked before uint64_t sites = 0; int marked = 0; for (uint32_t id = 0; id < bound; id++) if (set2[id] && bind[id] == VK_POOL_MARK_BINDING + 1) marked = 1; for (uint64_t i = fn_start; fn_start && !marked && i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) break; if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] && bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) sites++; else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE || (op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST)) early = 0; i += l; } if (marked || !sites) return free(bind), free(set2), NULL; *why = w[1] < 0x00010300 || w[1] > 0x00010600 ? "a SPIR-V version before 1.3 or after 1.6" : !ann_end || !fn_start ? "no decorations or no function" : vertex_stage && !marks_vs_ok ? "the GPU allows no stores in vertex-pipeline shaders" : fragment_stage && !marks_fs_ok ? "the GPU allows no stores in fragment shaders" : NULL; if (*why) return free(bind), free(set2), *status = -1, NULL; early = early && fragment_stage && entries == 1 && em_end && em_end < ann_end; // From SPIR-V 1.4 on an entry point lists every global variable its functions use, not only the inputs and outputs // (the translator's mesh shaders are 1.4): each entry point gets the marks variable, a word more. int iface = w[1] >= 0x00010400; // the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site uint64_t on = n + 3 + (uint64_t)entries + 20 + 30 + 9 * sites; uint32_t *o = malloc(on * 4); if (!o) return free(bind), free(set2), *status = -1, *why = "no memory", NULL; uint32_t next = bound; uint32_t new_uint = uint_t ? 0 : (uint_t = next++), new_c0 = c0 ? 0 : (c0 = next++), new_c1 = c1 ? 0 : (c1 = next++); uint32_t rta = next++, st = next++, ptr_st = next++, var = next++, new_ptr_u = ptr_u ? 0 : (ptr_u = next++); uint64_t p = 5; memcpy(o, w, 5 * 4); for (uint64_t i = 5; i < ann_end;) { // the header up to the annotations' end: entry points widened, the mode added uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (early && i == em_end) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = fs_entry, o[p++] = EM_EARLY_FRAGMENT_TESTS; memcpy(o + p, w + i, l * 4); if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) o[p] = (l + 1) << 16 | op, o[p + l] = var, p++; p += l; i += l; } o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = rta, o[p++] = DEC_ARRAY_STRIDE, o[p++] = 4; o[p++] = 5u << 16 | OP_MEMBER_DECORATE_, o[p++] = st, o[p++] = 0, o[p++] = DEC_OFFSET_, o[p++] = 0; o[p++] = 3u << 16 | SPV_OP_DECORATE, o[p++] = st, o[p++] = DEC_BLOCK; o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = var, o[p++] = SPV_DEC_DESCRIPTOR_SET, o[p++] = 2; o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = var, o[p++] = SPV_DEC_BINDING, o[p++] = VK_POOL_MARK_BINDING; memcpy(o + p, w + ann_end, (fn_start - ann_end) * 4), p += fn_start - ann_end; if (new_uint) o[p++] = 4u << 16 | SPV_OP_TYPE_INT, o[p++] = uint_t, o[p++] = 32, o[p++] = 0; if (new_c0) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c0, o[p++] = 0; if (new_c1) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c1, o[p++] = 1; o[p++] = 3u << 16 | OP_TYPE_RUNTIME_ARRAY, o[p++] = rta, o[p++] = uint_t; o[p++] = 3u << 16 | OP_TYPE_STRUCT_, o[p++] = st, o[p++] = rta; o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_st, o[p++] = SC_STORAGE_BUFFER, o[p++] = st; if (new_ptr_u) o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_u, o[p++] = SC_STORAGE_BUFFER, o[p++] = uint_t; o[p++] = 4u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_st, o[p++] = var, o[p++] = SC_STORAGE_BUFFER; for (uint64_t i = fn_start; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; memcpy(o + p, w + i, l * 4), p += l; if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] && bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) { uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4]; o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1; } i += l; } o[3] = next; free(bind), free(set2); *out_n = p; *status = 1; return o; } // --selftest with NX2NAVI_SPV_MARK_TEST=: each line " "; the module is moved to the pools // (spirv_pool_rebind) and marked, and written to when marked, for spirv-val. Prints the counts; 0 failures unless // a file cannot be read or written. static int spirv_mark_list_test(const char *list) { marks_vs_ok = marks_fs_ok = 1; FILE *f = fopen(list, "rb"); if (!f) return printf("spirv: pool marks test: %s cannot be read\n", list), 1; char line[2048]; int marked = 0, none = 0, missed = 0, bad = 0; while (fgets(line, sizeof line, f)) { char in[1024], out[1024]; if (sscanf(line, "%1023s %1023s", in, out) != 2) continue; FILE *g = fopen(in, "rb"); if (!g) { bad++; continue; } fseek(g, 0, SEEK_END); long sz = ftell(g); fseek(g, 0, SEEK_SET); uint32_t *w = sz > 20 ? malloc((size_t)sz) : NULL; if (!w || fread(w, 1, (size_t)sz, g) != (size_t)sz) { fclose(g), free(w), bad++; continue; } fclose(g); uint64_t n = (uint64_t)sz / 4, mn = 0; uint32_t *pooled = spirv_pool_rebind(w, n); int st; const char *why; uint32_t *m = spirv_pool_mark(pooled ? pooled : w, n, 1, &mn, &st, &why); if (st > 0) { marked++; if (!plat_file_write_atomic(out, m, (size_t)mn * 4)) { if (!bad) printf("spirv: pool marks test: %s cannot be written\n", out); bad++; } } else if (st < 0) { missed++; printf("spirv: pool marks test: %s not markable: %s\n", in, why); } else { none++; } free(m), free(pooled), free(w); } fclose(f); printf("spirv: pool marks test: %d marked, %d without pool texture accesses, %d not markable, %d file errors\n", marked, none, missed, bad); return bad; } // early_ok: a fragment shader whose pipeline has no alpha to coverage (spirv_pool_mark) static VkShaderModule make_module(const uint32_t *w, uint64_t words, int early_ok) { const uint32_t *orig_w = w; // for the log of a module the marks cannot take uint64_t orig_words = words, fixed_n = 0; uint32_t *fixed = spirv_fixup(w, words, &fixed_n); if (fixed) w = fixed, words = fixed_n; uint32_t *pooled = spirv_pool_rebind(w, words); if (pooled) { free(fixed); w = fixed = pooled; if (vk_pool_marks_enabled()) { uint64_t mk_n = 0; int st; const char *why; uint32_t *mk = spirv_pool_mark(w, words, early_ok, &mk_n, &st, &why); if (mk) { free(fixed); w = fixed = mk, words = mk_n; } else if (st < 0) { int nth = plat_inc32(&vk_pool_marks_missed); // the hash is FNV-1a of the words make_module got, as vk_dump hashes a shader's SPIR-V; it differs // from the translation's only where the pipeline patched the module first (added vertex outputs) if (nth <= 8) { uint32_t model = ~0u; for (uint64_t i = 5; i < words;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l) break; if (op == SPV_OP_ENTRY_POINT && l >= 2) { model = w[i + 1]; break; } i += l; } const char *stage = model == 0 ? "vertex" : model <= 3 ? "tessellation or geometry" : model == 4 ? "fragment" : model == 5 ? "compute" : model == 5364 ? "task" : model == 5365 ? "mesh" : "other"; rt_log("vk_pipeline: a %s shader that indexes the texture pool could not be marked (SPIR-V %u.%u, %llu " "words, hash %016llx): %s%s", stage, w[1] >> 16 & 0xff, w[1] >> 8 & 0xff, (unsigned long long)words, (unsigned long long)fnv(FNV0, orig_w, (size_t)orig_words * 4), why, nth == 1 ? "; textures are not evicted by disuse this run (NX2NAVI_TEX_DISUSE)" : ""); } } } } uint64_t rc_n = 0; uint32_t *rc = P.reconverge ? spirv_reconverge(w, words, &rc_n) : NULL; if (rc) { free(fixed); w = fixed = rc, words = rc_n; } uint64_t np_n = 0; uint32_t *np = P.nan_preserve ? spirv_nan_preserve(w, words, P.nan_preserve, &np_n) : NULL; if (np) { free(fixed); w = fixed = np, words = np_n; } uint64_t nb_n = 0; uint32_t nb_tests = 0, nb_sels = 0; uint32_t *nb = P.nan_bits ? spirv_nan_bits(w, words, &nb_n, &nb_tests, &nb_sels) : NULL; if (nb) { free(fixed); w = fixed = nb, words = nb_n; __atomic_fetch_add(&P.nan_bits_tests, nb_tests, __ATOMIC_RELAXED); __atomic_fetch_add(&P.nan_bits_selects, nb_sels, __ATOMIC_RELAXED); if (__atomic_add_fetch(&P.nan_bits_modules, 1, __ATOMIC_RELAXED) == 1) rt_log("vk_pipeline: first module with NaN tests made integer: %u tests, %u selects of NaN bits", nb_tests, nb_sels); } if (P.feedback >= 2) mm_hash = fnv(FNV0, w, (size_t)words * 4); VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)words * 4, .pCode = w}; VkShaderModule m = VK_NULL_HANDLE; VkResult r = vkCreateShaderModule(P.device, &ci, NULL, &m); free(fixed); return r == VK_SUCCESS ? m : VK_NULL_HANDLE; } // ---- descriptor layouts typedef struct { uint32_t n; VkDescriptorSetLayoutBinding b[MAX_BINDINGS]; VkPipeBinding pb[MAX_BINDINGS]; int pool; // a stage indexes the texture or sampler pool (set 2, not in b / pb) } Bindings; static VkDescriptorType descriptor_type(int kind, const GfxShaderRes *r) { int buffer = (r->type & 0xff) == 2; switch (kind) { case GFX_RES_UBO: return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; case GFX_RES_SSBO: return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; case GFX_RES_TEX: if (buffer) return VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER; if (r->separate) LOG_ONCE("vk_pipeline: a separate texture handle: bound as SAMPLED_IMAGE"); return r->separate ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; default: return buffer ? VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; } } static VkShaderStageFlags stage_bit(int stage) { return stage == GFX_STAGE_VS ? VK_SHADER_STAGE_VERTEX_BIT : stage == GFX_STAGE_FS ? VK_SHADER_STAGE_FRAGMENT_BIT : stage == GFX_STAGE_TS ? VK_SHADER_STAGE_TASK_BIT_EXT : stage == GFX_STAGE_MS ? VK_SHADER_STAGE_MESH_BIT_EXT : VK_SHADER_STAGE_COMPUTE_BIT; } static int is_mesh(const VkPipeState *s) { return s->vs && gfx_shader_stage(s->vs) == GFX_STAGE_MS; } // The shaders of the pipeline this thread builds, for the crash notes (platform.h plat_crash_context): the first 8 // bytes of each cache key, as gfx_shader.c names its cache files. static void crash_note(const char *what, const GfxShader *a, const GfxShader *b, const GfxShader *c) { static __thread char buf[160]; char *o = buf; o += snprintf(o, 40, "building %s", what); const GfxShader *sh[3] = {a, b, c}; for (int i = 0; i < 3; i++) if (sh[i]) { const uint8_t *k = gfx_shader_key(sh[i]); o += snprintf(o, 40, " stage %d %02x%02x%02x%02x%02x%02x%02x%02x", gfx_shader_stage(sh[i]), k[0], k[1], k[2], k[3], k[4], k[5], k[6], k[7]); } plat_crash_context(buf); } // One past the highest binding a shader uses (1 with no resources: binding 0 is the support buffer). static uint32_t binding_span(const GfxShader *s) { uint32_t span = 1; for (int k = 0; k < GFX_RES_KINDS; k++) { const GfxShaderRes *r; int n = gfx_shader_resources(s, k, &r); for (int i = 0; i < n; i++) if (r[i].binding + (uint32_t)(r[i].count ? r[i].count : 1) > span) span = r[i].binding + (r[i].count ? r[i].count : 1); } return span; } // Adds a stage's resources with bindings b >= 1 moved by off; binding 0 (support buffer) is shared. static int add_stage(Bindings *B, const GfxShader *s, uint32_t off) { int stage = gfx_shader_stage(s); VkShaderStageFlags bit = stage_bit(stage); if (!B->n) { B->b[0] = (VkDescriptorSetLayoutBinding){0, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1, 0, NULL}; B->pb[0] = (VkPipeBinding){0, (uint8_t)stage, VK_PIPE_SUPPORT, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1, 0, NULL}; B->n = 1; } B->b[0].stageFlags |= bit; B->pb[0].stages |= bit; for (int k = 0; k < GFX_RES_KINDS; k++) { const GfxShaderRes *r; int n = gfx_shader_resources(s, k, &r); for (int i = 0; i < n; i++) { if (r[i].pool) { // set 2 (spirv_pool_rebind), bound by the replay from the pools B->pool = 1; continue; } if (B->n == MAX_BINDINGS || !r[i].binding) return 0; VkDescriptorType t = descriptor_type(k, &r[i]); uint32_t binding = r[i].binding + off, count = r[i].count ? r[i].count : 1; B->b[B->n] = (VkDescriptorSetLayoutBinding){binding, t, count, bit, NULL}; B->pb[B->n] = (VkPipeBinding){(uint16_t)binding, (uint8_t)stage, (uint8_t)k, t, (uint16_t)count, bit, &r[i]}; B->n++; } } return 1; } // The first binding a module's resource variables use that the binding list b[0..n) lacks (module bindings >= 1 moved // by off), or -1. A reflection that misses a resource would otherwise hand the driver an invalid layout. static int module_binding_missing(const GfxShader *s, const VkDescriptorSetLayoutBinding *b, uint32_t n, uint32_t off) { uint64_t words = 0; const uint32_t *w = gfx_shader_spirv(s, &words); const GfxShaderRes *tr; int nt = gfx_shader_resources(s, GFX_RES_TEX, &tr); for (uint64_t i = 5; w && i < words;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (!len || i + len > words) break; int pooled = 0; // a pool array: set 2, not in this layout (spirv_pool_rebind) for (int t = 0; t < nt && op == SPV_OP_DECORATE && len >= 4; t++) if (tr[t].pool && tr[t].binding == (w[i + 3] & 0xffff)) pooled = 1; // GfxShaderRes.binding is 16 bits if (op == SPV_OP_DECORATE && len >= 4 && w[i + 2] == SPV_DEC_BINDING && !pooled) { uint32_t want = w[i + 3] + (w[i + 3] >= 1 ? off : 0), j = 0; while (j < n && b[j].binding != want) j++; if (j == n) return (int)w[i + 3]; } i += len; } return -1; } // Checks both stages of a graphics pipeline against its layout; 0 with why filled when one uses a missing binding. static int layout_covers(const VkPipeState *s, const Layout *l, uint32_t fs_off, char *why, size_t cap) { uint32_t nv = l->set1 ? l->n0 : l->n; int m = module_binding_missing(s->vs, l->b, nv, 0); const char *stage = "vertex"; if (m < 0 && s->fs) { m = l->set1 ? module_binding_missing(s->fs, l->b + l->n0, l->n - l->n0, 0) : module_binding_missing(s->fs, l->b, l->n, fs_off); stage = "fragment"; } if (m >= 0) snprintf(why, cap, "the %s shader uses binding %d, which its reflection and layout lack", stage, m); return m < 0; } // The update template of a set whose bindings are b[0..n): one entry per binding, in order, over the packed array the // replay fills from the set's first descriptor, a VkDescSlot per descriptor. The bind point follows the stages (compute // layouts hold the compute stage only). VK_NULL_HANDLE when templates are off, for an empty set, for a push set // without vkCmdPushDescriptorSetWithTemplateKHR or when creation fails: the replay then writes the set with write // structs. static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl, int push, VkPipelineLayout pl, uint32_t set) { if (!P.tpl || !n || (push && !P.tpl_push)) return VK_NULL_HANDLE; VkDescriptorUpdateTemplateEntry e[MAX_BINDINGS]; size_t off = 0; VkShaderStageFlags stages = 0; for (uint32_t i = 0; i < n; i++) { e[i] = (VkDescriptorUpdateTemplateEntry){b[i].binding, 0, b[i].descriptorCount, b[i].descriptorType, off, sizeof(VkDescSlot)}; off += sizeof(VkDescSlot) * b[i].descriptorCount; stages |= b[i].stageFlags; } VkDescriptorUpdateTemplateCreateInfo ci = { .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_UPDATE_TEMPLATE_CREATE_INFO, .descriptorUpdateEntryCount = n, .pDescriptorUpdateEntries = e, .templateType = push ? VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR : VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET, .descriptorSetLayout = sl, .pipelineBindPoint = stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, .pipelineLayout = pl, .set = set}; VkDescriptorUpdateTemplate t; if (P.tpl_create(P.device, &ci, NULL, &t) != VK_SUCCESS) { LOG_ONCE("vk_pipeline: descriptor update template creation failed: that set is written with write structs"); return VK_NULL_HANDLE; } return t; } // Finds or creates the layout of a binding list (sorted here); NULL on failure. Caller holds P.lock. // Sorts a binding list by binding and hashes it into *h; the descriptor total, or -1 when a binding repeats. static int64_t bindings_sort(Bindings *B, uint64_t *h) { for (uint32_t i = 1; i < B->n; i++) // insertion sort by binding; the lists are short for (uint32_t j = i; j > 0 && B->b[j - 1].binding > B->b[j].binding; j--) { VkDescriptorSetLayoutBinding t = B->b[j]; B->b[j] = B->b[j - 1]; B->b[j - 1] = t; VkPipeBinding u = B->pb[j]; B->pb[j] = B->pb[j - 1]; B->pb[j - 1] = u; } int64_t total = 0; for (uint32_t i = 0; i < B->n; i++) { if (i && B->b[i].binding == B->b[i - 1].binding) return -1; uint32_t k[4] = {B->b[i].binding, B->b[i].descriptorType, B->b[i].descriptorCount, B->b[i].stageFlags}; *h = fnv(*h, k, sizeof k); total += B->b[i].descriptorCount; } return total; } static Layout *layout_get(Bindings *B) { uint64_t h = FNV0; int64_t total = bindings_sort(B, &h); if (total < 0) return NULL; Layout **head = &layouts[h % LAYOUT_BUCKETS]; for (Layout *l = *head; l; l = l->next) if (l->hash == h && l->n == B->n && !memcmp(l->b, B->b, B->n * sizeof *B->b)) return l; Layout *l = calloc(1, sizeof *l + B->n * sizeof *B->b); if (!l) return NULL; l->hash = h, l->n = B->n; memcpy(l->b, B->b, B->n * sizeof *B->b); l->push = total <= (int64_t)P.max_push; VkDescriptorSetLayoutCreateInfo si = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0, .bindingCount = l->n, .pBindings = l->b}; if (vkCreateDescriptorSetLayout(P.device, &si, NULL, &l->set) != VK_SUCCESS) { free(l); return NULL; } VkDescriptorSetLayout sets3[3] = {l->set, vk_empty_set_layout(), vk_pool_set_layout()}; // set 2: the pools VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 3, .pSetLayouts = sets3}; if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) { vkDestroyDescriptorSetLayout(P.device, l->set, NULL); free(l); return NULL; } l->tpl[0] = make_tpl(l->b, l->n, l->set, l->push, l->layout, 0); l->next = *head; *head = l; P.stats.layouts++; P.stats.push_layouts += (uint64_t)l->push; return l; } // Independent layouts (P.indep): the vertex stage's bindings in set 0, the fragment stage's in set 1 (empty without a // fragment shader), each with its own support buffer at binding 0, the pipeline layout created with // VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT so stage libraries can use it. Only the P.push_stage set is a // push-descriptor set (a layout holds one), when it fits maxPushDescriptors; the other is allocated per draw from the // command buffer's pools. On success *B holds both lists, set 0 first, VkPipeBinding.set filled in. Caller holds P.lock. static Layout *layout_pair(Bindings *B, Bindings *F) { uint64_t h = fnv(FNV0, "indep", 5); int64_t tv = bindings_sort(B, &h); h = fnv(h, "|", 1); int64_t tf = bindings_sort(F, &h); if (tv < 0 || tf < 0 || B->n + F->n > MAX_BINDINGS) return NULL; uint32_t n0 = B->n, n = B->n + F->n; memcpy(B->b + n0, F->b, F->n * sizeof *F->b); memcpy(B->pb + n0, F->pb, F->n * sizeof *F->pb); for (uint32_t i = n0; i < n; i++) B->pb[i].set = 1; B->n = n; Layout **head = &layouts[h % LAYOUT_BUCKETS]; for (Layout *l = *head; l; l = l->next) if (l->hash == h && l->set1 && l->n == n && l->n0 == n0 && !memcmp(l->b, B->b, n * sizeof *B->b)) return l; Layout *l = calloc(1, sizeof *l + n * sizeof *B->b); if (!l) return NULL; l->hash = h, l->n = n, l->n0 = n0; memcpy(l->b, B->b, n * sizeof *B->b); l->push = P.push_stage == GFX_STAGE_VS && tv <= (int64_t)P.max_push; l->push1 = P.push_stage == GFX_STAGE_FS && F->n && tf <= (int64_t)P.max_push; VkDescriptorSetLayoutCreateInfo s0 = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0, .bindingCount = n0, .pBindings = l->b}; VkDescriptorSetLayoutCreateInfo s1 = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .flags = l->push1 ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0, .bindingCount = n - n0, .pBindings = l->b + n0}; if (vkCreateDescriptorSetLayout(P.device, &s0, NULL, &l->set) != VK_SUCCESS) { free(l); return NULL; } if (vkCreateDescriptorSetLayout(P.device, &s1, NULL, &l->set1) != VK_SUCCESS) { vkDestroyDescriptorSetLayout(P.device, l->set, NULL); free(l); return NULL; } VkDescriptorSetLayout sets[3] = {l->set, l->set1, vk_pool_set_layout()}; VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 3, .pSetLayouts = sets}; if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) { vkDestroyDescriptorSetLayout(P.device, l->set, NULL); vkDestroyDescriptorSetLayout(P.device, l->set1, NULL); free(l); return NULL; } l->tpl[0] = make_tpl(l->b, n0, l->set, l->push, l->layout, 0); l->tpl[1] = make_tpl(l->b + n0, n - n0, l->set1, l->push1, l->layout, 1); l->next = *head; *head = l; P.stats.layouts++; P.stats.push_layouts += (uint64_t)(l->push | l->push1); return l; } // Vertex outputs with independent layouts: each vertex shader declares the union of the inputs of every fragment // shader met with it (pipeline-keys.bin at the prewarm, then each pipeline built), so its module does not depend on the // fragment shader of one pipeline. Unwritten ones read (0, 0, 0, 1) as before. NX2NAVI_VS_OUTS=all declares all 32. #define VS_OUTS_CAP 8192 static struct { PlatMutex lock; const GfxShader *vs[VS_OUTS_CAP]; uint32_t mask[VS_OUTS_CAP]; GfxShaderIo io[VS_OUTS_CAP][32]; uint32_t n, full; } VO = {.lock = PLAT_MUTEX_INIT}; // Adds fs's inputs (packed ones excepted) to vs's union; returns the union, types in io (the first type met wins). static uint32_t vs_outs(const GfxShader *vs, const GfxShader *fs, GfxShaderIo io[32]) { GfxShaderIo fin[32]; uint32_t need = fs ? gfx_shader_inputs(fs, fin) : 0, mask; for (int i = 0; i < 32; i++) if (need >> i & 1 && fin[i].packed) need &= ~(1u << i); plat_mutex_lock(&VO.lock); uint32_t j = (uint32_t)(((uintptr_t)vs >> 4) * 0x9e3779b1u) & (VS_OUTS_CAP - 1); while (VO.vs[j] && VO.vs[j] != vs) j = (j + 1) & (VS_OUTS_CAP - 1); if (!VO.vs[j] && VO.n + 1 >= VS_OUTS_CAP * 3 / 4) { // full: this pair alone VO.full++; plat_mutex_unlock(&VO.lock); LOG_ONCE("vk_pipeline: vertex output union table full: outputs from the pair"); memset(io, 0, 32 * sizeof *io); for (int i = 0; i < 32; i++) if (need >> i & 1) io[i] = fin[i]; return need; } if (!VO.vs[j]) VO.vs[j] = vs, VO.n++; for (int i = 0; i < 32; i++) if (!(need >> i & 1)) continue; else if (!(VO.mask[j] >> i & 1)) VO.io[j][i] = fin[i], VO.mask[j] |= 1u << i; else if (VO.io[j][i].type == fin[i].type && VO.io[j][i].comps < fin[i].comps) VO.io[j][i].comps = fin[i].comps; mask = VO.mask[j]; memcpy(io, VO.io[j], 32 * sizeof *io); plat_mutex_unlock(&VO.lock); return mask; } // ---- pipeline cache table static Pipe *pipe_find(const void *key, size_t size, uint64_t h) { for (Pipe *p = pipes[h % PIPE_BUCKETS]; p; p = p->next) if (p->hash == h && p->key_size == size && !memcmp(p->key, key, size)) return p; return NULL; } // Stores a result (a failed one too, so it is not retried every draw). Caller holds P.lock. static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipeline, const Layout *l, const Bindings *B, VkPipelineBindPoint bp, uint32_t vs_inputs) { Pipe *p = calloc(1, sizeof *p + size); if (!p) return NULL; p->hash = h, p->key_size = size; memcpy(p->key, key, size); if (l && B) { p->bindings = malloc(B->n * sizeof *p->bindings); if (p->bindings) memcpy(p->bindings, B->pb, B->n * sizeof *p->bindings); } p->info = (VkPipeInfo){pipeline, l ? l->layout : VK_NULL_HANDLE, l ? l->set : VK_NULL_HANDLE, l ? l->push : 0, bp, p->bindings ? B->n : 0, p->bindings, vs_inputs}; p->info.set_layout1 = l ? l->set1 : VK_NULL_HANDLE; p->info.push1 = l ? l->push1 : 0; p->info.n_sets = l && l->set1 ? 2 : 1; p->info.pool = B ? B->pool : 0; for (uint32_t i = 0; i < p->info.n_bindings; i++) if (p->bindings[i].kind == GFX_RES_SSBO && p->bindings[i].res->cbuf_slot == VK_GPA_CB_SLOT) p->info.gpa = 1; if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1]; p->lay = l; Pipe **head = &pipes[h % PIPE_BUCKETS]; p->next = *head; *head = p; return p; } // ---- graphics pipelines typedef struct { // normalized: zero where the state is dynamic or unused const GfxShader *vs, *fs, *ts; // ts: the task stage of a mesh draw uint32_t color[8], depth, stencil; // VkFormat uint8_t n_color, samples, topo_class, minus_w; uint8_t clamp, poly_mode, a2c, logic_op; // logic_op: VkLogicOp + 1 when enabled uint8_t blend_en, dsb, fbl, pad; // dsb: an enabled target blends with a SRC1 factor; fbl: VkPipeState.feedback uint32_t wmask; uint8_t blend[8][6]; // src, dst, op, src alpha, dst alpha, alpha op (VkBlendFactor / VkBlendOp) uint32_t attr_fmt[32], attr_off[32]; uint8_t attr_bind[32]; uint8_t rate[17]; // per binding: 0 unused, 1 vertex, 2 instance uint8_t indep; // P.indep when built: the probe draws through both layouts uint8_t pad2[2]; } GKey; static int blendable(VkFormat f) { return f != VK_FORMAT_UNDEFINED && (format_features(f, 1) & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT); } static int target_count(const VkPipeState *s) { int n = 0; for (int i = 0; i < 8; i++) if (s->color_fmt[i]) n = i + 1; return n; } // SRC1 blend factors read the fragment shader's second output (dual-source blending); the pipeline then needs a // fragment module whose output 1 is Location 0, Index 1 (spirv_dual_source). static int is_src1(unsigned f) { return f >= NVN_BLEND_FUNC_SRC1_COLOR && f <= NVN_BLEND_FUNC_ONE_MINUS_SRC1_ALPHA; } // Runs per draw (gkey_fill, target_wmask): only the blend factors of blend-enabled targets are read until one is SRC1, // which NX2navi never sets (0 of 876 pipelines in the field and fight saves). // NX2NAVI_VK_NOOPT=1: pipelines built without the driver's optimiser, for a driver whose shader compiler crashes on // one (the test switch for such a report); slower shaders, the same picture. static VkPipelineCreateFlags noopt_flag(void) { static int on = -1; if (on < 0) { on = cfg_int("NX2NAVI_VK_NOOPT") != 0; if (on) rt_log("vk_pipeline: pipelines built without the driver's optimiser (NX2NAVI_VK_NOOPT)"); } return on ? VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT : 0; } // ---- pipeline creation feedback (NX2NAVI_PIPE_FEEDBACK, default 1; 0 off) // Each pipeline, library and compute build asks the driver whether pipeline-cache.bin (P.cache) held it // (VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) and how long the create took. The counts appear // with the prewarm, the stage library prebuild and every cache save, so a log tells whether a slow boot compiled or // read its pipelines: on NVIDIA the driver's own disk cache is kept per executable, so a renamed or new exe gets nothing // from it, and only pipeline-cache.bin can make its first boot quick. =2 also logs every compute build with a hash of // its final module, and every graphics pipeline build with the hashes of its final modules and of its fixed state, so // two runs can be compared for a module or a state that changes between them. enum { FB_GRAPHICS, FB_LIBRARY, FB_COMPUTE, FB_KINDS }; static struct { PlatMutex lock; uint64_t n[FB_KINDS], hit[FB_KINDS]; double hit_ms[FB_KINDS], miss_ms[FB_KINDS]; } FB = {.lock = PLAT_MUTEX_INIT}; // Chains fb into a create info's pNext (next: its current pNext). Returns what the create info's pNext becomes. static const void *fb_chain(VkPipelineCreationFeedbackCreateInfo *ci, VkPipelineCreationFeedback *fb, const void *next) { if (!P.feedback) return next; memset(fb, 0, sizeof *fb); *ci = (VkPipelineCreationFeedbackCreateInfo){.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO, .pNext = next, .pPipelineCreationFeedback = fb}; return ci; } static void fb_note(int kind, const VkPipelineCreationFeedback *fb) { if (!P.feedback || !P.cache || !(fb->flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) return; int hit = (fb->flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0; double ms = (double)fb->duration / 1e6; plat_mutex_lock(&FB.lock); FB.n[kind]++, FB.hit[kind] += (uint64_t)hit; if (hit) FB.hit_ms[kind] += ms; else FB.miss_ms[kind] += ms; plat_mutex_unlock(&FB.lock); } // One line: per kind, builds that came from pipeline-cache.bin of all builds so far, with the average time of each. static void fb_log(const char *when) { pb_log(when); if (!P.feedback || !P.cache) return; static const char *const kn[FB_KINDS] = {"pipelines", "libraries", "compute"}; char line[512]; int o = snprintf(line, sizeof line, "vk_pipeline: pipeline-cache.bin hits %s:", when); plat_mutex_lock(&FB.lock); for (int k = 0; k < FB_KINDS && o < (int)sizeof line - 96; k++) { uint64_t miss = FB.n[k] - FB.hit[k]; o += snprintf(line + o, sizeof line - o, " %s %llu of %llu (%.0f%%; avg %.2f ms from the file, %.1f ms otherwise)%s", kn[k], (unsigned long long)FB.hit[k], (unsigned long long)FB.n[k], FB.n[k] ? 100.0 * FB.hit[k] / FB.n[k] : 0.0, FB.hit[k] ? FB.hit_ms[k] / FB.hit[k] : 0.0, miss ? FB.miss_ms[k] / miss : 0.0, k + 1 < FB_KINDS ? ";" : ""); } plat_mutex_unlock(&FB.lock); rt_log("%s", line); } static int uses_src1(const VkPipeState *s) { for (uint32_t m = s->color.blend_en & 0xffu; m; m &= m - 1) { int i = __builtin_ctz(m); const blend_st *b = &s->blend[i]; if ((is_src1(b->src_rgb) || is_src1(b->dst_rgb) || is_src1(b->src_a) || is_src1(b->dst_a)) && i < target_count(s) && blendable(target_format(s->color_fmt[i], NULL, NULL))) return 1; } return 0; } // Colour write mask of target i: NVN's channel mask, zero for a target outside the fragment shader's output map (SPH // omap). Maxwell leaves such a target untouched; // Vulkan writes undefined values, and on AMD they lit grass in front of Link as flat lime. Zeros are wrong // too: NX2NAVI's half-res transparency pass binds its linear depth (R32F) as target 7, samples it in the same draws and // writes only targets 0-2, 4 and 5; zeroed, the depth that later passes rebuild from it was 0 in tiles, and the fog // over distant geometry showed rectangles of clear sky. // The same holds per component: a shader that writes a target's RGB and not its alpha leaves the alpha alone on // Maxwell, while Vulkan writes undefined values, and AMD's driver writes whatever its export registers held (NaN and // saturated pixels the bloom then spreads). Components outside the shader's omap are masked out, where it is known. static uint32_t target_wmask(const VkPipeState *s, int i) { uint32_t m = s->chmask.mask >> (4 * i) & 0xf; if (s->fs && !(gfx_shader_outputs(s->fs, NULL) >> i & 1)) m = 0; if (s->fs) { uint32_t c = gfx_shader_omap(s->fs) >> (4 * i) & 0xf; if (c) m &= c; } if (i == 1 && s->fs && P.f_dual_src && uses_src1(s)) m = 0; // output 1 is the second blend source (spirv_dual_source) return m; } uint32_t vk_pipeline_target_wmask(const VkPipeState *s, int i) { return target_wmask(s, i); } static void gkey_fill(const VkPipeState *s, GKey *k) { memset(k, 0, sizeof *k); k->vs = s->vs, k->fs = s->fs, k->ts = s->ts; k->indep = (uint8_t)P.indep; k->n_color = (uint8_t)target_count(s); for (int i = 0; i < k->n_color; i++) k->color[i] = target_format(s->color_fmt[i], NULL, NULL); int d, st; VkFormat df = target_format(s->depth_fmt, &d, &st); k->depth = d ? df : VK_FORMAT_UNDEFINED; k->stencil = st ? df : VK_FORMAT_UNDEFINED; k->samples = s->samples > 1 ? s->samples : 1; k->topo_class = (uint8_t)topology_class(vk_pipeline_topology(s->prim)); k->minus_w = s->depth_minus_w != 0; k->dsb = s->fs && P.f_dual_src && uses_src1(s); k->fbl = s->fs ? s->feedback : 0; if (!P.dyn_clamp) k->clamp = s->depth_clamp && P.f_depth_clamp; if (!P.dyn_poly) k->poly_mode = (uint8_t)polygon_mode(s->poly.mode); if (!P.dyn_a2c) k->a2c = s->ms.a2c != 0; if (P.f_logic_op && s->color.logic_op != NVN_LOGIC_OP_COPY) k->logic_op = (uint8_t)(s->color.logic_op + 1); for (int i = 0; i < k->n_color; i++) { int on = (s->color.blend_en >> i & 1) && blendable(k->color[i]); if (!P.dyn_blend_en && on) k->blend_en |= (uint8_t)(1u << i); if (!P.dyn_wmask) k->wmask |= target_wmask(s, i) << (4 * i); if (!P.dyn_blend_eq && on) { const blend_st *b = &s->blend[i]; uint8_t *o = k->blend[i]; o[0] = (uint8_t)blend_factor(b->src_rgb), o[1] = (uint8_t)blend_factor(b->dst_rgb), o[2] = (uint8_t)blend_op(b->eq_rgb); o[3] = (uint8_t)blend_factor(b->src_a), o[4] = (uint8_t)blend_factor(b->dst_a), o[5] = (uint8_t)blend_op(b->eq_a); } } if (is_mesh(s)) k->topo_class = 2; // mesh draws: no vertex input, the mesh stage declares its primitives if (!P.dyn_vi && !is_mesh(s)) { VtxInput v; vertex_input(s, &v); for (uint32_t i = 0; i < v.n_attr; i++) { uint32_t l = v.a[i].location; k->attr_fmt[l] = v.a[i].format, k->attr_off[l] = v.a[i].offset, k->attr_bind[l] = (uint8_t)v.a[i].binding; } for (uint32_t i = 0; i < v.n_bind; i++) k->rate[v.b[i].binding] = v.b[i].inputRate == VK_VERTEX_INPUT_RATE_INSTANCE ? 2 : 1; } } // Independent layouts: the outputs the vertex shader lacks from its union with s->fs's inputs added (vs_outs), types in // fin. The vertex module (and its pre-rasterization library) depends on the vertex shader and this mask only. static uint32_t vs_union_missing(const VkPipeState *s, GfxShaderIo fin[32]) { GfxShaderIo vout[32]; uint32_t need = vs_outs(s->vs, s->fs, fin), have = gfx_shader_outputs(s->vs, vout), missing = 0; if (s->fs) { // a location another fragment shader reads as another type: this pair's type (rare) GfxShaderIo fi[32]; uint32_t fneed = gfx_shader_inputs(s->fs, fi); for (int i = 0; i < 32; i++) if (fneed >> i & 1 && !fi[i].packed && (!(need >> i & 1) || fin[i].type != fi[i].type || fin[i].comps < fi[i].comps)) { if (need >> i & 1 && fin[i].type != fi[i].type) LOG_ONCE("vk_pipeline: varying %d read as type %d and %d by fragment shaders of one vertex shader", i, fin[i].type, fi[i].type); fin[i] = fi[i], need |= 1u << i; } } if (P.vs_outs_all) for (int i = 0; i < 32; i++) if (!(need >> i & 1)) fin[i] = (GfxShaderIo){GFX_TYPE_FLOAT, 4, 0}, need |= 1u << i; for (int i = 0; i < 32; i++) if (need >> i & 1 && !(have >> i & 1)) missing |= 1u << i; return missing; } static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Layout *l, uint32_t fs_off) { VkShaderModule vm = VK_NULL_HANDLE, fm = VK_NULL_HANDLE; uint64_t vw, fw, vm_hash = 0, fm_hash = 0; // the hashes: of the final modules, for the PIPE_FEEDBACK 2 line const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw); uint32_t *vpatched = NULL, *fpatched = NULL; int mesh = is_mesh(s); VkShaderModule tm = VK_NULL_HANDLE; if (mesh && s->ts) { // the task stage's bindings follow the fragment stage's (pipe_miss) uint64_t tw; const uint32_t *tcode = gfx_shader_spirv(s->ts, &tw); uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0); uint32_t *t = tcode && toff ? spirv_rebind(tcode, tw, toff) : NULL; tm = make_module(t ? t : tcode, tw, 0); free(t); } if (l->set1 && !mesh) { // independent layouts: outputs from the vertex shader's union, not from this fragment shader GfxShaderIo fin[32]; uint32_t missing = vs_union_missing(s, fin); if (missing) { uint64_t n2 = 0; vpatched = spirv_add_outputs(vcode, vw, missing, fin, &n2); if (!vpatched) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", missing); else vcode = vpatched, vw = n2; } } if (s->fs) { GfxShaderIo fin[32], vout[32]; uint32_t need = gfx_shader_inputs(s->fs, fin), have = mesh ? ~0u : gfx_shader_outputs(s->vs, vout), missing = 0; for (int i = 0; i < 32; i++) { if (!(need >> i & 1) || fin[i].packed) continue; if (!(have >> i & 1)) missing |= 1u << i; else if (!mesh && vout[i].type != fin[i].type) LOG_ONCE("vk_pipeline: varying %d is type %d in the vertex stage, %d in the fragment stage", i, vout[i].type, fin[i].type); } if (missing && !l->set1) { uint64_t n2 = 0; vpatched = spirv_add_outputs(vcode, vw, missing, fin, &n2); if (!vpatched) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", missing); else vcode = vpatched, vw = n2; } const uint32_t *fcode = gfx_shader_spirv(s->fs, &fw); if (k->dsb) { uint64_t n2 = 0; uint32_t *d = spirv_dual_source(fcode, fw, &n2); if (d) fcode = fpatched = d, fw = n2; else LOG_ONCE("vk_pipeline: SRC1 blend factors but the fragment shader writes no colour output 1"); } if (l->set1 || fs_off) { uint32_t *r = !fcode ? NULL : l->set1 ? spirv_set(fcode, fw, 1) : spirv_rebind(fcode, fw, fs_off); free(fpatched); fcode = fpatched = r; } // A bound colour target outside the shader's output map gets write mask 0 (target_wmask) if (fcode) fm = make_module(fcode, fw, !k->a2c), fm_hash = mm_hash; } vm = make_module(vcode, vw, 0), vm_hash = mm_hash; free(vpatched), free(fpatched); VkPipeline pipe = VK_NULL_HANDLE; if (!vm || (s->fs && !fm) || (mesh && s->ts && !tm)) goto out; VkPipelineShaderStageCreateInfo stages[3] = { {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = mesh ? VK_SHADER_STAGE_MESH_BIT_EXT : VK_SHADER_STAGE_VERTEX_BIT, .module = vm, .pName = "main"}, {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_FRAGMENT_BIT, .module = fm, .pName = "main"}, {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_TASK_BIT_EXT, .module = tm, .pName = "main"}}; uint32_t n_stages = 1; if (s->fs) n_stages = 2; if (tm) { if (!s->fs) stages[1] = stages[2]; n_stages++; } VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqv, rqf; uint64_t sw; const uint32_t *scode = gfx_shader_spirv(s->vs, &sw); // vcode may be a patched copy that is freed by now pin_subgroup32(&stages[0], &rqv, scode, sw); if (s->fs) pin_subgroup32(&stages[1], &rqf, gfx_shader_spirv(s->fs, &sw), sw); VkVertexInputAttributeDescription va[32]; VkVertexInputBindingDescription vb[17]; uint32_t na = 0, nb = 0; if (!P.dyn_vi) { VtxInput v; vertex_input(s, &v); for (uint32_t i = 0; i < v.n_attr; i++) va[na++] = (VkVertexInputAttributeDescription){v.a[i].location, v.a[i].binding, v.a[i].format, v.a[i].offset}; for (uint32_t i = 0; i < v.n_bind; i++) vb[nb++] = (VkVertexInputBindingDescription){v.b[i].binding, v.b[i].stride, v.b[i].inputRate}; } VkPipelineVertexInputStateCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, .vertexBindingDescriptionCount = nb, .pVertexBindingDescriptions = vb, .vertexAttributeDescriptionCount = na, .pVertexAttributeDescriptions = va}; static const VkPrimitiveTopology class_base[4] = {VK_PRIMITIVE_TOPOLOGY_POINT_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_PATCH_LIST}; VkPipelineInputAssemblyStateCreateInfo ia = {.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, .topology = class_base[k->topo_class]}; VkPipelineViewportDepthClipControlCreateInfoEXT clip = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT, .negativeOneToOne = k->minus_w}; VkPipelineViewportStateCreateInfo vp = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pNext = &clip}; // NVN's default is the last vertex (SetProvokingVertexMode, nvn_ext.c); Vulkan's is the first. VkPipelineRasterizationProvokingVertexStateCreateInfoEXT pvs = { .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_PROVOKING_VERTEX_STATE_CREATE_INFO_EXT, .provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT}; VkPipelineRasterizationStateCreateInfo rs = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, .pNext = P.pv_last ? &pvs : NULL, .depthClampEnable = k->clamp, .polygonMode = (VkPolygonMode)k->poly_mode, .lineWidth = 1.0f}; VkPipelineMultisampleStateCreateInfo ms = {.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, .rasterizationSamples = (VkSampleCountFlagBits)k->samples, .alphaToCoverageEnable = k->a2c}; VkPipelineDepthStencilStateCreateInfo ds = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, .maxDepthBounds = 1.0f}; VkPipelineColorBlendAttachmentState cba[8]; memset(cba, 0, sizeof cba); for (int i = 0; i < k->n_color; i++) { const uint8_t *b = k->blend[i]; cba[i].blendEnable = k->blend_en >> i & 1; cba[i].srcColorBlendFactor = (VkBlendFactor)b[0], cba[i].dstColorBlendFactor = (VkBlendFactor)b[1]; cba[i].colorBlendOp = (VkBlendOp)b[2], cba[i].srcAlphaBlendFactor = (VkBlendFactor)b[3]; cba[i].dstAlphaBlendFactor = (VkBlendFactor)b[4], cba[i].alphaBlendOp = (VkBlendOp)b[5]; cba[i].colorWriteMask = k->wmask >> (4 * i) & 0xf; } VkPipelineColorBlendStateCreateInfo cb = {.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, .logicOpEnable = k->logic_op != 0, .logicOp = k->logic_op ? (VkLogicOp)(k->logic_op - 1) : VK_LOGIC_OP_COPY, .attachmentCount = k->n_color, .pAttachments = cba}; VkPipelineDynamicStateCreateInfo dy = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, .dynamicStateCount = P.n_dyn, .pDynamicStates = P.dyn}; VkDynamicState mdyn[sizeof P.dyn / sizeof P.dyn[0] + 1]; if (mesh) { // a mesh pipeline must not list vertex input, topology or primitive restart as dynamic uint32_t n = 0; for (uint32_t i = 0; i < P.n_dyn; i++) if (P.dyn[i] != VK_DYNAMIC_STATE_VERTEX_INPUT_EXT && P.dyn[i] != VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE && P.dyn[i] != VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY && P.dyn[i] != VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE) mdyn[n++] = P.dyn[i]; if (vk_ctx()->has_vrs) mdyn[n++] = VK_DYNAMIC_STATE_FRAGMENT_SHADING_RATE_KHR; // vk_exec_draw.c mesh_shading_rate dy.dynamicStateCount = n, dy.pDynamicStates = mdyn; } VkPipelineRenderingCreateInfo ri = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, .colorAttachmentCount = k->n_color, .pColorAttachmentFormats = (const VkFormat *)k->color, .depthAttachmentFormat = (VkFormat)k->depth, .stencilAttachmentFormat = (VkFormat)k->stencil}; VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &ri, .stageCount = n_stages, .pStages = stages, .pVertexInputState = mesh ? NULL : &vi, .pInputAssemblyState = mesh ? NULL : &ia, .pViewportState = &vp, .pRasterizationState = &rs, .pMultisampleState = &ms, .pDepthStencilState = &ds, .pColorBlendState = &cb, .pDynamicState = &dy, .layout = l->layout, .flags = noopt_flag()}; if (k->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; if (k->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; VkPipelineCreationFeedbackCreateInfo fci; VkPipelineCreationFeedback fb; pi.pNext = fb_chain(&fci, &fb, pi.pNext); uint64_t p0 = perf_begin(); crash_note("a graphics pipeline", s->vs, s->fs, s->ts); VkResult r = pipe_create(0, &pi, &pipe); plat_crash_context(NULL); perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it if (r != VK_SUCCESS) { rt_log("vk_pipeline: vkCreateGraphicsPipelines failed (%d)", (int)r); pipe = VK_NULL_HANDLE; } else { perf_count(PERF_CNT_PIPE_BUILT); fb_note(FB_GRAPHICS, &fb); if (P.feedback >= 2) { // the state hash: what the create info takes from the key besides the modules uint64_t st = fnv(fnv(FNV0, cba, sizeof cba), k->color, sizeof k->color); if (!P.dyn_vi) // the vertex layout is pipeline state then (zero in a mesh pipeline's key) st = fnv(fnv(fnv(fnv(st, k->attr_fmt, sizeof k->attr_fmt), k->attr_off, sizeof k->attr_off), k->attr_bind, sizeof k->attr_bind), k->rate, sizeof k->rate); uint32_t bits[] = {(uint32_t)mesh, k->topo_class, k->minus_w, k->clamp, k->poly_mode, k->samples, k->a2c, k->logic_op, k->n_color, k->fbl, k->dsb, l->set1 != 0, fs_off, pi.flags, k->depth, k->stencil}; rt_log("vk_pipeline: graphics: modules %016llx %016llx, state %016llx (mesh %d, topology class %u, samples %u, " "colors %u, feedback loop %u, depth format %u, stage-independent layout %d, flags 0x%x), %s, %.1f ms", (unsigned long long)vm_hash, (unsigned long long)fm_hash, (unsigned long long)fnv(st, bits, sizeof bits), mesh, (unsigned)k->topo_class, (unsigned)k->samples, (unsigned)k->n_color, (unsigned)k->fbl, (unsigned)k->depth, l->set1 != 0, (unsigned)pi.flags, pb_last ? "from pipeline-binaries.bin" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled", (double)fb.duration / 1e6); } } out: if (vm) vkDestroyShaderModule(P.device, vm, NULL); if (fm) vkDestroyShaderModule(P.device, fm, NULL); if (tm) vkDestroyShaderModule(P.device, tm, NULL); return pipe; } // ---- graphics pipeline library (vk_caps.gpl) // A pipeline is linked from four libraries, each cached by the part of the pipeline key it depends on: vertex input // (topology class, vertex attributes without dynamic vertex input), pre-rasterization (vertex shader, the outputs its // union adds, static rasterization state), fragment shader (fragment shader, dual-source rewrite, samples) and fragment // output (targets, blend, samples). With independent layouts each stage's set depends on its shader alone, so the // pre-rasterization library uses {set 0, null} and the fragment shader library {null, set 1}. The fast link runs on the // thread of the draw that needs it (no link-time optimisation); the optimised pipeline is the job pipe_miss queued // (a monolithic build with the same modules), which replaces the fast one in the Pipe when done. Libraries and fast // pipelines live until vk_pipeline_shutdown. // Background build signalling, both with P.lock: work_cv wakes the pipeline workers, done_cv wakes threads waiting for a // pipeline or a stage library to be published. static PlatCond work_cv = PLAT_COND_INIT, done_cv = PLAT_COND_INIT; static PlatCond lib_cv = PLAT_COND_INIT; // wakes the library workers (lib_worker): work_cv's signal must reach a pool worker #define LIB_BUCKETS 1024 enum { LIB_VI, LIB_PR, LIB_FS, LIB_FO }; typedef struct Lib { struct Lib *next; uint64_t hash; VkPipeline pipe; uint32_t size; uint8_t pre, used, building, pad; struct LibJob *job; uint8_t key[]; } Lib; // pre: built by the boot prebuild ("stage library prebuild" below); used: a draw's link found it; building: a thread // builds it or job holds it queued (pipe not set yet); entries live until libs_destroy static Lib *libs[LIB_BUCKETS]; typedef struct { uint32_t kind, topo_class, attr_fmt[32], attr_off[32]; uint8_t attr_bind[32], rate[17], pad[3]; } ViKey; typedef struct { uint32_t kind, missing; const GfxShader *vs; uint8_t clamp, poly_mode, minus_w, pad; GfxShaderIo fin[32]; } PrKey; typedef struct { uint32_t kind; uint8_t dsb, samples, a2c, pad; const GfxShader *fs; } FsKey; typedef struct { uint32_t kind; uint8_t n_color, samples, a2c, logic_op, blend_en, fbl, pad[2]; uint32_t color[8], depth, stencil, wmask; uint8_t blend[8][6]; } FoKey; // fbl: GKey.fbl, the feedback loop flags of the library (gpl_feedback) static uint64_t qpc(void) { return plat_now_ns(); } static double ms_since(uint64_t t0) { return (double)(qpc() - t0) / 1e6; } static VkPipelineLayout half_layout(Layout *l, int h) { if (l->half[h]) return l->half[h]; VkDescriptorSetLayout sets[3] = {h ? VK_NULL_HANDLE : l->set, h ? l->set1 : VK_NULL_HANDLE, vk_pool_set_layout()}; VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 3, .pSetLayouts = sets}; if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->half[h]) != VK_SUCCESS) l->half[h] = VK_NULL_HANDLE; return l->half[h]; } // Builds one library from the pipeline's state; key is the library's key (PrKey: fin holds the added outputs). static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPipelineLayout lay, const void *key) { static const VkGraphicsPipelineLibraryFlagsEXT part[4] = { VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT, VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT, VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT, VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT}; VkGraphicsPipelineLibraryCreateInfoEXT gl = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, .flags = part[kind]}; VkPipelineRenderingCreateInfo ri = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, .pNext = &gl}; VkPipelineDynamicStateCreateInfo dy = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, .dynamicStateCount = P.n_dyn, .pDynamicStates = P.dyn}; VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &ri, .flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | noopt_flag(), .pDynamicState = &dy}; VkShaderModule m = VK_NULL_HANDLE; VkPipelineShaderStageCreateInfo st = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pName = "main"}; VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rq; VkVertexInputAttributeDescription va[32]; VkVertexInputBindingDescription vb[17]; VkPipelineVertexInputStateCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; static const VkPrimitiveTopology class_base[4] = {VK_PRIMITIVE_TOPOLOGY_POINT_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_PATCH_LIST}; VkPipelineInputAssemblyStateCreateInfo ia = {.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, .topology = class_base[k->topo_class]}; VkPipelineViewportDepthClipControlCreateInfoEXT clip = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT, .negativeOneToOne = k->minus_w}; VkPipelineViewportStateCreateInfo vp = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pNext = &clip}; VkPipelineRasterizationProvokingVertexStateCreateInfoEXT pvs = { .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_PROVOKING_VERTEX_STATE_CREATE_INFO_EXT, .provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT}; VkPipelineRasterizationStateCreateInfo rs = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, .pNext = P.pv_last ? &pvs : NULL, .depthClampEnable = k->clamp, .polygonMode = (VkPolygonMode)k->poly_mode, .lineWidth = 1.0f}; VkPipelineMultisampleStateCreateInfo ms = {.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, .rasterizationSamples = (VkSampleCountFlagBits)k->samples, .alphaToCoverageEnable = k->a2c}; VkPipelineDepthStencilStateCreateInfo ds = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, .maxDepthBounds = 1.0f}; VkPipelineColorBlendAttachmentState cba[8]; VkPipelineColorBlendStateCreateInfo cb = {.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; uint32_t *patched = NULL; VkPipeline pipe = VK_NULL_HANDLE; if (kind == LIB_VI) { if (!P.dyn_vi) { VtxInput v; vertex_input(s, &v); for (uint32_t i = 0; i < v.n_attr; i++) va[i] = (VkVertexInputAttributeDescription){v.a[i].location, v.a[i].binding, v.a[i].format, v.a[i].offset}; for (uint32_t i = 0; i < v.n_bind; i++) vb[i] = (VkVertexInputBindingDescription){v.b[i].binding, v.b[i].stride, v.b[i].inputRate}; vi.vertexAttributeDescriptionCount = v.n_attr, vi.pVertexAttributeDescriptions = va; vi.vertexBindingDescriptionCount = v.n_bind, vi.pVertexBindingDescriptions = vb; } pi.pVertexInputState = &vi, pi.pInputAssemblyState = &ia; } else if (kind == LIB_PR) { const PrKey *pk = key; uint64_t vw, sw, n2 = 0; const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw), *scode = vcode; sw = vw; if (pk->missing && (patched = spirv_add_outputs(vcode, vw, pk->missing, pk->fin, &n2))) vcode = patched, vw = n2; else if (pk->missing) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", pk->missing); if (!(m = make_module(vcode, vw, 0))) goto out; st.stage = VK_SHADER_STAGE_VERTEX_BIT, st.module = m; pin_subgroup32(&st, &rq, scode, sw); pi.stageCount = 1, pi.pStages = &st, pi.pViewportState = &vp, pi.pRasterizationState = &rs; pi.layout = lay; } else if (kind == LIB_FS) { if (s->fs) { uint64_t fw, sw, n2 = 0; const uint32_t *fcode = gfx_shader_spirv(s->fs, &fw), *scode = fcode; sw = fw; uint32_t *d = NULL; if (k->dsb && fcode) { if ((d = spirv_dual_source(fcode, fw, &n2))) fcode = d, fw = n2; else LOG_ONCE("vk_pipeline: SRC1 blend factors but the fragment shader writes no colour output 1"); } patched = fcode ? spirv_set(fcode, fw, 1) : NULL; free(d); if (!patched || !(m = make_module(patched, fw, !k->a2c))) goto out; st.stage = VK_SHADER_STAGE_FRAGMENT_BIT, st.module = m; pin_subgroup32(&st, &rq, scode, sw); pi.stageCount = 1, pi.pStages = &st; } pi.pMultisampleState = &ms, pi.pDepthStencilState = &ds; pi.layout = lay; } else { memset(cba, 0, sizeof cba); for (int i = 0; i < k->n_color; i++) { const uint8_t *b = k->blend[i]; cba[i].blendEnable = k->blend_en >> i & 1; cba[i].srcColorBlendFactor = (VkBlendFactor)b[0], cba[i].dstColorBlendFactor = (VkBlendFactor)b[1]; cba[i].colorBlendOp = (VkBlendOp)b[2], cba[i].srcAlphaBlendFactor = (VkBlendFactor)b[3]; cba[i].dstAlphaBlendFactor = (VkBlendFactor)b[4], cba[i].alphaBlendOp = (VkBlendOp)b[5]; cba[i].colorWriteMask = k->wmask >> (4 * i) & 0xf; } cb.logicOpEnable = k->logic_op != 0, cb.logicOp = k->logic_op ? (VkLogicOp)(k->logic_op - 1) : VK_LOGIC_OP_COPY; cb.attachmentCount = k->n_color, cb.pAttachments = cba; ri.colorAttachmentCount = k->n_color, ri.pColorAttachmentFormats = (const VkFormat *)k->color; ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil; pi.pColorBlendState = &cb, pi.pMultisampleState = &ms; // Whether the pipeline may sample an attachment it renders to is part of the fragment output state: the flags // go on this library alone, and a pipeline linked from it has them (gpl_feedback). const FoKey *ok = key; if (ok->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; if (ok->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; } VkPipelineCreationFeedbackCreateInfo fci; VkPipelineCreationFeedback fb; pi.pNext = fb_chain(&fci, &fb, pi.pNext); crash_note("a pipeline library", s->vs, s->fs, NULL); if (pipe_create(1, &pi, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); if (pipe && (kind == LIB_PR || kind == LIB_FS)) fb_note(FB_LIBRARY, &fb); // the shader libraries: the others compile nothing out: if (m) vkDestroyShaderModule(P.device, m, NULL); free(patched); return pipe; } // The cached library for key. Caller holds P.lock. build 0: lookup only (VK_NULL_HANDLE and *miss when absent); // build 1: a missing one is built with P.lock released (a failed build is cached too). // The library entry for key, NULL when absent. Caller holds P.lock. static Lib *lib_find(uint64_t h, const void *key, uint32_t size) { for (Lib *b = libs[h % LIB_BUCKETS]; b; b = b->next) if (b->hash == h && b->size == size && !memcmp(b->key, key, size)) return b; return NULL; } // Enters a library built with P.lock released; another thread's entry made meanwhile wins and pipe is destroyed. // Returns the entered handle. Caller holds P.lock. static VkPipeline lib_put(uint64_t h, const void *key, uint32_t size, VkPipeline pipe) { Lib *b = lib_find(h, key, size); if (b && b->building) { // another thread's build publishes the entry if (pipe) vkDestroyPipeline(P.device, pipe, NULL); return VK_NULL_HANDLE; } if (b) { if (pipe) vkDestroyPipeline(P.device, pipe, NULL); return b->pipe; } if (!(b = calloc(1, sizeof *b + size))) return pipe; b->hash = h, b->size = size, b->pipe = pipe; memcpy(b->key, key, size); b->next = libs[h % LIB_BUCKETS]; libs[h % LIB_BUCKETS] = b; return pipe; } // Shader libraries a waiting draw needs (lib_wait, link_queue): queued here ahead of every pipeline job, so an idle // worker builds each at once and the vertex and fragment libraries of one draw build in parallel. A job no worker took // is built by the first thread that needs its library (lib_get; lib_wait only without library workers). A job with p // is a waiting draw's fast link of p (after its library jobs), run on a worker so the drawing thread only waits. // Under P.lock. typedef struct LibJob { struct LibJob *next; Lib *b; int kind; Layout *l; VkPipeState s; GKey k; Pipe *p; } LibJob; static struct { LibJob *head, *tail; int idle; // workers asleep on work_cv int running; // jobs workers are building int on; // NX2NAVI_PIPE_WAIT: 1 workers (default), 0 the drawing thread builds them (draw); // set by vk_pipeline_async_enable, before the replay runs int lib_workers; // workers that take only these jobs (lib_worker), started with the pool uint64_t waits, queued, by_worker, stolen, timeouts, links; } U; // A building entry for key, NULL when out of memory. Caller holds P.lock and found none. static Lib *lib_new(uint64_t h, const void *key, uint32_t size) { Lib *b = calloc(1, sizeof *b + size); if (!b) return NULL; b->hash = h, b->size = size, b->building = 1; memcpy(b->key, key, size); b->next = libs[h % LIB_BUCKETS]; libs[h % LIB_BUCKETS] = b; return b; } // Builds b (building, not queued) with P.lock released and publishes it; a failed build is entered as VK_NULL_HANDLE. static VkPipeline lib_build(Lib *b, int kind, const VkPipeState *s, const GKey *k, Layout *l) { VkPipelineLayout lay = kind == LIB_PR ? half_layout(l, 0) : kind == LIB_FS ? half_layout(l, 1) : VK_NULL_HANDLE; VkPipeline pipe = VK_NULL_HANDLE; if (lay || kind == LIB_VI || kind == LIB_FO) { plat_mutex_unlock(&P.lock); uint64_t t0 = qpc(); pipe = lib_create(kind, s, k, lay, b->key); double ms = ms_since(t0); plat_mutex_lock(&P.lock); P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms; if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms; if (kind == LIB_PR && ((const PrKey *)(const void *)b->key)->missing) P.g.pr_added++, P.g.pr_added_ms += ms; if (ms >= 25) P.g.lib_slow[kind]++, P.g.lib_slow_ms[kind] += ms; } if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind); b->pipe = pipe, b->building = 0; plat_cond_broadcast(&done_cv); return pipe; } // Takes b's queued job out of the queue and builds it on this thread. Caller holds P.lock. static void lib_steal(Lib *b) { LibJob *j = b->job, **pp = &U.head, *prev = NULL; while (*pp && *pp != j) prev = *pp, pp = &(*pp)->next; if (*pp) *pp = j->next; if (U.tail == j) U.tail = prev; b->job = NULL; lib_build(b, j->kind, &j->s, &j->k, j->l); free(j); } static VkPipeline lib_get(int kind, const VkPipeState *s, const GKey *k, Layout *l, const void *key, uint32_t size, int build, int *miss) { uint64_t h = key_hash(key, size); Lib *have = lib_find(h, key, size); while (have && have->building) { // being built (wait) or queued (build it here) if (!build) { *miss = 1; return VK_NULL_HANDLE; } if (have->job) U.stolen++, lib_steal(have); else plat_cond_wait(&done_cv, &P.lock); } if (have) { if (have->pre && !have->used) have->used = 1, P.g.pre_used[kind]++; return have->pipe; } VkPipelineLayout lay = kind == LIB_PR ? half_layout(l, 0) : kind == LIB_FS ? half_layout(l, 1) : VK_NULL_HANDLE; if (!build || ((kind == LIB_PR || kind == LIB_FS) && !lay)) { *miss = 1; return VK_NULL_HANDLE; } Lib *b = lib_new(h, key, size); // entered first, so other threads wait for this build instead of repeating it if (b) return lib_build(b, kind, s, k, l); plat_mutex_unlock(&P.lock); uint64_t t0 = qpc(); VkPipeline pipe = lib_create(kind, s, k, lay, key); double ms = ms_since(t0); plat_mutex_lock(&P.lock); if ((have = lib_find(h, key, size)) && !have->building) { // another thread built it meanwhile if (pipe) vkDestroyPipeline(P.device, pipe, NULL); return have->pipe; } P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms; if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms; if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind); return lib_put(h, key, size, pipe); } static void libs_destroy(void) { for (int i = 0; i < LIB_BUCKETS; i++) { for (Lib *b = libs[i], *n; b; b = n) { n = b->next; if (b->pipe) vkDestroyPipeline(P.device, b->pipe, NULL); free(b); } libs[i] = NULL; } } // The four library keys of pipeline s (key k). static void lib_keys(const VkPipeState *s, const GKey *k, ViKey *v, PrKey *pk, FsKey *fk, FoKey *ok) { memset(v, 0, sizeof *v); v->kind = LIB_VI, v->topo_class = k->topo_class; memcpy(v->attr_fmt, k->attr_fmt, sizeof v->attr_fmt), memcpy(v->attr_off, k->attr_off, sizeof v->attr_off); memcpy(v->attr_bind, k->attr_bind, sizeof v->attr_bind), memcpy(v->rate, k->rate, sizeof v->rate); memset(pk, 0, sizeof *pk); GfxShaderIo fin[32]; pk->kind = LIB_PR, pk->vs = s->vs, pk->clamp = k->clamp, pk->poly_mode = k->poly_mode, pk->minus_w = k->minus_w; pk->missing = vs_union_missing(s, fin); for (int i = 0; i < 32; i++) if (pk->missing >> i & 1) pk->fin[i] = fin[i]; memset(fk, 0, sizeof *fk); fk->kind = LIB_FS, fk->fs = s->fs, fk->dsb = k->dsb, fk->samples = k->samples, fk->a2c = k->a2c; memset(ok, 0, sizeof *ok); ok->kind = LIB_FO, ok->n_color = k->n_color, ok->samples = k->samples, ok->a2c = k->a2c, ok->logic_op = k->logic_op; ok->blend_en = k->blend_en, ok->fbl = k->fbl, ok->depth = k->depth, ok->stencil = k->stencil, ok->wmask = k->wmask; memcpy(ok->color, k->color, sizeof ok->color), memcpy(ok->blend, k->blend, sizeof ok->blend); } // p, which a draw missed, has its first usable pipeline: how long that took is how long its draws were skipped or // waited. fast 0: it had no fast link and this is the optimised build. Caller holds P.lock. static void ready_note(Pipe *p, int fast) { if (!p->t_miss) return; double ms = (double)(plat_now_ns() - p->t_miss) / 1e6; p->t_miss = 0; P.g.ready_n[ms < 4 ? 0 : ms < 8 ? 1 : ms < 17 ? 2 : ms < 50 ? 3 : ms < 150 ? 4 : 5]++; P.g.ready_ms += ms; if (ms > P.g.ready_max) P.g.ready_max = ms; P.g.ready_nofast += !fast; } // NX2NAVI_GPL_FEEDBACK (default 1; 0 = as before): a pipeline whose fragment shader samples an attachment of its own pass // (VkPipeState.feedback: soft particles and water reading the depth buffer, decals, full-screen passes that read the // target they write) gets a fast link like any other. The feedback loop flags belong to the fragment output state, so // they are part of that library's key (FoKey.fbl) and of nothing else; the shader libraries are the ones the same // shaders use without a loop. Before, such a pipeline had no fast link at all and its draws waited for, or were // skipped during, the whole optimised build. On a first visit to Kokiri Forest (RTX 5090, every stage library in the // driver's cache, DLSS Frame Generation on, so no frame is held back) full-screen draws of such pipelines waited // 267 ms at the area's first frame and 410 ms in the middle of a walk; linked, 3 ms. static int gpl_feedback(void) { static int on = -1; if (on < 0) { on = cfg_bool("NX2NAVI_GPL_FEEDBACK"); if (!on) rt_log("vk_pipeline: gpl: pipelines with a feedback loop get no fast link (NX2NAVI_GPL_FEEDBACK=0)"); } return on; } // A fast-linked pipeline for s from its stage libraries. Caller holds P.lock. // build 0 (drawing thread): links only when the shader libraries exist (vertex input and fragment output libraries, // which hold no shader, are built here); *final 0 = not yet, a worker builds them. build 1 (worker): builds the // missing libraries and links with P.lock released. VK_NULL_HANDLE with *final 1 when GPL cannot express the pipeline // (shared layout; a feedback loop with NX2NAVI_GPL_FEEDBACK=0) or a build failed. static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc, int build, int *final) { Layout *l = (Layout *)lc; *final = 1; if (!l->set1 || (k->fbl && !gpl_feedback())) return VK_NULL_HANDLE; uint64_t t0 = qpc(); ViKey vk_; PrKey pk; FsKey fk; FoKey ok; lib_keys(s, k, &vk_, &pk, &fk, &ok); int miss = 0; VkPipeline lib[4]; lib[1] = lib_get(LIB_PR, s, k, l, &pk, sizeof pk, build, &miss); // shader libraries first: no work when absent lib[2] = miss ? VK_NULL_HANDLE : lib_get(LIB_FS, s, k, l, &fk, sizeof fk, build, &miss); if (miss) { if (!build) *final = 0, P.g.defer++; return VK_NULL_HANDLE; } lib[0] = lib_get(LIB_VI, s, k, l, &vk_, sizeof vk_, 1, &miss); lib[3] = lib_get(LIB_FO, s, k, l, &ok, sizeof ok, 1, &miss); VkPipeline pipe = VK_NULL_HANDLE; if (lib[0] && lib[1] && lib[2] && lib[3]) { VkPipelineLibraryCreateInfoKHR li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR, .libraryCount = 4, .pLibraries = lib}; VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &li, .layout = l->layout}; if (build) plat_mutex_unlock(&P.lock); uint64_t t1 = qpc(); crash_note("a library link", s->vs, s->fs, NULL); if (vkCreateGraphicsPipelines(P.device, VK_NULL_HANDLE, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); double ms = ms_since(t1); // the link alone: library builds are counted apart if (build) plat_mutex_lock(&P.lock); P.g.link_ms += ms; if (ms > P.g.link_max) P.g.link_max = ms; } if (pipe) P.g.links++, P.g.fb_links += k->fbl != 0; else if (!P.g.link_fail++) rt_log("vk_pipeline: gpl: fast link failed; such draws wait or skip as without GPL"); double total = ms_since(t0); if (!build && total > 5.0 && P.g.slow++ < 16) rt_log("vk_pipeline: gpl: fast link on the drawing thread took %.1f ms", total); return pipe; } // ---- background graphics pipeline builds (vk_pipeline_async_enable) // A miss in vk_pipeline_graphics_async enters the Pipe as building and queues a job; the replay skips draws with it // until a worker publishes the handle. The job holds copies of the state and the key. What they point to outlives it: // shaders (gfx_shader_get results) live for the process (gfx_shader_drop_memory is the shader probe's, which builds no // pipelines), layouts until vk_pipeline_shutdown, which drains the queue first. Workers call vkCreateGraphicsPipelines // without P.lock: P.cache is created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT, so the driver // synchronises builds into it; only the copy of its data in cache_save is kept apart from them (cache_rw). Compute // pipelines stay synchronous: a skipped // dispatch can change what the game computes. // // Two queues: draws' misses (A.head) and the boot prebuild (A.low, see "pipeline keys" below). Workers take the low // queue only when the other is empty and run its jobs at PLAT_PRIO_BELOW_NORMAL, so guest threads win. A draw // that finds a prebuild job still queued moves it to the draw queue; one that finds it running raises its worker. typedef struct Job { struct Job *next, *prev; // prev: low queue only Pipe *p; const Layout *l; uint32_t fs_off; uint8_t low; // in the low queue, or running at below-normal priority uint8_t prebuild, demanded; // from the key file; a draw asked for it before it was ready int worker; // running on A.thread[worker]; -1 queued VkPipeState s; GKey k; } Job; #define MAX_WORKERS 4 static struct { int on; // set once by vk_pipeline_async_enable, before the replay runs int threads; PlatThread *thread[MAX_WORKERS]; uint8_t prio_low[MAX_WORKERS]; // worker runs below normal (under P.lock) Job *head, *tail; // draws' misses Job *low, *low_tail; // prebuild uint32_t inflight; // queued or running, both queues int low_running; // pool workers running a job taken from the low queue PlatCond idle; // with P.lock: no job queued or running } A = {.idle = PLAT_COND_INIT}; // ---- pipeline keys (pipeline-keys.bin next to the pipeline cache) // Every graphics pipeline built in a game run appends a KeyRec: the VkPipeState it was built from with the shader // pointers zeroed and the shaders named by their gfx_shader input ids instead, which do not depend on the translator, // so the record rebuilds the pipeline without guest memory. At the next boot vk_pipeline_prewarm (rt_boot, before the // guest runs; NX2NAVI_PIPE_PREBUILD=0 skips it) loads each record's shaders, translating the ones not cached under the // current translator from their stored inputs (gfx_shader_prewarm; a record whose shader has neither is dropped), // enters each Pipe as building under the key a draw would compute, queues it to the low queue and builds the queue on // every core. Records are deduplicated by a hash of their bytes; they are collected under P.lock by the worker that // built the pipeline and appended by the present thread (vk_pipeline_autosave) or at exit (vk_pipeline_save). A file // with another magic, version or record size is ignored and rewritten; a version 3 file (no task stage) is converted // and rewritten. Bump KEYS_VERSION when VkPipeState or the record changes meaning without changing size. #define KEYS_VERSION 4 // 3: shaders named by input id (gfx_shader.c); 4: mesh pipelines, with their task stage typedef struct { char magic[8]; uint32_t version, rec_size; } KeysHeader; typedef struct { uint8_t vs[32], fs[32]; uint32_t has_fs, pad; VkPipeState s; } KeyRecV3; typedef struct { uint8_t vs[32], fs[32], ts[32]; // gfx_shader_input_id of each stage (vs: the mesh stage of a mesh pipeline); fs all // zero with has_fs 0, ts with has_ts 0 uint32_t has_fs, has_ts; VkPipeState s; // vs, fs, ts zero } KeyRec; static struct { char path[PLAT_PATH_MAX * 2]; int header_ok; // the file exists with our header: append uint64_t *seen; // open addressing set of record hashes (0 empty) uint32_t seen_cap, seen_n; KeyRec *pend; // records not written yet uint32_t pend_n, pend_cap; uint8_t *file; // the file as read at boot, until the prewarm is done with it size_t file_n; // records in it PlatMutex io; // one writer // prebuild progress (under P.lock) int loading; // the prewarm still queues records int warm; // vk_pipeline_prewarm owns the file (prebuild_start leaves it) uint32_t left; // prebuild jobs queued or running uint32_t built, failed, promoted; double t0; // now_ms() when the prebuild started double until; // now_ms() after which the boot builders stop (comment at prewarm_run); 0 none } K = {.io = PLAT_MUTEX_INIT}; static uint64_t rec_hash(const KeyRec *r) { uint64_t h = fnv(FNV0, r, sizeof *r); return h ? h : 1; } // 1 if h was new. Caller holds P.lock. static int seen_add(uint64_t h) { if ((K.seen_n + 1) * 2 > K.seen_cap) { uint32_t cap = K.seen_cap ? K.seen_cap * 2 : 4096; uint64_t *t = calloc(cap, sizeof *t); if (!t) return 1; for (uint32_t i = 0; i < K.seen_cap; i++) { if (!K.seen[i]) continue; uint32_t j = (uint32_t)K.seen[i] & (cap - 1); while (t[j]) j = (j + 1) & (cap - 1); t[j] = K.seen[i]; } free(K.seen); K.seen = t, K.seen_cap = cap; } uint32_t j = (uint32_t)h & (K.seen_cap - 1); while (K.seen[j]) { if (K.seen[j] == h) return 0; j = (j + 1) & (K.seen_cap - 1); } K.seen[j] = h; K.seen_n++; return 1; } static void rec_fill(KeyRec *r, const VkPipeState *s) { memset(r, 0, sizeof *r); memcpy(r->vs, gfx_shader_input_id(s->vs), 32); if (s->fs) memcpy(r->fs, gfx_shader_input_id(s->fs), 32), r->has_fs = 1; if (s->ts) memcpy(r->ts, gfx_shader_input_id(s->ts), 32), r->has_ts = 1; r->s = *s; r->s.vs = r->s.fs = r->s.ts = NULL; r->s.prim_restart = 0; // dynamic state: the same pipeline either way } // A pipeline was built for a draw: queue its record for the file. Caller holds P.lock. static void keys_note(const VkPipeState *s) { if (!A.on || !K.path[0]) return; static const uint8_t none[32]; if (s->fs && !memcmp(gfx_shader_input_id(s->fs), none, 32)) return; // the runtime's own shader (gfx_shader_own) if (s->ts && !memcmp(gfx_shader_input_id(s->ts), none, 32)) return; KeyRec r; rec_fill(&r, s); if (!seen_add(rec_hash(&r))) return; if (K.pend_n == K.pend_cap) { uint32_t cap = K.pend_cap ? K.pend_cap * 2 : 64; KeyRec *t = realloc(K.pend, cap * sizeof *t); if (!t) return; K.pend = t, K.pend_cap = cap; } K.pend[K.pend_n++] = r; } // Appends the pending records (present thread or exit; never the render thread). static void keys_flush(void) { if (!K.path[0]) return; plat_mutex_lock(&K.io); plat_mutex_lock(&P.lock); KeyRec *r = K.pend; uint32_t n = K.pend_n; K.pend = NULL, K.pend_n = K.pend_cap = 0; plat_mutex_unlock(&P.lock); if (n) { FILE *f = fopen(K.path, K.header_ok ? "ab" : "wb"); int ok = f != NULL; if (ok && !K.header_ok) { KeysHeader h = {{'N', 'X', '2', 'N', 'P', 'K', 'E', 'Y'}, KEYS_VERSION, sizeof(KeyRec)}; ok = fwrite(&h, sizeof h, 1, f) == 1; } if (ok) ok = fwrite(r, sizeof *r, n, f) == n; if (ok) ok = plat_file_sync(f); // appended records on the disk before the next append if (f) ok = !fclose(f) && ok; if (ok) K.header_ok = 1; else rt_log("vk_pipeline: cannot write %s", K.path); } free(r); plat_mutex_unlock(&K.io); } // Reads the key file into K.file and its record hashes into the set. Caller holds P.lock (ensure_init). static void keys_open(void) { if (!A.on || !P.cache_path[0]) return; snprintf(K.path, sizeof K.path, "%s", P.cache_path); char *slash = strrchr(K.path, '\\'), *fslash = strrchr(K.path, '/'); if (fslash > slash) slash = fslash; snprintf(slash ? slash + 1 : K.path, sizeof K.path - (size_t)(slash ? slash + 1 - K.path : 0), "pipeline-keys.bin"); gfx_shader_record_at(K.path); // shader-answers.bin next to it (gfx_shader.c) FILE *f = fopen(K.path, "rb"); if (!f) return; KeysHeader h; fseek(f, 0, SEEK_END); long len = ftell(f); fseek(f, 0, SEEK_SET); int head = len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8); if (head && h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec)) { K.header_ok = 1; size_t n = ((size_t)len - sizeof h) / sizeof(KeyRec); if (n && (K.file = malloc(n * sizeof(KeyRec))) && fread(K.file, sizeof(KeyRec), n, f) == n) K.file_n = n; else free(K.file), K.file = NULL; for (size_t i = 0; i < K.file_n; i++) seen_add(rec_hash((const KeyRec *)K.file + i)); if (len != (long)(sizeof h + n * sizeof(KeyRec))) // a torn append: rewrite the file from what was read K.header_ok = 0; } else if (head && h.version == 3 && h.rec_size == sizeof(KeyRecV3)) { // the vertex pipelines of a version 3 file keep their prewarm: each record without a task stage, and the // file rewritten in this version below (header_ok 0) size_t n = ((size_t)len - sizeof h) / sizeof(KeyRecV3); KeyRecV3 *old = n ? malloc(n * sizeof *old) : NULL; if (old && fread(old, sizeof *old, n, f) == n && (K.file = calloc(n, sizeof(KeyRec)))) { KeyRec *r = (KeyRec *)K.file; for (size_t i = 0; i < n; i++) { memcpy(r[i].vs, old[i].vs, 32), memcpy(r[i].fs, old[i].fs, 32); r[i].has_fs = old[i].has_fs, r[i].s = old[i].s; seen_add(rec_hash(&r[i])); } K.file_n = n; rt_log("vk_pipeline: %s: %zu version 3 records converted; the file is rewritten", K.path, n); } free(old); } else { rt_log("vk_pipeline: %s has another format or version; it is rewritten", K.path); } fclose(f); if (!K.header_ok && K.file_n) { // keep the whole records of a torn file KeyRec *t = malloc(K.file_n * sizeof *t); if (t) memcpy(t, K.file, K.file_n * sizeof *t), K.pend = t, K.pend_n = K.pend_cap = (uint32_t)K.file_n; } } static double now_ms(void) { return (double)plat_now_ns() / 1e6; } // Caller holds P.lock. static void prebuild_report(void) { rt_log("vk_pipeline: prebuild done: %u built, %u failed, %u asked for by a draw first; %.0f ms after the device was " "ready", K.built, K.failed, K.promoted, now_ms() - K.t0); } static void job_lower(Job *j); // Runs a waiting draw's job taken from U on a worker: a library build or, for a job with p, p's fast link (gpl_link // build 1 builds a library still missing and waits for one another worker builds). Releases P.lock while building. static void u_run(LibJob *lj) { U.by_worker++, U.running++; if (lj->p) { Pipe *p = lj->p; if (!p->info.pipeline) { int final = 0; VkPipeline f = gpl_link(&lj->s, &lj->k, lj->l, 1, &final); // releases P.lock while linking if (f && !p->info.pipeline) { p->fast = p->info.pipeline = f; ready_note(p, 1); job_lower(p->job); // its job now only does the optimised build } else if (f) { vkDestroyPipeline(P.device, f, NULL); // the job published one meanwhile } } p->link_queued = 0; U.links++; plat_cond_broadcast(&done_cv); } else { lib_build(lj->b, lj->kind, &lj->s, &lj->k, lj->l); } U.running--; free(lj); } // NX2NAVI_PIPE_FAST_WORKERS (default 1; 0 = as before): the library workers also take the fast link of a draw-queue job // (fast_take). A draw that is skipped rather than waiting leaves its pipeline to the pool, where a job is first // fast-linked and then goes back in the queue for its optimised build. The pool keeps one worker out of optimised // builds, so with the others inside one (about 100 ms each with the driver's cache warm, 350 ms cold) that single // worker linked every new pipeline of a burst, one after another at 5 to 10 ms each. On a first visit to Kokiri // Forest (RTX 5090, libraries in the driver's cache) 180 of the 444 pipelines draws missed took over 50 ms to become // usable, the longest 1.3 s, and their draws were missing from every frame until then; with the library workers // linking, 23 did, 19 of them mesh pipelines, which have no fast link. static int fast_workers(void) { static int on = -1; if (on < 0) on = cfg_bool("NX2NAVI_PIPE_FAST_WORKERS"); return on; } // The first draw-queue job whose pipeline still lacks its fast link, taken out of the queue; NULL when there is none. // Caller holds P.lock. static Job *fast_take(void) { if (!P.gpl || !fast_workers()) return NULL; for (Job *j = A.head, *prev = NULL; j; prev = j, j = j->next) { Pipe *p = j->p; if (p->info.pipeline || p->fast_tried || p->link_queued || !p->lay || is_mesh(&j->s)) continue; if (prev) prev->next = j->next; else A.head = j->next; if (A.tail == j) A.tail = prev; j->next = j->prev = NULL; return j; } return NULL; } // A worker for stage libraries and fast links only: a waiting draw's jobs (U) first, so a draw that waits finds one // free even while the pool builds pipelines and the drawing thread never builds a library itself; then the fast link // of a draw-queue job (fast_take), whose optimised build goes on to the pool. static void lib_worker(void *arg) { (void)arg; plat_mutex_lock(&P.lock); for (;;) { Job *j = NULL; while (!U.head && !(j = fast_take())) plat_cond_wait(&lib_cv, &P.lock); if (!j) { LibJob *lj = U.head; if (!(U.head = lj->next)) U.tail = NULL; if (lj->b) lj->b->job = NULL; u_run(lj); continue; } Pipe *p = j->p; int final; p->fast_tried = 1; // no other thread links it meanwhile; a draw that waits for it waits for this link VkPipeline f = gpl_link(&j->s, &j->k, p->lay, 1, &final); // releases P.lock while building if (f && !p->info.pipeline) { p->fast = p->info.pipeline = f, P.g.wlinks++, P.g.flinks++; ready_note(p, 1); plat_cond_broadcast(&done_cv); } else if (f) { vkDestroyPipeline(P.device, f, NULL); } if (p->info.pipeline) { // the optimised build: ahead of the prebuild, behind other draws' misses j->low = 1, P.g.requeued++; j->next = A.low; if (A.low) A.low->prev = j; else A.low_tail = j; A.low = j; } else { // no fast link for this one: its whole build, next in the draw queue if (!(j->next = A.head)) A.tail = j; A.head = j; } plat_cond_signal(&work_cv); } } static void pipe_worker(void *arg) { int me = (int)(intptr_t)arg; plat_mutex_lock(&P.lock); for (;;) { // With 3 or more workers one never takes low-queue work, so a waiting draw's library starts at once // (not with NX2NAVI_PIPE_WAIT=draw). while (!U.head && !A.head && !(A.low && (A.threads < 3 || !U.on || A.low_running < A.threads - 1))) { U.idle++; plat_cond_wait(&work_cv, &P.lock); U.idle--; } if (U.head) { // a library a draw waits for LibJob *lj = U.head; if (!(U.head = lj->next)) U.tail = NULL; if (lj->b) lj->b->job = NULL; if (A.prio_low[me]) A.prio_low[me] = 0, plat_thread_priority(PLAT_PRIO_NORMAL); u_run(lj); // releases P.lock while building continue; } Job *j = A.head; int lowrun = !j; if (j) { if (!(A.head = j->next)) A.tail = NULL; } else { j = A.low; if ((A.low = j->next)) A.low->prev = NULL; else A.low_tail = NULL; A.low_running++; } j->next = j->prev = NULL; j->worker = me; if (j->low != A.prio_low[me]) { // under P.lock, so a draw raising this thread (promote) is not undone A.prio_low[me] = j->low; plat_thread_priority(j->low ? PLAT_PRIO_BELOW_NORMAL : PLAT_PRIO_NORMAL); } // Step 1 for a job a draw waits for: its stage libraries and the fast link. Step 2, the optimised build, goes to // the head of the low queue, so a burst of new pipelines gets every fast link before any optimised build. if (P.gpl && (!j->prebuild || j->demanded) && !j->p->info.pipeline && !j->p->fast_tried && j->p->lay && !is_mesh(&j->s)) { int final; VkPipeline f = gpl_link(&j->s, &j->k, j->p->lay, 1, &final); // releases P.lock while building j->p->fast_tried = 1; if (f && !j->p->info.pipeline) { j->p->fast = j->p->info.pipeline = f, P.g.wlinks++; ready_note(j->p, 1); plat_cond_broadcast(&done_cv); j->low = 1, j->worker = -1, P.g.requeued++; j->prev = NULL, j->next = A.low; if (A.low) A.low->prev = j; else A.low_tail = j; A.low = j; A.low_running -= lowrun; continue; } else if (f) { vkDestroyPipeline(P.device, f, NULL); } } plat_mutex_unlock(&P.lock); VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off); plat_mutex_lock(&P.lock); if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one ready_note(j->p, 0); if (pipe && j->p->fast) P.g.swaps++; j->p->building = 0; j->p->job = NULL; if (pipe) { P.stats.graphics++, P.stats.async_built++; if (!j->prebuild) keys_note(&j->s); } else { P.stats.failures++; rt_log("vk_pipeline: graphics pipeline not built: pipeline creation failed"); } if (j->prebuild) { if (pipe) K.built++; else K.failed++; K.promoted += j->demanded; if (!--K.left && !K.loading) prebuild_report(); } A.low_running -= lowrun; if (!--A.inflight) plat_cond_broadcast(&A.idle); plat_cond_broadcast(&done_cv); if (pipe && (!j->prebuild || j->demanded)) perf_count(PERF_CNT_PIPE_ASYNC); // a draw waited for it free(j); } return; } static int hint_on(void); // Starts the pool on first use: a quarter of the logical processors, 2 to 4. Caller holds P.lock. static int workers_start(void) { if (A.threads) return 1; uint32_t cpus = plat_cpu_usable(); int n = cpus / 4 < 2 ? 2 : cpus / 4 > MAX_WORKERS ? MAX_WORKERS : (int)(cpus / 4); for (int i = 0; i < n; i++) { PlatThread *t = plat_thread_new(pipe_worker, (void *)(intptr_t)A.threads, "pipe worker"); if (!t) break; A.thread[A.threads++] = t; // kept: promote() raises a worker running a prebuild job } // Workers for stage libraries and fast links alone: two (the vertex and fragment libraries of one waiting draw in // parallel), and as many as the pool has with the look-ahead, which queues the libraries of a whole submission's // new pipelines at once (vk_pipeline_hint), or with the fast links of skipped draws (lib_worker). They sleep // unless a library or a fast link is queued. int nlib = hint_on() || fast_workers() ? n : 2; for (int i = 0; A.threads && U.on && P.gpl && i < nlib; i++) if (plat_thread_new(lib_worker, NULL, "pipe lib worker")) U.lib_workers++; if (A.threads) rt_log("vk_pipeline: %d background graphics pipeline workers, %d for waiting draws' libraries", A.threads, U.lib_workers); else rt_log("vk_pipeline: cannot start pipeline workers; building synchronously"); return A.threads != 0; } static void workers_drain(void) { plat_mutex_lock(&P.lock); K.loading = 0; for (Job *j = A.low, *n; j; j = n) { // queued prebuild jobs are dropped, not built n = j->next; j->p->building = 0, j->p->job = NULL; A.inflight--, K.left--; free(j); } A.low = A.low_tail = NULL; for (LibJob *j = U.head, *n; j; j = n) { // queued library jobs too: their entries read as failed builds n = j->next; j->b->job = NULL, j->b->building = 0; free(j); } U.head = U.tail = NULL; while (A.inflight) plat_cond_wait(&A.idle, &P.lock); while (U.running) plat_cond_wait(&done_cv, &P.lock); plat_mutex_unlock(&P.lock); } void vk_pipeline_async_enable(void) { U.on = strcmp(cfg_value("NX2NAVI_PIPE_WAIT"), "draw") != 0; const char *e = cfg_get("NX2NAVI_PIPE_SYNC"); if (e && *e && strcmp(e, "0")) return; A.on = 1; } int vk_pipeline_async_on(void) { return A.on; } // A draw needs a pipeline a prebuild job owns: the job moves to the draw queue, or its worker to normal priority. // Caller holds P.lock. static void promote(Pipe *p) { Job *j = p->job; if (!j || !j->low) return; if (j->prebuild && !j->demanded && !p->info.pipeline) p->t_miss = plat_now_ns(); j->low = 0, j->demanded = 1; if (j->worker < 0) { if (j->prev) j->prev->next = j->next; else A.low = j->next; if (j->next) j->next->prev = j->prev; else A.low_tail = j->prev; j->next = j->prev = NULL; if (A.tail) A.tail->next = j; else A.head = j; A.tail = j; plat_cond_signal(&work_cv); if (U.lib_workers) plat_cond_signal(&lib_cv); // its fast link (lib_worker) } else if (A.prio_low[j->worker]) { A.prio_low[j->worker] = 0; plat_thread_set_priority(A.thread[j->worker], PLAT_PRIO_NORMAL); } } // Enters a missing pipeline: built here (mode 0), or queued for a draw (1) or for the prebuild (2). Failures are // entered too. *queued: 1 when a job owns it. Caller holds P.lock. static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode, int *queued) { *queued = 0; const char *why = NULL; Bindings B = {0}; Layout *l = NULL; uint32_t span = binding_span(s->vs); int mesh = is_mesh(s); if (mesh) { // Mesh draws (VK_EXT_mesh_shader): one set holding the mesh stage's bindings, the fragment stage's after them and // the task stage's after those (create_graphics renumbers both modules); no libraries (the GPL fast link is for // vertex pipelines). They go to the workers like the others: built here, each new one stopped the frame for its // whole compile (0.26-0.51 s in Kokiri Forest), and they are recorded for the boot prewarm (keys_note). uint32_t fspan = s->fs ? binding_span(s->fs) : 1; if (!vk_ctx()->has_mesh_shader) why = "the GPU has no mesh shaders"; else if ((s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS) || (s->ts && gfx_shader_stage(s->ts) != GFX_STAGE_TS)) why = "stage mismatch"; else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1)) || (s->ts && !add_stage(&B, s->ts, span - 1 + fspan - 1))) why = "too many bindings"; else if (!(l = layout_get(&B))) why = "descriptor layout creation failed"; } else if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) why = "stage mismatch"; else if (k->topo_class == 3) why = "patches need tessellation stages, which are not translated"; else if (P.indep) { Bindings F = {0}; if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&F, s->fs, 0))) why = "too many bindings"; else if (!(l = layout_pair(&B, &F))) why = "descriptor layout creation failed"; B.pool |= F.pool; span = 1; } else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1))) why = "too many bindings"; else if (!(l = layout_get(&B))) why = "descriptor layout creation failed"; char lack[128]; if (!why && !layout_covers(s, l, span - 1, lack, sizeof lack)) why = lack, l = NULL; VkPipeline pipe = VK_NULL_HANDLE; Job *j = NULL; if (!why && mode && workers_start() && (j = calloc(1, sizeof *j))) { *j = (Job){.l = l, .fs_off = span - 1, .low = mode == 2, .prebuild = mode == 2, .worker = -1, .s = *s, .k = *k}; } else if (!why && !(pipe = create_graphics(s, k, l, span - 1))) { why = "pipeline creation failed"; } if (why) { P.stats.failures++; rt_log("vk_pipeline: graphics pipeline not built: %s", why); } else if (!j) { P.stats.graphics++; if (mode != 2) keys_note(s); } Pipe *p = pipe_add(k, sizeof *k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_GRAPHICS, gfx_shader_inputs(s->vs, NULL)); if (!j || !p) { free(j); return p; } j->p = p; p->building = 1, p->job = j; if (mode != 2) p->t_miss = plat_now_ns(); if (mode == 2) { j->prev = A.low_tail; if (A.low_tail) A.low_tail->next = j; else A.low = j; A.low_tail = j; K.left++; } else { if (A.tail) A.tail->next = j; else A.head = j; A.tail = j; if (U.lib_workers && fast_workers()) plat_cond_signal(&lib_cv); // its fast link (lib_worker) } A.inflight++; plat_cond_signal(&work_cv); *queued = 1; return p; } // A job still in the draw queue goes to the head of the low queue: its pipeline has (or is about to have) a fast-linked // one, so the optimised build waits behind other draws' misses. Caller holds P.lock. static void job_lower(Job *j) { if (!j || j->worker >= 0 || j->low) return; if (A.head == j) { if (!(A.head = j->next)) A.tail = NULL; } else { Job *q = A.head; while (q && q->next != j) q = q->next; if (!q) return; if (!(q->next = j->next)) A.tail = q; } j->low = 1, P.g.requeued++; j->prev = NULL, j->next = A.low; if (A.low) A.low->prev = j; else A.low_tail = j; A.low = j; } // A draw waits for pipeline s: its missing pre-rasterization and fragment shader libraries go to the head of the // worker queues (U), so they build in parallel on idle workers, and this thread sleeps until both are published or // end (now_ms) passes. A job no worker took within 1 ms (all busy) is built here. Caller holds P.lock. static void u_push(LibJob *j) { if (U.tail) U.tail->next = j; else U.head = j; U.tail = j; } // Queues s's missing pre-rasterization and fragment shader libraries (b: both entries). 0 when one cannot be entered // (no layout half, out of memory). Caller holds P.lock. static int lib_queue(const VkPipeState *s, const GKey *k, Layout *l, Lib *b[2]) { ViKey vk_; PrKey pk; FsKey fk; FoKey ok; lib_keys(s, k, &vk_, &pk, &fk, &ok); U.waits++; const void *key[2] = {&pk, &fk}; uint32_t size[2] = {sizeof pk, sizeof fk}; int queued = 0; for (int i = 0; i < 2; i++) { uint64_t h = key_hash(key[i], size[i]); if ((b[i] = lib_find(h, key[i], size[i]))) continue; if (!half_layout(l, i)) return 0; // gpl_link reports it LibJob *j = calloc(1, sizeof *j); if (!j || !(b[i] = lib_new(h, key[i], size[i]))) { free(j); return 0; } *j = (LibJob){.b = b[i], .kind = i ? LIB_FS : LIB_PR, .l = l, .s = *s, .k = *k}; b[i]->job = j; u_push(j); U.queued++, queued++; } if (queued) plat_cond_broadcast(&work_cv), plat_cond_broadcast(&lib_cv); return 1; } // A waiting draw's whole fast link of p on the library workers: its library jobs, then the link job behind them. // The caller then waits for p->info.pipeline. 0 when nothing was queued (the caller links as before). Under P.lock. static int link_queue(Pipe *p, const VkPipeState *s, const GKey *k, Layout *l) { Lib *b[2]; if (!U.lib_workers || !lib_queue(s, k, l, b)) return 0; LibJob *j = calloc(1, sizeof *j); if (!j) return 0; *j = (LibJob){.l = l, .s = *s, .k = *k, .p = p}; p->link_queued = 1; u_push(j); plat_cond_broadcast(&work_cv), plat_cond_broadcast(&lib_cv); return 1; } static void lib_wait(const VkPipeState *s, const GKey *k, Layout *l, double end) { Lib *b[2]; if (!lib_queue(s, k, l, b)) return; double steal_at = now_ms() + 1.0; for (;;) { if (!b[0]->building && !b[1]->building) return; double now = now_ms(); if (now >= end) { U.timeouts++; return; } Lib *q = b[0]->job ? b[0] : b[1]->job ? b[1] : NULL; if (q && !U.lib_workers && (now >= steal_at || !U.idle)) { U.stolen++; lib_steal(q); // releases P.lock while building continue; } double until = q && steal_at < end ? steal_at : end; plat_cond_wait_ms(&done_cv, &P.lock, (uint32_t)(until - now) + 1); } } // ---- stage libraries ahead of their draws (NX2NAVI_PIPE_AHEAD, default 1; 0 off; vk_exec.c "looking ahead") // The replay names the state of a draw it has not reached yet. When no pipeline is known for it, its pre-rasterization // and fragment shader libraries are queued to the library workers now, so they build while the draws before it are // recorded, in parallel with each other and with those of the other draws named: the draw's own lookup then finds both // and links on its thread in microseconds instead of waiting for them or being skipped while they build. Nothing else // is entered, no pipeline, no optimised build and no key record: a state the scan guessed wrong costs two libraries // at most. The feedback loop bits, the topology and the targets do not matter here, as no shader library depends on // them. Returns the libraries queued. static int hint_on(void) { static int on = -1; if (on < 0) on = cfg_bool("NX2NAVI_PIPE_AHEAD"); return on; } int vk_pipeline_hint(const VkPipeState *s) { if (!A.on || !U.on || !hint_on() || !s->vs || !ensure_init() || !P.gpl || !P.indep || is_mesh(s)) return 0; if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) return 0; GKey k; gkey_fill(s, &k); if (k.topo_class == 3) return 0; uint64_t h = key_hash(&k, sizeof k); int queued = 0; plat_mutex_lock(&P.lock); if (!pipe_find(&k, sizeof k, h)) { Bindings B = {0}, F = {0}; Layout *l = add_stage(&B, s->vs, 0) && (!s->fs || add_stage(&F, s->fs, 0)) ? layout_pair(&B, &F) : NULL; Lib *b[2]; uint64_t q0 = U.queued, w0 = U.waits; if (l && l->set1 && workers_start() && lib_queue(s, &k, l, b)) queued = (int)(U.queued - q0); U.waits = w0, U.queued = q0; // those count waiting draws' jobs P.g.hints++, P.g.hint_libs += (uint64_t)queued; } plat_mutex_unlock(&P.lock); return queued; } static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int async, int *pending, uint32_t wait_ms) { if (info) *info = NULL; if (pending) *pending = 0; if (!s->vs || !ensure_init()) return VK_NULL_HANDLE; GKey k; gkey_fill(s, &k); uint64_t h = key_hash(&k, sizeof k); plat_mutex_lock(&P.lock); Pipe *p = pipe_find(&k, sizeof k, h); int queued; if (p) { if (!p->building) P.stats.hits++; } else { p = pipe_miss(s, &k, h, async, &queued); } // A draw that would wait (wait_ms) gets its missing shader libraries built at once, the vertex and fragment ones in // parallel on idle workers (lib_wait; about 0.12 ms per library from the driver's disk cache, 5-180 ms cold), and // links them here: its own job may queue behind optimised builds of earlier pipelines. The job then only does the // optimised build, from the low queue. Other draws leave the libraries to the job and are skipped meanwhile. // NX2NAVI_PIPE_WAIT=draw builds them on this thread one after the other instead, as before; NX2NAVI_GPL_DRAW_BUILD=0 // leaves them to the job. static int draw_build = -1; if (draw_build < 0) { const char *e = cfg_get("NX2NAVI_GPL_DRAW_BUILD"); draw_build = !(e && !strcmp(e, "0")); rt_log("vk_pipeline: a draw that waits for its pipeline %s", !draw_build ? "leaves its libraries to its job" : U.on ? "has its libraries built in parallel on the workers, ahead of other jobs" : "builds its libraries itself (NX2NAVI_PIPE_WAIT=draw)"); } double end = wait_ms ? now_ms() + wait_ms : 0; if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && !p->link_queued && p->lay && !is_mesh(s) && wait_ms && draw_build && U.on && link_queue(p, s, &k, (Layout *)p->lay)) { // the libraries and the link on the library workers; this thread waits below without building p->fast_tried = 1; job_lower(p->job); } else if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && !p->link_queued && p->lay && !is_mesh(s)) { int final = 0, build = wait_ms != 0 && draw_build; uint64_t p0 = perf_begin(), t0 = qpc(); if (build && U.on) { job_lower(p->job); // its worker would only wait for the same libraries; it links them if this draw cannot lib_wait(s, &k, (Layout *)p->lay, end); // releases P.lock while waiting build = 0; // links here when both libraries exist (gpl_link build 0) } else if (build) { p->fast_tried = 1; // the job, if a worker takes it meanwhile, goes straight to the optimised build } VkPipeline f = p->info.pipeline ? VK_NULL_HANDLE : gpl_link(s, &k, p->lay, build, &final); // build 1 releases P.lock perf_end(PERF_PIPE, p0); if (wait_ms && draw_build) { double ms = ms_since(t0); P.g.dlinks++, P.g.dlink_ms += ms; if (ms > P.g.dlink_max) P.g.dlink_max = ms; if (ms > 5.0 && P.g.slow++ < 16) rt_log("vk_pipeline: gpl: a waiting draw %s its libraries in %.1f ms", U.on ? "got" : "built", ms); } if (final) p->fast_tried = 1; if (f && !p->info.pipeline) { p->fast = p->info.pipeline = f; ready_note(p, 1); job_lower(p->job); plat_cond_broadcast(&done_cv); } else if (f) { vkDestroyPipeline(P.device, f, NULL); // a worker published one while the lock was free } } if (p && p->building && !p->info.pipeline) { promote(p); uint64_t p0 = wait_ms ? perf_begin() : 0; // the wait counts as the render thread's pipeline time double now; while (p->building && !p->info.pipeline && wait_ms && (now = now_ms()) < end) plat_cond_wait_ms(&done_cv, &P.lock, (uint32_t)(end - now) + 1); if (wait_ms) perf_end(PERF_PIPE, p0); if (wait_ms && !p->building) perf_count(PERF_CNT_PIPE_WAIT); } int wait = p && p->building && !p->info.pipeline; if (wait) P.stats.pending_skips++; VkPipeline pipe = p ? p->info.pipeline : VK_NULL_HANDLE; plat_mutex_unlock(&P.lock); if (wait) { if (pending) *pending = 1; return VK_NULL_HANDLE; } if (!p) return VK_NULL_HANDLE; if (info) *info = &p->info; return pipe; } VkPipeline vk_pipeline_graphics(const VkPipeState *s, const VkPipeInfo **info) { return graphics(s, info, 0, NULL, 0); } VkPipeline vk_pipeline_graphics_async(const VkPipeState *s, const VkPipeInfo **info, int *pending, uint32_t wait_ms) { return graphics(s, info, A.on, pending, wait_ms); } // ---- boot prewarm from the key file (comment at "pipeline keys") static int id_cmp(const void *x, const void *y) { return memcmp(x, y, 32); } // Builds queued prebuild jobs until the low queue is empty or K.until passes, beside the pipe_worker pool (same // bookkeeping). Jobs left queued at K.until stay for the pool. static void warm_builder(void *arg) { (void)arg; plat_mutex_lock(&P.lock); while (A.low && !(K.until && now_ms() >= K.until)) { Job *j = A.low; if ((A.low = j->next)) A.low->prev = NULL; else A.low_tail = NULL; j->next = j->prev = NULL; j->low = 0; // not on a pool thread: promote() has nothing to raise plat_mutex_unlock(&P.lock); VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off); plat_mutex_lock(&P.lock); if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one ready_note(j->p, 0); if (pipe && j->p->fast) P.g.swaps++; j->p->building = 0; j->p->job = NULL; if (pipe) { P.stats.graphics++, P.stats.async_built++, K.built++; } else { P.stats.failures++, K.failed++; rt_log("vk_pipeline: graphics pipeline not built: pipeline creation failed"); } K.promoted += j->demanded; K.left--; if (!--A.inflight) plat_cond_broadcast(&A.idle); plat_cond_broadcast(&done_cv); free(j); } plat_mutex_unlock(&P.lock); return; } static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]); static int pm_skipped(void); static void pm_set(void); // vk_pipeline_prewarm_progress's numbers: the phase, and the pipelines finished (built or failed) of those queued, copied // while prewarm_run waits for its builders. static int32_t pw_phase; static int64_t pw_pipe_done, pw_pipe_total; void vk_pipeline_prewarm_progress(VkPrewarmProgress *p) { p->phase = __atomic_load_n(&pw_phase, __ATOMIC_ACQUIRE); gfx_shader_prewarm_progress(&p->sh_done, &p->sh_total, &p->sh_translated); p->pipe_done = __atomic_load_n(&pw_pipe_done, __ATOMIC_RELAXED); p->pipe_total = __atomic_load_n(&pw_pipe_total, __ATOMIC_RELAXED); } static void prewarm_run(void); // The boot's share of the pipeline prebuild with the graphics pipeline library. The boot builds the recorded // pipelines in record order, so those met first in play (the title screen, the first places) come first, for at most // this long after the shaders are ready; the rest stay queued for the pool workers, which build them below normal // priority while the game runs. A draw that meets one of them before it is built gets a pipeline linked from stage // libraries at once (gpl_link), as for any new pipeline in play. After a driver update, when every pipeline is built // from nothing, the boot thus waits this long instead of for all of them (15 minutes and more with a long play // history). Without the library a draw would have to wait for a whole build, so the boot waits for all; so it does // with NX2NAVI_PREWARM_FULL=1. #define PREWARM_PIPE_MS 20000.0 void vk_pipeline_prewarm(void) { prewarm_run(); __atomic_store_n(&pw_phase, 3, __ATOMIC_RELEASE); } static void prewarm_run(void) { const char *gpu = rt_config()->gpu; uint32_t pr_hist[16] = {0}, fs_hist[4] = {0}; // library state of the recorded pipelines (lb_start) if (!A.on || (gpu && !strcmp(gpu, "null"))) return; if (!cfg_bool("NX2NAVI_PIPE_PREBUILD")) { // prebuild_start logs the off case if (ensure_init()) lb_start(pr_hist, fs_hist); return; } double t0 = now_ms(); K.warm = 1; if (!ensure_init() || !K.file_n) { rt_log("vk_pipeline: prewarm: no pipeline keys (%s)", K.path[0] ? K.path : ""); if (P.ready) lb_start(pr_hist, fs_hist); return; } const KeyRec *r = (const KeyRec *)K.file; size_t n = K.file_n, m = 0; uint8_t (*ids)[32] = malloc(3 * n * 32); const GfxShader **sh = calloc(3 * n, sizeof *sh); if (!ids || !sh) { free(ids), free(sh); return; } for (size_t i = 0; i < n; i++) { memcpy(ids[m++], r[i].vs, 32); if (r[i].has_fs) memcpy(ids[m++], r[i].fs, 32); if (r[i].has_ts) memcpy(ids[m++], r[i].ts, 32); } qsort(ids, m, 32, id_cmp); size_t u = 0; for (size_t i = 0; i < m; i++) if (!u || memcmp(ids[u - 1], ids[i], 32)) memcpy(ids[u++], ids[i], 32); int cpus = (int)plat_cpu_usable(); const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2; __atomic_store_n(&pw_phase, 1, __ATOMIC_RELEASE); gfx_shader_prewarm((const uint8_t(*)[32])ids, u, sh, cpus, procs < 1 ? 1 : procs); double t1 = now_ms(); vk_mem_stage_log("the shader prewarm"); if (P.indep) { // vertex output unions first: workers may take a job before the queue is complete uint64_t pair = 0, uni = 0, pairs = 0; for (size_t i = 0; i < n; i++) { const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp); const uint8_t (*f)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL; if (!v || !f || !sh[v - ids] || !sh[f - ids] || gfx_shader_stage(sh[v - ids]) != GFX_STAGE_VS) continue; GfxShaderIo io[32]; vs_outs(sh[v - ids], sh[f - ids], io); pair += (uint64_t)__builtin_popcount(gfx_shader_inputs(sh[f - ids], NULL)), pairs++; } for (uint32_t j = 0; j < VS_OUTS_CAP; j++) uni += VO.vs[j] ? (uint64_t)__builtin_popcount(VO.mask[j]) : 0; rt_log("vk_pipeline: vertex output unions: %u vertex shaders, %.2f locations each; %llu pairs read %.2f each", VO.n, VO.n ? (double)uni / VO.n : 0.0, (unsigned long long)pairs, pairs ? (double)pair / pairs : 0.0); } if (pm_skipped()) { // the boot's pipeline builds killed an earlier start on this GPU and driver plat_mutex_lock(&P.lock); free(K.file); K.file = NULL; plat_mutex_unlock(&P.lock); free(ids), free(sh); lb_start(pr_hist, fs_hist); return; } pm_set(); uint32_t queued = 0, dropped = 0, known = 0; plat_mutex_lock(&P.lock); K.t0 = t1; K.loading = 1; for (size_t i = 0; i < n; i++) { const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp); const uint8_t (*f)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL; const uint8_t (*t)[32] = r[i].has_ts ? bsearch(r[i].ts, ids, u, 32, id_cmp) : NULL; const GfxShader *vs = v ? sh[v - ids] : NULL, *fs = f ? sh[f - ids] : NULL, *ts = t ? sh[t - ids] : NULL; if (!vs || (r[i].has_fs && !fs) || (r[i].has_ts && !ts)) { dropped++; continue; } VkPipeState s = r[i].s; s.vs = vs, s.fs = fs, s.ts = ts; GKey k; gkey_fill(&s, &k); if (!is_mesh(&s)) { // the stage libraries lb_start prebuilds are for vertex pipelines pr_hist[(k.clamp & 1) | (k.poly_mode & 3) << 1 | (k.minus_w & 1) << 3]++; fs_hist[k.samples >= 8 ? 3 : k.samples >= 4 ? 2 : k.samples >= 2 ? 1 : 0]++; } uint64_t h = key_hash(&k, sizeof k); int q = 0; if (pipe_find(&k, sizeof k, h)) known++; else pipe_miss(&s, &k, h, 2, &q), queued += (uint32_t)q; } K.loading = 0; plat_mutex_unlock(&P.lock); free(ids), free(sh); __atomic_store_n(&pw_pipe_done, 0, __ATOMIC_RELAXED); __atomic_store_n(&pw_pipe_total, (int64_t)queued, __ATOMIC_RELAXED); __atomic_store_n(&pw_phase, 2, __ATOMIC_RELEASE); const char *full = cfg_get("NX2NAVI_PREWARM_FULL"); K.until = P.gpl && !(full && !strcmp(full, "1")) ? now_ms() + PREWARM_PIPE_MS : 0; // before the builders start PlatThread *t[64]; int nt = 0; for (int i = 0; i < cpus && i < 64 && queued; i++) if ((t[nt] = plat_thread_new(warm_builder, NULL, "pipe prewarm"))) nt++; if (!nt) warm_builder(NULL); double logged = now_ms(); for (int i = 0; i < nt; i++) while (!plat_thread_join_ns(t[i], 100000000u)) { // every 100 ms the numbers, every 2 s a log line uint32_t fin = __atomic_load_n(&K.built, __ATOMIC_RELAXED) + __atomic_load_n(&K.failed, __ATOMIC_RELAXED); __atomic_store_n(&pw_pipe_done, (int64_t)fin, __ATOMIC_RELAXED); if (now_ms() - logged < 2000) continue; logged = now_ms(); rt_log("vk_pipeline: prewarm: %u of %u pipelines, %.0f ms", fin, queued, now_ms() - t1); } __atomic_store_n(&pw_pipe_done, (int64_t)queued, __ATOMIC_RELAXED); plat_mutex_lock(&P.lock); uint32_t later = 0; for (Job *j = A.low; j; j = j->next) later += j->prebuild; if (!later) // all built: wait for those a pool worker took while (K.left) plat_cond_wait(&done_cv, &P.lock); else // the boot's time is up: the pool builds the rest while the game runs plat_cond_broadcast(&work_cv); rt_log("vk_pipeline: prewarm: %zu records in %s: %u pipelines built, %u left to build in the background, %u " "failed, %u already built, %u dropped (shader neither cached nor stored); shaders %.0f ms, pipelines %.0f " "ms on %d threads, total %.0f ms", n, K.path, K.built, later, K.failed, known, dropped, t1 - t0, now_ms() - t1, nt, now_ms() - t0); free(K.file); K.file = NULL; plat_mutex_unlock(&P.lock); fb_log("after the boot prewarm"); vk_mem_stage_log("the recorded pipelines' prebuild"); vk_pipeline_prewarm_marker_clear(); // the boot's share is done; what the pool builds later runs during play lb_start(pr_hist, fs_hist); } // ---- boot prebuild crash marker // The boot builds the recorded pipelines (warm_builder) before the game starts. A driver whose compiler crashes on one // of them would kill every start at the same point, and the preparation journal above does not cover these builds. // So /pipeline-prewarm-running.txt exists while they run: written before the first one, deleted when the boot's // share ends (or when the boot stops waiting and lets the game start). A start that finds it left over records this // GPU and driver in /pipeline-prewarm-skip.txt, and starts on them build no recorded pipelines ahead of play: // each one is built at its first draw instead, as with NX2NAVI_PIPE_PREBUILD=0. Another driver version, or deleting // the file, brings the prebuild back. Only the boot's own share is covered: a crash during play, where the pool also // builds what the boot left, says nothing about these builds. static int cache_dir(char *dir, size_t n); static void gpu_key(char *out, size_t n); static char PM_run[PLAT_PATH_MAX * 2 + 48]; static volatile int32_t pm_on; static int pm_skipped(void) { char dir[PLAT_PATH_MAX * 2], sp[sizeof PM_run], key[32]; if (!cache_dir(dir, sizeof dir)) return 0; snprintf(PM_run, sizeof PM_run, "%s/pipeline-prewarm-running.txt", dir); snprintf(sp, sizeof sp, "%s/pipeline-prewarm-skip.txt", dir); gpu_key(key, sizeof key); uint64_t n = 0; uint8_t *left = plat_file_read(PM_run, &n); if (left) { char text[256]; int len = snprintf(text, sizeof text, "# The boot's pipeline prebuild did not finish on this GPU and driver (runtime/vk_pipeline.c " "pm_skipped): starts skip it\n# while this line matches. Delete this file to build ahead of play " "again.\n%s\n", key); if (plat_file_write_atomic(sp, text, (size_t)len)) plat_file_delete(PM_run); rt_log("vk_pipeline: the last start died while the boot built pipelines (%s was left); %s now skips that " "prebuild on this GPU and driver (%s)", PM_run, sp, key); free(left); } uint8_t *skip = plat_file_read(sp, &n); int on = 0; for (char *line = (char *)skip, *next; line && *line && !on; line = next) { next = strchr(line, '\n'); if (next) *next++ = 0; size_t l = strlen(line); while (l && (line[l - 1] == '\r' || line[l - 1] == ' ')) line[--l] = 0; on = line[0] != '#' && !strcmp(line, key); } free(skip); if (on) rt_log("vk_pipeline: prewarm: %zu recorded pipelines not built ahead of play: an earlier start died while building " "them on this GPU and driver (%s lists %s); each builds at its first draw", K.file_n, sp, key); return on; } static void pm_set(void) { if (!PM_run[0]) return; char key[32]; gpu_key(key, sizeof key); // straight to the disk: the file has to be there if the driver takes the process down a moment later if (plat_file_write_atomic(PM_run, key, strlen(key))) plat_xchg32(&pm_on, 1); } void vk_pipeline_prewarm_marker_clear(void) { if (plat_xchg32(&pm_on, 0)) plat_file_delete(PM_run); } // ensure_init's hook: without a prewarm (NX2NAVI_PIPE_PREBUILD=0, or the device came up first elsewhere) the records // are not built. Caller holds no lock. static void prebuild_start(void) { plat_mutex_lock(&P.lock); if (K.file && !K.warm) { rt_log("vk_pipeline: prebuild off%s; %zu records not built", !cfg_bool("NX2NAVI_PIPE_PREBUILD") ? " (NX2NAVI_PIPE_PREBUILD=0)" : " (no boot prewarm)", K.file_n); free(K.file); K.file = NULL; } plat_mutex_unlock(&P.lock); } // NX2NAVI_SPV_OVERRIDE= (test): a compute shader whose SPIR-V hashes (FNV-1a over its bytes, the name vk_dump.c // gives it) to a file /.spv runs that SPIR-V instead, an edited copy with the same interface, to see a // shader's intermediate values in a frame dump. NULL: no override (malloc'd words otherwise). static uint32_t *spv_override(const uint32_t *code, uint64_t words, uint64_t *out) { const char *dir = cfg_get("NX2NAVI_SPV_OVERRIDE"); if (!dir || !*dir) return NULL; uint64_t h = 0xcbf29ce484222325ull; for (uint64_t i = 0; i < words * 4; i++) h = (h ^ ((const uint8_t *)code)[i]) * 0x100000001b3ull; char path[600]; snprintf(path, sizeof path, "%s/%016llx.spv", dir, (unsigned long long)h); FILE *f = fopen(path, "rb"); if (!f) return NULL; fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET); uint32_t *w = n > 20 && n % 4 == 0 ? malloc((size_t)n) : NULL; if (w && fread(w, 1, (size_t)n, f) != (size_t)n) free(w), w = NULL; fclose(f); if (w) *out = (uint64_t)n / 4, rt_log("vk_pipeline: compute shader %016llx replaced by %s", (unsigned long long)h, path); return w; } // A skipped froxel's exit in the fog's first compute pass: the block at word at (an OpLabel) makes five image writes, four // of (0,0,0,1) and last the mask value (1/255,0,0,0), and returns. static int fog_skip_block(const uint32_t *w, uint64_t n, uint64_t at, const uint64_t *def, uint32_t bound) { int writes = 0; for (uint64_t i = at + (w[at] >> 16); i < n;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (!len || i + len > n) return 0; if (op == 253) return writes == 5; // OpReturn if (op == 248 || op == 249 || op == 250) return 0; // OpLabel, OpBranch, OpBranchConditional if (op == 99) { // OpImageWrite if (len < 4 || w[i + 3] >= bound || ++writes > 5) return 0; uint64_t c = def[w[i + 3]]; if (!c || (w[c] & 0xffff) != 80 || w[c] >> 16 != 7) return 0; // OpCompositeConstruct of four float v[4]; for (int k = 0; k < 4; k++) { uint32_t id = w[c + 3 + k]; uint64_t f = id < bound ? def[id] : 0; if (!f || (w[f] & 0xffff) != 43 || w[f] >> 16 != 4) return 0; // OpConstant of one word memcpy(&v[k], &w[f + 3], 4); } if (writes == 5 ? !(v[0] > 0.0039f && v[0] < 0.0040f && v[1] == 0 && v[2] == 0 && v[3] == 0) : !(v[0] == 0 && v[1] == 0 && v[2] == 0 && v[3] == 1)) return 0; } i += len; } return 0; } // The volumetric fog's first compute pass skips a froxel behind a coarse depth table and writes it as empty, and the next // pass's mask keeps it empty. That table comes from the frame before, so in the columns a near object has just left, every // froxel past the object's old depth came out with no fog: a dark block of the fog grid beside the object, which the fog's // history then smeared into a trail, and worst at the screen edges, where near objects move fastest. A froxel behind // today's depth is never seen, so computing it only costs a little time: the comparison is made never true (x > x). // Returns the patched copy of that shader, NULL for any other. Found by its shape rather than a hash, so it holds across // translator updates: a branch whose other side is that exit, taken on !(distance > table) || flag. // NX2NAVI_FOG_CULL=1 keeps the game's skip. static uint32_t *spirv_fog_keep_froxels(const uint32_t *w, uint64_t n) { const char *e = cfg_get("NX2NAVI_FOG_CULL"); if (n < 6 || (e && !strcmp(e, "1"))) return NULL; uint32_t bound = w[3]; int writes = 0; for (uint64_t i = 5; i < n;) { uint32_t len = w[i] >> 16; if (!len || i + len > n) return NULL; writes += (w[i] & 0xffff) == 99; i += len; } if (writes < 5) return NULL; uint64_t *def = calloc(bound, sizeof *def); // word of each id's definition, for the instructions looked at below if (!def) return NULL; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (op == 248 && len >= 2 && w[i + 1] < bound) def[w[i + 1]] = i; // OpLabel else if ((op == 43 || op == 80 || op == 166 || op == 168 || op == 186) && len >= 3 && w[i + 2] < bound) def[w[i + 2]] = i; i += len; } uint64_t cmp = 0, prev = 0; int found = 0; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (op == 250 && len >= 4 && prev && (w[prev] & 0xffff) == 247 && w[prev + 1] == w[i + 3] && w[i + 1] < bound && w[i + 3] < bound && def[w[i + 3]] && fog_skip_block(w, n, def[w[i + 3]], def, bound)) { // OpSelectionMerge just before uint64_t c = def[w[i + 1]]; for (int k = 0; c && (w[c] & 0xffff) == 166 && k < 2; k++) { // OpLogicalOr uint32_t a = w[c + 3 + k]; uint64_t no = a < bound ? def[a] : 0; if (!no || (w[no] & 0xffff) != 168 || w[no + 3] >= bound) continue; // OpLogicalNot uint64_t gt = def[w[no + 3]]; if (gt && (w[gt] & 0xffff) == 186) cmp = gt, found++; // OpFOrdGreaterThan } } prev = i; i += len; } free(def); uint32_t *o = found == 1 ? malloc(n * 4) : NULL; if (!o) return NULL; memcpy(o, w, n * 4); o[cmp + 4] = o[cmp + 3]; LOG_ONCE("vk_pipeline: fog: froxels behind last frame's depth are computed too"); return o; } // A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised). // cached_only (needs P.cache_control): from P.cache or the driver's own disk cache, never compiled; VK_NULL_HANDLE when // neither holds it (the driver returns VK_PIPELINE_COMPILE_REQUIRED, a success code, with no pipeline). static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cached_only) { uint64_t words; const uint32_t *code = gfx_shader_spirv(cs, &words); uint64_t ow = 0; uint32_t *over = spv_override(code, words, &ow); if (over) code = over, words = ow; uint32_t *fog = spirv_fog_keep_froxels(code, words); if (fog) free(over), over = fog, code = fog; VkShaderModule m = make_module(code, words, 0); if (!m) return free(over), VK_NULL_HANDLE; VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqc; VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, .layout = l->layout, .flags = noopt_flag()}; if (cached_only) ci.flags |= VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT; pin_subgroup32(&ci.stage, &rqc, code, words); // Subgroup operations in a workgroup whose width is a whole number of subgroups need every subgroup full: the // driver may otherwise run partial ones, and a ballot over the culling's candidates then counted differently from // one frame to the next uint32_t lx = local_size_x(code, words); if (P.f_full_subgroups && vk_caps.sg_size && lx && lx % vk_caps.sg_size == 0 && uses_subgroup_ops(code, words)) ci.stage.flags |= VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; VkPipeline pipe = VK_NULL_HANDLE; VkPipelineCreationFeedbackCreateInfo fci; VkPipelineCreationFeedback fb; ci.pNext = fb_chain(&fci, &fb, ci.pNext); crash_note("a compute pipeline", cs, NULL, NULL); if (pipe_create(2, &ci, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); if (pipe) fb_note(FB_COMPUTE, &fb); cc_hit = !pipe ? -1 : pb_last ? 1 : !P.feedback || !P.cache || !(fb.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) ? -1 : (fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0; if (P.feedback >= 2 && (pipe || !cached_only)) rt_log("vk_pipeline: compute %016llx: %llu words, module %016llx, %s, %.1f ms", (unsigned long long)fnv(FNV0, code, words * 4), (unsigned long long)words, (unsigned long long)mm_hash, !pipe ? "failed" : pb_last ? "from pipeline-binaries.bin" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled", (double)fb.duration / 1e6); vkDestroyShaderModule(P.device, m, NULL); free(over); return pipe; } // The layout of a compute shader in *B; NULL with *why on failure. Caller holds P.lock. static Layout *compute_layout(const GfxShader *cs, Bindings *B, const char **why) { Layout *l = NULL; if (gfx_shader_stage(cs) != GFX_STAGE_CS) *why = "not a compute shader"; else if (!add_stage(B, cs, 0)) *why = "too many bindings"; else if (!(l = layout_get(B))) *why = "descriptor layout creation failed"; return l; } // ---- stage library prebuild // Two places build what a draw would otherwise build while the game waits: // - nx2navi.exe --prepare-shaders (lb_warm_prepared), after the translation, on every core: for each prepared variant // (shader-prepared.bin) the pre-rasterization library of a vertex shader, the fragment-shader library of a fragment // shader and the pipeline of a compute shader, each built from a private shader copy and destroyed at once. What // stays is the driver's own disk cache: on AMD a library built again in a later process takes 0.1 ms instead of // 55 ms (vertex) / 28 ms (fragment). Our pipeline-cache.bin is not written there: // it would grow by about 10 KB per library (~240 MB) and be read into memory at every boot. // - boot (lb_worker), after the pipeline-key prewarm, on two threads (lb_start) at PLAT_PRIO_IDLE (the short P.lock // sections and shader loads at normal priority, so a preempted thread never holds the lock the replay needs; a // compute pipeline's compile below normal priority, lb_compute): // compute pipelines of every recorded and prepared compute shader (vk_pipeline_compute builds compute // synchronously on the replay thread; 163 prepared): those of the shaders dispatched most recently stay, the others // are built into the pipeline cache and destroyed (comment at compute-used.bin). Then the libraries of the first // NX2NAVI_LIB_PREBUILD_KEEP variants play recorded (priority 0), kept in libs[] with their shaders. KEEP defaults to // 0: a kept library costs 140-460 KB of process memory (10,000 kept took 9 GB in run 1) to save a rebuild the driver // cache makes 0.1 ms. // Library state: the vertex output union known now (vs_union_peek; the prewarm fills it from pipeline-keys.bin, the // preparation has none), the clamp, polygon mode and depth mode most recorded pipelines use (defaults without a key // file: clamp 0, fill, -w..w), single-sampled, no dual source; the state dynamic on this GPU is 0 in every key. // A draw that meets such a variant in a new pipeline then gets its libraries from libs[] or the driver cache and // fast-links (gpl_link). NX2NAVI_LIB_PREBUILD=0 turns the boot part off, =N stops it after N variants; // NX2NAVI_LIB_PREBUILD_THREADS sets its thread count, NX2NAVI_LIB_PREBUILD_BIG its compute size limit (LB_BIG_WORDS). #define LB_MAX_THREADS 16 // The boot part builds a compute shader over LB_BIG_WORDS SPIR-V words only from the pipeline caches (compute_create // cached_only); one that no cache holds is left to its first dispatch, like any shader outside the warm list. Play // recorded four straight-line compute shaders of 0.87-4.4 million words (one function of up to 888K instructions; the // median of 3,648 compute shaders is 7,351 words, p99 64,844). NVIDIA 621.01 on an RTX 4090 compiled them on four // prebuild threads for minutes at 100% CPU each while the process grew from 12 GB to 28-31 GB, until RAM ran out and the // game stuttered and went black a few minutes after boot. None finished, so no cache held one and every boot did it // again. NX2NAVI_LIB_PREBUILD_BIG sets the limit in words; 0 builds every size. #define LB_BIG_WORDS (512u * 1024) static struct { GfxWarmEnt *e; size_t n; size_t n_keep; // e[0..n_keep): compute shaders whose pipeline stays (cu_keep) volatile int64_t next; volatile int32_t stop, running; int threads; PlatThread *th[LB_MAX_THREADS]; uint8_t clamp, poly, minus_w, samples; // library state (GKey fields) double t0, ms[3]; // ms: build time per kind (vertex, fragment, compute) // warmed: built for the caches only and destroyed; nolib: vertex and fragment variants skipped with the graphics // pipeline library off // skipped: variants of the skip list (SK below) uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, skipped, done, cs_draw; uint64_t big_words, deferred; // LB_BIG_WORDS; compute shaders over it left to their dispatch uint64_t big_unused; // compute shaders over it that no run dispatched: not built at all double warm_ms[3]; volatile int32_t kept; // libraries entered in libs[] so far int32_t keep_n; int idle; // build at PLAT_PRIO_IDLE (boot), normal (preparation) PlatMutex stat; } LB = {.stat = PLAT_MUTEX_INIT}; static __thread int lb_me = -1; // on a boot prebuild thread (lb_worker): its index in LB.th // ---- preparation that outlives a driver crash. A bug in the driver's shader compiler can kill the process inside // vkCreate*Pipelines: NVIDIA 615.78.08 on an RTX 5080 crashed on the same vertex library at the same point of every // preparation, natively and under Wine, so preparation never finished (issue #30 and a report on PR #60). // While lb_warm_prepared builds, slot k of /shader-prepare-journal.bin holds the variant its worker k has handed to // the driver (input id and stage; zeros between variants). The writes go straight to the OS, so they outlast the // process, and a run that finishes deletes the file. The next preparation or boot (skip_init) moves the variants a // leftover journal names into /shader-prepare-skip.txt with this GPU and driver version, and preparation and the // boot prebuild (lb_worker) skip what that file lists for them. The variants the other workers were building at the // crash are skipped along with the culprit; like it, they build at their first draw, where the driver can still crash. // Another driver version builds them again, and so does deleting the file. tools/prepare_shaders.py runs the build // again (--warm-shaders) after a run that left a journal. #define JR_SLOT 64 // bytes per journal slot: input id, stage, zeros #define JR_SLOTS 64 // lb_warm_prepared's thread cap static struct { PlatFile *journal; // while lb_warm_prepared builds uint8_t (*iid)[32]; // the skip list's variants for this GPU and driver size_t n; int loaded; } SK; static int cache_dir(char *dir, size_t n) { // the folder of pipeline-cache.bin and shader-prepared.bin snprintf(dir, n, "%s", P.cache_path); char *cut = plat_last_sep(dir); if (!cut) return 0; *cut = 0; return 1; } static void gpu_key(char *out, size_t n) { // vendor:device:driver version, as the skip list names a GPU and driver const VkPhysicalDeviceProperties *p = &vk_ctx()->props; snprintf(out, n, "%04x:%04x:%08x", p->vendorID, p->deviceID, p->driverVersion); } static const char *stage_name(int st) { return st == GFX_STAGE_VS ? "vertex" : st == GFX_STAGE_FS ? "fragment" : st == GFX_STAGE_CS ? "compute" : "other"; } static int hex_digit(int c) { return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; } // A journal a crashed run left into the skip list, then the skip list's variants for this GPU and driver into SK. static void skip_init(const char *dir) { if (SK.loaded) return; SK.loaded = 1; char jp[PLAT_PATH_MAX * 2 + 48], sp[PLAT_PATH_MAX * 2 + 48], key[32]; snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir); snprintf(sp, sizeof sp, "%s/shader-prepare-skip.txt", dir); gpu_key(key, sizeof key); uint64_t jn = 0, sn = 0; uint8_t *j = plat_file_read(jp, &jn); char *s = (char *)plat_file_read(sp, &sn); int keep_journal = 0; if (j) { static const char head[] = "# Shader variants the graphics driver was building when the process died during preparation\n" "# (runtime/vk_pipeline.c skip_init): input id, stage, GPU vendor:device:driver version. Preparation skips\n" "# them on that GPU and driver. Delete this file to build them again.\n"; size_t cap = (s ? sn : sizeof head) + (jn / JR_SLOT + 1) * 128, len = 0; char *out = malloc(cap); int added = 0; if (out) { len = s ? sn : sizeof head - 1; memcpy(out, s ? s : head, len); if (len && out[len - 1] != '\n') out[len++] = '\n'; for (uint64_t o = 0; o + 33 <= jn; o += JR_SLOT) { int set = 0; for (int b = 0; b < 32; b++) set |= j[o + b]; if (!set) continue; char hex[65]; for (int b = 0; b < 32; b++) snprintf(hex + 2 * b, 3, "%02x", j[o + b]); len += (size_t)snprintf(out + len, cap - len, "%s %s %s\n", hex, stage_name(j[o + 32]), key); added++; } out[len] = 0; // parsed below as the list's text } if (added && plat_file_write_atomic(sp, out, len)) { rt_log("vk_pipeline: the last shader preparation died while the driver built %d variants; %s now lists them " "and preparation skips them on this GPU and driver (%s)", added, sp, key); free(s), s = out, sn = len, out = NULL; } else if (added) { rt_log("vk_pipeline: cannot write %s; %s stays for the next run", sp, jp); keep_journal = 1; } free(out); free(j); } if (!keep_journal) plat_file_delete(jp); for (char *line = s, *next; line && *line; line = next) { next = strchr(line, '\n'); if (next) *next++ = 0; char hex[65], st[16], k[32]; uint8_t iid[32]; int ok = line[0] != '#' && sscanf(line, "%64s %15s %31s", hex, st, k) == 3 && strlen(hex) == 64 && !strcmp(k, key); for (int b = 0; ok && b < 32; b++) { int hi = hex_digit(hex[2 * b]), lo = hex_digit(hex[2 * b + 1]); if (hi < 0 || lo < 0) ok = 0; else iid[b] = (uint8_t)(hi << 4 | lo); } uint8_t (*grown)[32] = ok ? realloc(SK.iid, (SK.n + 1) * sizeof *SK.iid) : NULL; if (grown) memcpy(grown[SK.n++], iid, 32), SK.iid = grown; } free(s); if (SK.n) rt_log("vk_pipeline: %zu shader variants skipped ahead of play on this GPU and driver (%s)", SK.n, sp); } static int skip_listed(const uint8_t iid[32]) { for (size_t i = 0; i < SK.n; i++) if (!memcmp(SK.iid[i], iid, 32)) return 1; return 0; } static void skip_count(void) { plat_mutex_lock(&LB.stat); LB.skipped++; plat_mutex_unlock(&LB.stat); } // Journal slot `slot`: the variant e its worker hands to the driver next, or zeros (e NULL) when the build returned. static void journal_mark(int slot, const GfxWarmEnt *e) { if (!SK.journal || slot < 0 || slot >= JR_SLOTS) return; uint8_t rec[JR_SLOT] = {0}; if (e) memcpy(rec, e->iid, 32), rec[32] = e->stage; plat_file_pwrite(SK.journal, rec, sizeof rec, (uint64_t)slot * JR_SLOT); } // Compute shaders whose pipeline a thread is compiling now (P.lock): a cold driver takes about 2 s for one of Ocarina's // large compute shaders. The replay used to compile under P.lock, which held every other pipeline lookup that long, and // compiled a second copy of a pipeline the boot prebuild (lb_compute) was already compiling. #define CS_FLIGHT 32 static const GfxShader *cs_flight[CS_FLIGHT]; static int cs_flight_has(const GfxShader *cs) { for (int i = 0; i < CS_FLIGHT; i++) if (cs_flight[i] == cs) return 1; return 0; } static volatile uint8_t cs_flight_want[CS_FLIGHT]; // a dispatch waits for the slot's pipeline (vk_pipeline_compute) static PlatThread *cs_flight_th[CS_FLIGHT]; // the boot prebuild thread compiling it below normal priority // th: the thread a waiting dispatch raises to normal priority, NULL when it compiles at normal priority already. static int cs_flight_add(const GfxShader *cs, PlatThread *th) { // 0 when the table is full (the caller compiles anyway) for (int i = 0; i < CS_FLIGHT; i++) if (!cs_flight[i]) return cs_flight[i] = cs, cs_flight_want[i] = 0, cs_flight_th[i] = th, 1; return 0; } // A dispatch waits for cs's build: a builder that would only have filled the pipeline cache keeps the pipeline. Returns // the thread to raise, the first time. static PlatThread *cs_flight_wanted(const GfxShader *cs) { PlatThread *th = NULL; for (int i = 0; i < CS_FLIGHT; i++) if (cs_flight[i] == cs && !cs_flight_want[i]) cs_flight_want[i] = 1, th = cs_flight_th[i]; return th; } // Without P.lock, for the thread that holds cs's slot: whether a dispatch came to wait. static int cs_flight_is_wanted(const GfxShader *cs) { for (int i = 0; i < CS_FLIGHT; i++) if (__atomic_load_n(&cs_flight[i], __ATOMIC_RELAXED) == cs) return cs_flight_want[i]; return 0; } static int cs_flight_remove(const GfxShader *cs) { // 1 when a dispatch waited for it int want = 0; for (int i = 0; i < CS_FLIGHT; i++) if (cs_flight[i] == cs) cs_flight[i] = NULL, want |= cs_flight_want[i]; return want; } // ---- compute pipelines the boot keeps (compute-used.bin next to pipeline-keys.bin; NX2NAVI_COMPUTE_KEEP) // The boot prebuild (lb_compute) used to build the pipeline of every compute shader of the warm list (the ones play // recorded and the prepared ones, shader-prepared.bin) and keep all of them for the process: 4,066 on an installed // copy (42,344 variants in its list, 42,142 of them prepared), of which a boot into the first village dispatched 95. // What that cost was twenty of them. A copy's list has about twenty compute shaders of 0.9 to 2.0 million SPIR-V // words: the game's tensor-core upscaler programs, which the game never dispatches natively (vk_exec_draw.c // native_tensor) and which no run of the usage record below ever dispatched. A pipeline of one holds about 100 MiB // of device memory from the moment it exists, used or not, and the driver keeps that memory when the pipeline is // destroyed. On an RTX 5090 (NVIDIA 621.01) a boot pass that read all 4,066 pipelines from pipeline-cache.bin took the // device memory from 386 to 2,398 MiB although it destroyed 3,949 of them at once; the same pass before the twenty // were in the cache left it at 386, and building only the twenty took it to 1,370 (--probe-prewarm, 2026-11-10; the // game showed 608 to 2,743 MiB over the same stage). The 4,046 others are cheap: all of them kept cost 2 MiB of device // memory and 0.7 GB of the process's commit. So a copy with prepared shaders and a filled pipeline cache held 2 GiB // for nothing from the title screen on, on cards that need that memory for textures. // What the boot does now: // - A compute shader over LB.big_words is built ahead only when a run has dispatched it (lb_compute, its keep // argument). One that no run dispatched is not created at all: not from the pipeline caches and not by the // background builder (bg_main). If the game does dispatch one for the first time, that dispatch builds it, which // holds the frame for the compile once (0.65 to 1.9 s measured for such a shader), and the record has it built ahead // from the next boot on. // - Of the others the boot keeps the pipelines of the NX2NAVI_COMPUTE_KEEP (default 512) dispatched most recently, and // builds every other one into the pipeline cache and destroys it at once (lb_compute keep 0), from a copy of its // shader that is released again (lb_worker): the cache then holds it, so its first dispatch creates it without // compiling. On that machine 94 first dispatches in play took 0.24 ms on average and 1.6 ms at most, 14 of them in // the busiest 33 ms; reading the shader from the disk at that moment adds 0.05 to 0.15 ms (--probe-prewarm). // The record: one entry per compute shader a run dispatched (its input id) with the number of the last run that did; // each run that dispatches anything takes the next number. Kept at boot are the keep_n entries with the highest // numbers, so a short run does not push out what a long one used, and nothing is forgotten until newer shaders take // its place. Without a record (the first run that has one) nothing is kept and the run pays the first dispatches once. // NX2NAVI_COMPUTE_KEEP=all builds and keeps every compute shader of the warm list, the large ones too, as before; 0 // keeps none. #define CU_VERSION 1 typedef struct { char magic[8]; uint32_t version, rec_size, run, pad; } CuHeader; typedef struct { uint8_t iid[32]; uint32_t run, pad; } CuRec; static struct { PlatMutex lock; int opened, dirty; char path[PLAT_PATH_MAX * 2 + 32]; CuRec *rec; uint8_t *keep; // per record read at boot: among the keep_n dispatched most recently uint32_t n, cap, n_file; // n_file: the records read at boot (keep has one byte for each) uint32_t *slot; // open addressing by the id's first bytes: record index + 1, 0 empty uint32_t slot_cap; uint32_t run; // this run's number int quiet; // --probe-prewarm: its dispatches are not the game's, the file stays as it is int32_t keep_n; // NX2NAVI_COMPUTE_KEEP; -1: all uint32_t kept; // records flagged in keep } CU = {.lock = PLAT_MUTEX_INIT}; // What the first dispatches of compute shaders cost this run (under P.lock): each creates its pipeline on the replay // thread. burst: the creations that fell into one window of CF_WINDOW_MS, the worst such window kept, which is what a // player would feel as one long frame when a new area starts. #define CF_WINDOW_MS 33.0 static struct { uint64_t n, hit, pre_used; // hit: the pipeline cache held it (cc_hit 1); pre_used: kept from the boot and dispatched double ms, max_ms, hit_ms; double win_t0, win_ms, worst_ms; uint32_t win_n, worst_n; } CF; static uint32_t *cu_slot(const uint8_t iid[32]) { // the id's slot, or the empty one it would take. Caller holds CU.lock. uint64_t h; memcpy(&h, iid, 8); for (uint32_t j = (uint32_t)(h * 0x9e3779b97f4a7c15ull >> 32) & (CU.slot_cap - 1);; j = (j + 1) & (CU.slot_cap - 1)) if (!CU.slot[j] || !memcmp(CU.rec[CU.slot[j] - 1].iid, iid, 32)) return &CU.slot[j]; } // Room for one more record and a table at most half full. 0 when out of memory. Caller holds CU.lock. static int cu_room(void) { if (CU.n == CU.cap) { uint32_t cap = CU.cap ? CU.cap * 2 : 1024; CuRec *t = realloc(CU.rec, cap * sizeof *t); if (!t) return 0; CU.rec = t, CU.cap = cap; } if ((CU.n + 1) * 2 > CU.slot_cap) { uint32_t cap = CU.slot_cap ? CU.slot_cap * 2 : 4096; uint32_t *t = calloc(cap, sizeof *t), *old = CU.slot; if (!t) return 0; CU.slot = t, CU.slot_cap = cap; for (uint32_t i = 0; i < CU.n; i++) *cu_slot(CU.rec[i].iid) = i + 1; free(old); } return 1; } static int cu_run_cmp(const void *a, const void *b) { // {run, index} pairs, the highest run first const uint32_t *x = a, *y = b; return x[0] != y[0] ? (x[0] < y[0] ? 1 : -1) : (x[1] > y[1]) - (x[1] < y[1]); } // The number of records in a compute-used.bin image (f, size) with its header in *h, or -1 when it is not one. static long cu_parse(const uint8_t *f, uint64_t size, CuHeader *h) { if (!f || size < sizeof *h) return -1; memcpy(h, f, sizeof *h); if (memcmp(h->magic, "NX2NCUSE", 8) || h->version != CU_VERSION || h->rec_size != sizeof(CuRec)) return -1; return (long)((size - sizeof *h) / sizeof(CuRec)); } // Reads the record and marks what the boot keeps. Caller holds P.lock (ensure_init), after keys_open. static void cu_open(void) { if (!A.on || !K.path[0]) return; const char *e = cfg_value("NX2NAVI_COMPUTE_KEEP"); plat_mutex_lock(&CU.lock); int count = cfg_int("NX2NAVI_COMPUTE_KEEP"); // a value that is neither "all" nor a number reads as the default CU.keep_n = !strcmp(e, "all") ? -1 : count < 0 ? 0 : count; snprintf(CU.path, sizeof CU.path, "%s", K.path); char *cut = plat_last_sep(CU.path); snprintf(cut ? cut + 1 : CU.path, sizeof CU.path - (size_t)(cut ? cut + 1 - CU.path : 0), "compute-used.bin"); uint64_t size = 0; uint8_t *f = plat_file_read(CU.path, &size); CuHeader h; long n = cu_parse(f, size, &h); if (n >= 0) { for (long i = 0; i < n && cu_room(); i++) { CuRec r; memcpy(&r, f + sizeof h + (size_t)i * sizeof r, sizeof r); uint32_t *s = cu_slot(r.iid); if (*s) continue; // an id twice: the first entry stands CU.rec[CU.n] = r, *s = ++CU.n; } CU.run = h.run; } else if (f) { rt_log("vk_pipeline: %s has another format or version; it is rewritten", CU.path); } free(f); CU.n_file = CU.n, CU.run++; uint32_t (*order)[2] = CU.n ? malloc(CU.n * sizeof *order) : NULL; CU.keep = CU.n ? calloc(CU.n, 1) : NULL; if (order && CU.keep) { for (uint32_t i = 0; i < CU.n; i++) order[i][0] = CU.rec[i].run, order[i][1] = i; qsort(order, CU.n, sizeof *order, cu_run_cmp); for (uint32_t i = 0; i < CU.n && (CU.keep_n < 0 || i < (uint32_t)CU.keep_n); i++) CU.keep[order[i][1]] = 1, CU.kept++; } free(order); CU.opened = 1; rt_log("vk_pipeline: %s: %u compute shaders dispatched in earlier runs; the boot keeps the pipelines of %s%u " "dispatched most recently (NX2NAVI_COMPUTE_KEEP %s)", CU.path, CU.n, CU.keep_n < 0 ? "every recorded compute shader, among them the " : "the ", CU.kept, e); plat_mutex_unlock(&CU.lock); } // 1 when the boot keeps the pipeline of the compute shader with this input id. static int cu_keep(const uint8_t iid[32]) { plat_mutex_lock(&CU.lock); int keep = CU.keep_n < 0 || !CU.opened; // without the record (no cache folder): as before, every one if (!keep && CU.slot_cap) { uint32_t s = *cu_slot(iid); keep = s && s <= CU.n_file && CU.keep && CU.keep[s - 1]; } plat_mutex_unlock(&CU.lock); return keep; } // The first dispatch of cs in this run: its entry takes this run's number. The runtime's own shaders (no input id) // are built at their first use every run and are not recorded. static void cu_note(const GfxShader *cs) { static const uint8_t none[32]; const uint8_t *iid = gfx_shader_input_id(cs); if (!CU.opened || !memcmp(iid, none, 32)) return; plat_mutex_lock(&CU.lock); if (cu_room()) { uint32_t *s = cu_slot(iid); if (!*s) { memset(&CU.rec[CU.n], 0, sizeof *CU.rec); memcpy(CU.rec[CU.n].iid, iid, 32); *s = ++CU.n; } if (CU.rec[*s - 1].run != CU.run) CU.rec[*s - 1].run = CU.run, CU.dirty = 1; } plat_mutex_unlock(&CU.lock); } // Writes the record to path. 1 when written. // One writer at a time, from the copy of the record to the end of the file write: the autosave thread and a save at // exit can both be here, and the older copy must not land on the newer file. static PlatMutex cu_io = PLAT_MUTEX_INIT; static int cu_write(const char *path) { plat_mutex_lock(&cu_io); plat_mutex_lock(&CU.lock); uint32_t n = CU.n; plat_mutex_unlock(&CU.lock); // allocated outside CU.lock, which a dispatch takes (cu_note); room for the entries that come meanwhile size_t cap = sizeof(CuHeader) + ((size_t)n + 256) * sizeof(CuRec), size = 0; uint8_t *out = malloc(cap); plat_mutex_lock(&CU.lock); CuHeader h = {{'N', 'X', '2', 'N', 'C', 'U', 'S', 'E'}, CU_VERSION, sizeof(CuRec), CU.run, 0}; if (out && sizeof h + CU.n * sizeof(CuRec) <= cap) { size = sizeof h + CU.n * sizeof(CuRec); memcpy(out, &h, sizeof h); if (CU.n) memcpy(out + sizeof h, CU.rec, CU.n * sizeof(CuRec)); CU.dirty = 0; } plat_mutex_unlock(&CU.lock); int ok = size && plat_file_write_atomic(path, out, size); if (!ok) { // still to be written: the next flush tries again plat_mutex_lock(&CU.lock); CU.dirty = 1; plat_mutex_unlock(&CU.lock); } free(out); plat_mutex_unlock(&cu_io); return ok; } // Writes the record when a dispatch changed it (the autosave thread and the save at exit). static void cu_flush(void) { if (!CU.opened || CU.quiet || !*(volatile int *)&CU.dirty) return; if (!cu_write(CU.path)) LOG_ONCE("vk_pipeline: cannot write %s; the next saves try again", CU.path); } // A compute pipeline created at a dispatch took ms. Caller holds P.lock. static void cf_note(double ms, int hit) { double now = now_ms(); CF.n++, CF.ms += ms; if (ms > CF.max_ms) CF.max_ms = ms; if (hit == 1) CF.hit++, CF.hit_ms += ms; if (now - CF.win_t0 > CF_WINDOW_MS) CF.win_t0 = now, CF.win_n = 0, CF.win_ms = 0; CF.win_n++, CF.win_ms += ms; if (CF.win_ms > CF.worst_ms) CF.worst_ms = CF.win_ms, CF.worst_n = CF.win_n; } static void cf_log(void) { plat_mutex_lock_shared(&P.lock); typeof(CF) c = CF; // the line is written after the unlock: every dispatch needs P.lock, and this plat_mutex_unlock_shared(&P.lock); // runs on the autosave thread below normal priority if (c.n || c.pre_used) rt_log("vk_pipeline: compute pipelines created at their first dispatch: %llu (avg %.2f ms, longest %.2f ms; the " "pipeline cache held %llu of them, avg %.2f ms); the %.0f ms that held the most of it: %u pipelines, " "%.1f ms; of the compute pipelines kept from the boot, %llu were dispatched", (unsigned long long)c.n, c.n ? c.ms / c.n : 0.0, c.max_ms, (unsigned long long)c.hit, c.hit ? c.hit_ms / c.hit : 0.0, CF_WINDOW_MS, c.worst_n, c.worst_ms, (unsigned long long)c.pre_used); } // The background builder of large compute shaders (comment at bg_main). static struct { const GfxShader **cs; uint8_t (*iid)[32]; uint64_t *words; size_t n, cap; PlatThread *th; const GfxShader *volatile building; // the shader on the driver now, NULL between builds volatile int32_t raised; // a dispatch waited for it and raised the thread uint64_t built, failed, there; double ms; } BG; VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { if (info) *info = NULL; if (!cs || !ensure_init()) return VK_NULL_HANDLE; struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); plat_mutex_lock(&P.lock); Pipe *p = pipe_find(&k, sizeof k, h); double w0 = !p && cs_flight_has(cs) ? now_ms() : 0; int bg = 0; while (!p && cs_flight_has(cs)) { // another thread is compiling it: its result, not a second compile // a build that only fills the pipeline cache keeps its pipeline for this dispatch, and a boot prebuild thread // compiling it below normal priority goes to normal PlatThread *builder = cs_flight_wanted(cs); if (builder) plat_thread_set_priority(builder, PLAT_PRIO_NORMAL); if (BG.building == cs && BG.th && !plat_xchg32(&BG.raised, 1)) // the background builder: normal priority now plat_thread_set_priority(BG.th, PLAT_PRIO_NORMAL), bg = 1; plat_mutex_unlock(&P.lock); plat_sleep_ms(1); plat_mutex_lock(&P.lock); p = pipe_find(&k, sizeof k, h); } if (w0 && now_ms() - w0 >= 2.0) { static int logs; if (logs++ < 16) rt_log("vk_exec: a dispatch waited %.0f ms for its compute pipeline (%s)", now_ms() - w0, bg ? "built on the background thread" : "built by another thread"); } if (p) { P.stats.hits++; } else { const char *why = NULL; Bindings B = {0}; Layout *l = compute_layout(cs, &B, &why); VkPipeline pipe = VK_NULL_HANDLE; if (!why) { int flight = cs_flight_add(cs, NULL); plat_mutex_unlock(&P.lock); // the compile runs without the lock uint64_t p0 = perf_begin(); double c0 = now_ms(); pipe = compute_create(cs, l, 0); double cms = now_ms() - c0; int hit = cc_hit; perf_end(PERF_PIPE, p0); plat_mutex_lock(&P.lock); if (flight) cs_flight_remove(cs); if (!pipe) why = "pipeline creation failed"; else cf_note(cms, hit); } if ((p = pipe_find(&k, sizeof k, h))) { // another thread added it meanwhile (no flight slot was free) if (pipe) vkDestroyPipeline(P.device, pipe, NULL); } else { if (why) { P.stats.failures++; rt_log("vk_pipeline: compute pipeline not built: %s", why); } else { P.stats.compute++; LB.cs_draw++; } p = pipe_add(&k, sizeof k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_COMPUTE, 0); } } if (p && !p->used) { // the first dispatch of this run: the boot of the next runs keeps this shader's pipeline p->used = 1; CF.pre_used += p->pre; if (p->info.pipeline) cu_note(cs); } plat_mutex_unlock(&P.lock); if (!p) return VK_NULL_HANDLE; if (info) *info = &p->info; return p->info.pipeline; } static void lb_lock(void) { plat_thread_priority(PLAT_PRIO_NORMAL); plat_mutex_lock(&P.lock); } static void lb_unlock(void) { plat_mutex_unlock(&P.lock); if (LB.idle) plat_thread_priority(PLAT_PRIO_IDLE); } static void lb_count(int kind, int what, double ms) { // what: 0 built, 1 already there, 2 failed, 3 cache only plat_mutex_lock(&LB.stat); if (what == 3) LB.warmed[kind]++, LB.warm_ms[kind] += ms; else (what == 0 ? LB.built : what == 1 ? LB.have : LB.fail)[kind]++, LB.ms[kind] += ms; plat_mutex_unlock(&LB.stat); } // vs_union_missing for vs alone, without entering vs into the union table (it holds VS_OUTS_CAP shaders). static uint32_t vs_union_peek(const GfxShader *vs, GfxShaderIo fin[32]) { GfxShaderIo vout[32]; uint32_t need = 0, have = gfx_shader_outputs(vs, vout), missing = 0; memset(fin, 0, 32 * sizeof *fin); plat_mutex_lock_shared(&VO.lock); uint32_t j = (uint32_t)(((uintptr_t)vs >> 4) * 0x9e3779b1u) & (VS_OUTS_CAP - 1); while (VO.vs[j] && VO.vs[j] != vs) j = (j + 1) & (VS_OUTS_CAP - 1); if (VO.vs[j]) need = VO.mask[j], memcpy(fin, VO.io[j], 32 * sizeof *fin); plat_mutex_unlock_shared(&VO.lock); if (P.vs_outs_all) for (int i = 0; i < 32; i++) if (!(need >> i & 1)) fin[i] = (GfxShaderIo){GFX_TYPE_FLOAT, 4, 0}, need |= 1u << i; for (int i = 0; i < 32; i++) if (need >> i & 1 && !(have >> i & 1)) missing |= 1u << i; return missing; } // A compute shader over LB.big_words that no pipeline cache holds: its first dispatch builds it (vk_pipeline_compute). static void lb_defer(const uint8_t iid[32], uint64_t words) { plat_mutex_lock(&LB.stat); LB.deferred++; plat_mutex_unlock(&LB.stat); rt_log("vk_pipeline: stage library prebuild: compute shader %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) %s: " "left to its first dispatch (NX2NAVI_LIB_PREBUILD_BIG %llu)", iid[0], iid[1], iid[2], iid[3], iid[4], iid[5], iid[6], iid[7], (unsigned long long)words, P.cache_control ? "not in the pipeline caches" : "too large to build without a cache probe (no pipelineCreationCacheControl)", (unsigned long long)LB.big_words); } // ---- large compute shaders on a background thread (NX2NAVI_LIB_PREBUILD_BIG_BG, default 1; 0 leaves them to their // first dispatch as before) // The boot prebuild leaves a compute shader over LB.big_words that no pipeline cache holds (lb_compute). Its first // dispatch then compiled it on the replay thread: frames of 0.65-1.9 s in Kokiri Forest on an RTX 5090 after a // translator change. One thread builds them instead, once the boot prebuild is done, one at a time and the smallest // first: four compiled side by side grew the process by 16-19 GB on an RTX 4090 (comment at LB_BIG_WORDS). The shader // being built is in cs_flight, so a dispatch that needs it waits for this build (vk_pipeline_compute raises the // thread to normal priority then) instead of compiling a second copy. What is built is entered for the process, as // lb_compute's pipelines are, and lands in pipeline-cache.bin for the next boot. static int bg_on(void) { static int on = -1; if (on < 0) on = cfg_bool("NX2NAVI_LIB_PREBUILD_BIG_BG"); return on; } // A large compute shader no cache holds goes to the background builder (lb_worker threads; under LB.stat). static void bg_queue(const GfxShader *cs, const uint8_t iid[32], uint64_t words) { plat_mutex_lock(&LB.stat); if (BG.n == BG.cap) { size_t cap = BG.cap ? BG.cap * 2 : 16; const GfxShader **c = realloc(BG.cs, cap * sizeof *c); if (c) BG.cs = c; uint8_t (*i)[32] = realloc(BG.iid, cap * sizeof *i); if (i) BG.iid = i; uint64_t *w = realloc(BG.words, cap * sizeof *w); if (w) BG.words = w; if (c && i && w) BG.cap = cap; } int ok = BG.n < BG.cap; if (ok) BG.cs[BG.n] = cs, memcpy(BG.iid[BG.n], iid, 32), BG.words[BG.n] = words, BG.n++; plat_mutex_unlock(&LB.stat); if (!ok) { lb_defer(iid, words); return; } rt_log("vk_pipeline: stage library prebuild: compute shader %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) not in " "the pipeline caches: built on the background thread after the prebuild (NX2NAVI_LIB_PREBUILD_BIG_BG)", iid[0], iid[1], iid[2], iid[3], iid[4], iid[5], iid[6], iid[7], (unsigned long long)words); } // NX2NAVI_BIG_BUILD_FREE_MB (default 8192; 0 no check): the system commit headroom a large compute build waits for. // A build of one of these shaders took the process up by 1-3 GB on an RTX 5090 and four at once by 16-19 GB on an // RTX 4090; started on a machine near its commit limit it can end the game. While the headroom is short the shader // waits, and if the game dispatches it first it is built there, as without the background builder. static int bg_room(double *used, double *limit) { static double need = -1; if (need < 0) need = strtod(cfg_value("NX2NAVI_BIG_BUILD_FREE_MB"), NULL); double proc; plat_commit_info(&proc, used, limit); return need <= 0 || *limit < 0 || *limit - *used >= need; } // NX2NAVI_BIG_BUILD_MAX_MB (default 4096; 0 no cap): how far the builder may raise the process commit before it stops // and leaves the rest to their first dispatch. One of these shaders costs the driver 1-4 GB: on a 31 GB machine four of // them took the process from 15 to 29 GB and system commit to 60 of 65 GB, which is where a player feels the game // stutter. The builder exists to remove hitches, not to trade them for paging, so a machine that cannot hold them all // builds only what it can and pays one hitch per shader later. static double bg_growth_cap(void) { static double cap = -1; if (cap < 0) cap = strtod(cfg_value("NX2NAVI_BIG_BUILD_MAX_MB"), NULL); return cap; } // The builder compiles into a pipeline cache of its own (own_cache at pipe_create_in), not into P.cache. A create into // P.cache holds cache_rw shared from start to end, and the cache is only saved at a moment when nobody holds it: on an // RX 7900 XT (26.10.1) the largest of these shaders (4,874,502 SPIR-V words) was still compiling 590 s after the boot, // pipeline-cache.bin was not written once in that time, and the game was closed before the build ended, so the next // boot found neither that shader nor the pipelines built in play and started the same build again. bg_merge moves the // result into P.cache when the build is done, at a moment with no build into P.cache running: the lock is tried as // cache_save tries it, because a waiting exclusive request would hold back every new build. static void bg_merge(VkPipelineCache own, int built) { int locked = 0; while (built && !LB.stop && !(locked = plat_mutex_trylock(&cache_rw))) plat_sleep_ms(2); if (locked) { VkResult r = vkMergePipelineCaches(P.device, P.cache, 1, &own); plat_mutex_unlock(&cache_rw); if (r == VK_SUCCESS) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED); else rt_log("vk_pipeline: background compute build: its cache did not merge into %s (%d)", P.cache_path, (int)r); } vkDestroyPipelineCache(P.device, own, NULL); } static void bg_main(void *arg) { (void)arg; plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); // The queue can grow while this thread builds (bg_queue runs on the replay thread and reallocs these arrays), so // the sort and every read below happen under LB.stat; an entry is copied out before its compile so no pointer into // the arrays is held across it. plat_mutex_lock(&LB.stat); for (size_t i = 1; i < BG.n; i++) // smallest first: insertion sort of a handful for (size_t j = i; j > 0 && BG.words[j] < BG.words[j - 1]; j--) { const GfxShader *c = BG.cs[j]; BG.cs[j] = BG.cs[j - 1], BG.cs[j - 1] = c; uint64_t w = BG.words[j]; BG.words[j] = BG.words[j - 1], BG.words[j - 1] = w; uint8_t t[32]; memcpy(t, BG.iid[j], 32), memcpy(BG.iid[j], BG.iid[j - 1], 32), memcpy(BG.iid[j - 1], t, 32); } plat_mutex_unlock(&LB.stat); double t_all = now_ms(); double c_start, c_used0, c_limit0; plat_commit_info(&c_start, &c_used0, &c_limit0); for (size_t i = 0; !LB.stop; i++) { uint8_t idbuf[32]; uint64_t words = 0; const GfxShader *cs; plat_mutex_lock(&LB.stat); int have = i < BG.n; if (have) { cs = BG.cs[i]; memcpy(idbuf, BG.iid[i], 32); words = BG.words[i]; } plat_mutex_unlock(&LB.stat); if (!have) break; const uint8_t *iid = idbuf; struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; const char *why = NULL; double used, limit, c_now; plat_commit_info(&c_now, &used, &limit); double cap = bg_growth_cap(); if (cap > 0 && c_now - c_start >= cap) { rt_log("vk_pipeline: background compute build: stopping, the process grew %.0f MB in all (NX2NAVI_BIG_BUILD_MAX_MB " "%.0f); the shaders left build at their first dispatch", c_now - c_start, cap); break; } int waited = 0, no_room = 0; double w0 = now_ms(); while (!LB.stop && !bg_room(&used, &limit)) { // not under P.lock: a dispatch builds it meanwhile if (!waited++) rt_log("vk_pipeline: background compute build: waiting for commit headroom (system %.0f of %.0f MB, " "NX2NAVI_BIG_BUILD_FREE_MB %s)", used, limit, cfg_value("NX2NAVI_BIG_BUILD_FREE_MB")); if (now_ms() - w0 > 30000.0) { // waiting is only useful when the headroom will come back rt_log("vk_pipeline: background compute build: no commit headroom after %.0f s; the shaders left build at " "their first dispatch", (now_ms() - w0) / 1000.0); no_room = 1; break; } plat_sleep_ms(2000); } if (LB.stop || no_room) break; plat_mutex_lock(&P.lock); int there = cs_flight_has(cs) || pipe_find(&k, sizeof k, h); // a dispatch got to it first Layout *l = there ? NULL : compute_layout(cs, &B, &why); int flight = l ? cs_flight_add(cs, NULL) : 0; // vk_pipeline_compute raises this thread through BG.raised if (l) BG.building = cs; plat_mutex_unlock(&P.lock); if (!l) { if (there) BG.there++; else BG.failed++, rt_log("vk_pipeline: background compute build: %s", why ? why : "no layout"); continue; } VkPipelineCache own = VK_NULL_HANDLE; VkPipelineCacheCreateInfo oci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO}; if (P.cache && vkCreatePipelineCache(P.device, &oci, NULL, &own) != VK_SUCCESS) own = VK_NULL_HANDLE; double t0 = now_ms(); own_cache = own; // without one the build goes into P.cache as before VkPipeline pipe = compute_create(cs, l, 0); own_cache = VK_NULL_HANDLE; double ms = now_ms() - t0; // The pipeline stays: only a shader a run dispatched is queued here (lb_compute), and it is entered before // the merge below, so a dispatch that comes meanwhile finds it. plat_mutex_lock(&P.lock); if (flight) cs_flight_remove(cs); BG.building = NULL; Pipe *e = NULL; if (pipe && !pipe_find(&k, sizeof k, h) && (e = pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0))) { e->pre = 1; P.stats.compute++; } else if (pipe) { // a dispatch built its own meanwhile (no flight slot was free), or out of memory vkDestroyPipeline(P.device, pipe, NULL); } plat_mutex_unlock(&P.lock); if (own) bg_merge(own, pipe != VK_NULL_HANDLE); // after the entry: a dispatch waiting for it does not wait for this if (plat_xchg32(&BG.raised, 0)) plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); if (pipe) BG.built++, BG.ms += ms; else BG.failed++; double c_proc, c_used, c_limit; plat_commit_info(&c_proc, &c_used, &c_limit); rt_log("vk_pipeline: background compute build: %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) %s in %.0f ms; " "commit: process %.0f MB, system %.0f of %.0f MB", iid[0], iid[1], iid[2], iid[3], iid[4], iid[5], iid[6], iid[7], (unsigned long long)words, pipe ? "built" : "failed", ms, c_proc, c_used, c_limit); } rt_log("vk_pipeline: background compute builds %s: %llu built (%.0f ms in all), %llu failed, %llu dispatched before " "their turn; %.0f ms after the prebuild", LB.stop ? "stopped" : "done", (unsigned long long)BG.built, BG.ms, (unsigned long long)BG.failed, (unsigned long long)BG.there, now_ms() - t_all); } // The last boot prebuild worker starts the builder when it queued anything. static void bg_start(void) { if (!BG.n || LB.stop || BG.th) return; if (!(BG.th = plat_thread_new(bg_main, NULL, "big compute"))) rt_log("vk_pipeline: cannot start the background compute builder; %zu large compute shaders build at their first " "dispatch", BG.n); } // keep 0: the pipeline is built for the pipeline cache only and destroyed at once, unless a dispatch came to wait for // it meanwhile; a shader over LB.big_words is then not built at all (comment at compute-used.bin). cs may be a copy // the caller releases afterwards when keep is 0: nothing here holds on to it then. static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep) { struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; const char *why = NULL; uint64_t words = 0; gfx_shader_spirv(cs, &words); int big = LB.big_words && words > LB.big_words; // only from the caches (LB_BIG_WORDS) if (big && !keep) { // no run dispatched it: a pipeline of it would hold about 100 MiB of device memory unused plat_mutex_lock(&LB.stat); LB.big_unused++; plat_mutex_unlock(&LB.stat); return; } if (big && !P.cache_control) { if (bg_on()) bg_queue(cs, iid, words); else lb_defer(iid, words); return; } lb_lock(); int busy = cs_flight_has(cs); // the replay compiles it now (vk_pipeline_compute) Layout *l = busy || pipe_find(&k, sizeof k, h) ? NULL : compute_layout(cs, &B, &why); PlatThread *me = LB.idle && lb_me >= 0 ? LB.th[lb_me] : NULL; int have = !l && !why, flight = l ? cs_flight_add(cs, me) : 0; lb_unlock(); if (!l) { lb_count(2, have ? 1 : 2, 0); return; } double t0 = now_ms(); // The compile runs below normal priority: after an update or a driver update every one of these pipelines is // compiled again (about 140 ms each on an RTX 5090's driver, minutes of two threads for a long play history), and // at normal priority those threads took cores from the game's own on a CPU with few of them. While the shader is in // flight a dispatch of it waits for this build (vk_pipeline_compute) and raises the thread to normal: a compile left // at idle priority under load held that frame for seconds. The dispatch may have come between the unlock and the // call below, which would undo its raise, hence the look at the slot after it. Without a slot or a handle nobody // could raise the thread, so it compiles at normal priority. if (LB.idle && flight && me) { plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); if (cs_flight_is_wanted(cs)) plat_thread_priority(PLAT_PRIO_NORMAL); } else if (LB.idle) { plat_thread_priority(PLAT_PRIO_NORMAL); } VkPipeline pipe = compute_create(cs, l, big); double ms = now_ms() - t0; int compiled = pipe && cc_hit != 1, kept = 0; lb_lock(); if (flight && cs_flight_remove(cs)) keep = 1; // a dispatch waits for it Pipe *e = NULL; if (pipe && pipe_find(&k, sizeof k, h)) { vkDestroyPipeline(P.device, pipe, NULL); have = 1; } else if (pipe && keep && (e = pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0))) { e->pre = 1, kept = 1; P.stats.compute++; } else if (pipe) { vkDestroyPipeline(P.device, pipe, NULL); // P.stats.compute does not count it, so the autosave would not see what its compile added to P.cache if (compiled) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED); } lb_unlock(); if (!pipe && big && bg_on()) bg_queue(cs, iid, words); else if (!pipe && big) lb_defer(iid, words); else lb_count(2, !pipe ? 2 : have ? 1 : kept ? 0 : 3, ms); } // The pre-rasterization (vertex) or fragment-shader library of one shader with the LB state. // keep 0: build into the driver's pipeline cache only (sh is a private copy; the library is destroyed). static void lb_library(const GfxShader *sh, int vertex, int keep) { int kind = vertex ? LIB_PR : LIB_FS, stat = vertex ? 0 : 1; VkPipeState s; vk_pipe_state_defaults(&s); GKey k; memset(&k, 0, sizeof k); k.clamp = LB.clamp, k.poly_mode = LB.poly, k.minus_w = LB.minus_w, k.samples = LB.samples; PrKey pk; FsKey fk; memset(&pk, 0, sizeof pk), memset(&fk, 0, sizeof fk); const void *key; uint32_t size; if (vertex) { GfxShaderIo fin[32]; s.vs = sh; pk.kind = LIB_PR, pk.vs = sh, pk.clamp = k.clamp, pk.poly_mode = k.poly_mode, pk.minus_w = k.minus_w; pk.missing = vs_union_peek(sh, fin); for (int i = 0; i < 32; i++) if (pk.missing >> i & 1) pk.fin[i] = fin[i]; key = &pk, size = sizeof pk; } else { s.fs = sh; fk.kind = LIB_FS, fk.fs = sh, fk.samples = k.samples; key = &fk, size = sizeof fk; } uint64_t h = key_hash(key, size); Bindings B = {0}, F = {0}; VkPipelineLayout lay = VK_NULL_HANDLE; lb_lock(); int have = keep && lib_find(h, key, size) != NULL; if (!have && add_stage(vertex ? &B : &F, sh, 0)) { Layout *l = layout_pair(&B, &F); if (l) lay = half_layout(l, vertex ? 0 : 1); } lb_unlock(); if (have || !lay) { lb_count(stat, have ? 1 : 2, 0); return; } double t0 = now_ms(); VkPipeline pipe = lib_create(kind, &s, &k, lay, key); double ms = now_ms() - t0; if (!keep) { if (pipe) vkDestroyPipeline(P.device, pipe, NULL); lb_count(stat, pipe ? 3 : 2, ms); return; } lb_lock(); have = lib_find(h, key, size) != NULL; VkPipeline got = lib_put(h, key, size, pipe); if (!have && pipe) { Lib *b = lib_find(h, key, size); if (b) b->pre = 1; } lb_unlock(); lb_count(stat, !pipe ? 2 : have || got != pipe ? 1 : 0, ms); } // A vertex or fragment variant with the graphics pipeline library off: no stage library to build ahead. static void lb_skip_lib(void) { plat_mutex_lock(&LB.stat); LB.nolib++; plat_mutex_unlock(&LB.stat); } static void lb_report(const char *how) { float mem_used, mem_total, mem_proc; plat_mem_info(&mem_used, &mem_total, &mem_proc); plat_mutex_lock_shared(&LB.stat); rt_log("vk_pipeline: stage library prebuild %s: %llu of %zu variants in %.0f ms; kept: vertex %llu built (avg %.1f ms), " "%llu there; fragment %llu built (avg %.1f ms), %llu there; compute %llu built (avg %.1f ms), %llu there; driver " "cache only: vertex %llu (avg %.1f ms), fragment %llu (avg %.1f ms), compute %llu (avg %.1f ms); %llu failed, %llu not in the shader cache, " "%llu vertex/fragment skipped (graphics pipeline library off), %llu skipped (the driver crashed on them before, " "shader-prepare-skip.txt), %llu large compute shaders left to their first dispatch (NX2NAVI_LIB_PREBUILD_BIG), %zu " "queued for the background builder (NX2NAVI_LIB_PREBUILD_BIG_BG), %llu large compute shaders not built because " "no run has dispatched them; process working set %.0f MB", how, (unsigned long long)LB.done, LB.n, now_ms() - LB.t0, (unsigned long long)LB.built[0], LB.built[0] ? LB.ms[0] / LB.built[0] : 0.0, (unsigned long long)LB.have[0], (unsigned long long)LB.built[1], LB.built[1] ? LB.ms[1] / LB.built[1] : 0.0, (unsigned long long)LB.have[1], (unsigned long long)LB.built[2], LB.built[2] ? LB.ms[2] / LB.built[2] : 0.0, (unsigned long long)LB.have[2], (unsigned long long)LB.warmed[0], LB.warmed[0] ? LB.warm_ms[0] / LB.warmed[0] : 0.0, (unsigned long long)LB.warmed[1], LB.warmed[1] ? LB.warm_ms[1] / LB.warmed[1] : 0.0, (unsigned long long)LB.warmed[2], LB.warmed[2] ? LB.warm_ms[2] / LB.warmed[2] : 0.0, (unsigned long long)(LB.fail[0] + LB.fail[1] + LB.fail[2]), (unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (unsigned long long)LB.skipped, (unsigned long long)LB.deferred, BG.n, (unsigned long long)LB.big_unused, (double)mem_proc); plat_mutex_unlock_shared(&LB.stat); } // Boot: compute pipelines (the first LB.n_keep entered for the process, the others only built into the pipeline // cache) and kept libraries. static void lb_worker(void *arg) { lb_me = (int)(intptr_t)arg; // its handle is LB.th[lb_me] plat_thread_priority(PLAT_PRIO_IDLE); for (;;) { size_t i = (size_t)plat_inc64(&LB.next) - 1; if (i >= LB.n || LB.stop) break; if (skip_listed(LB.e[i].iid)) { skip_count(); plat_mutex_lock(&LB.stat); LB.done++; plat_mutex_unlock(&LB.stat); continue; } // A compute shader whose pipeline does not stay is read as a copy of this thread's own and released after its // build: kept in the shader cache's memory, the 3,900 of an installed copy's list were 240 MB of the process's // commit for shaders most runs never dispatch. A dispatch that does come reads its shader then (0.05 to // 0.15 ms from the files this pass has just read). The shaders of kept pipelines and libraries stay loaded: // their pipeline and library entries point at them. int keep = i < LB.n_keep, owned = 0; plat_thread_priority(PLAT_PRIO_NORMAL); // gfx_shader's cache lock, disk reads const GfxShader *sh = keep || LB.e[i].stage != GFX_STAGE_CS ? gfx_shader_load_input(LB.e[i].iid, 0) : gfx_shader_load_private(LB.e[i].iid, &owned); plat_thread_priority(PLAT_PRIO_IDLE); if (!sh) { plat_mutex_lock(&LB.stat); LB.noshader++; plat_mutex_unlock(&LB.stat); } else if (gfx_shader_stage(sh) == GFX_STAGE_CS) { lb_compute(sh, LB.e[i].iid, keep); } else if (!owned && (gfx_shader_stage(sh) == GFX_STAGE_VS || gfx_shader_stage(sh) == GFX_STAGE_FS)) { if (P.gpl) lb_library(sh, gfx_shader_stage(sh) == GFX_STAGE_VS, 1); // a kept library points at sh: never a copy else lb_skip_lib(); } if (owned) gfx_shader_release_private(sh); plat_mutex_lock(&LB.stat); LB.done++; plat_mutex_unlock(&LB.stat); } if (!plat_dec32(&LB.running)) { lb_report(LB.stop ? "stopped" : "done"); fb_log("after the stage library prebuild"); vk_mem_stage_log("the compute pipeline prebuild"); bg_start(); } return; } // The library state from the key file's records (prewarm); defaults without one. static void lb_state(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { // state index: clamp | poly (0 fill, 1 line, 2 point -> VkPolygonMode) << 1 | minus_w << 3 (pr); samples log2 (fs) uint32_t best = 0, bf = 0; for (uint32_t i = 1; i < 16; i++) best = pr_hist[i] > pr_hist[best] ? i : best; for (uint32_t i = 1; i < 4; i++) bf = fs_hist[i] > fs_hist[bf] ? i : bf; if (!pr_hist[best]) best = 8; // no key file: NX2navi clips -w..w (all 2706 records of a recorded key file) LB.clamp = (uint8_t)(best & 1), LB.poly = (uint8_t)(best >> 1 & 3), LB.minus_w = (uint8_t)(best >> 3 & 1); LB.samples = (uint8_t)(1u << bf); } // ---- the driver's cache of stage libraries, warmed again during play // // The preparation (lb_warm_prepared) builds every prepared variant's stage library once, so the driver's disk cache // holds it and the library a draw needs later takes a few milliseconds instead of a compile: 3 ms against 100 ms for // a vertex and 340 ms for a fragment library on an RX 7900 XT (driver 26.10.1). The driver's cache does not outlive // everything, though: a driver update empties it, the AMD driver keeps a limited amount and drops old entries, and a // change to what a library is built with (the dynamic states it lists, as when dynamic vertex input was turned off for // this driver) gives every library another entry. After any of them each pipeline a draw first needs compiles cold: // the draw is skipped for the frames that takes, or waits for it. Measured on that card with the cache cold, a walk // through a village not seen before skipped 11,180 draws while their pipelines built and 94 draws waited 5.5 s in // all, one of them 1.9 s. // // So every start checks: a sample of the prepared fragment libraries is built (lw_probe), and when their median time // says the cache is cold, the whole prepared list is built again into the driver's cache on a few idle-priority // threads while the game runs (lw_worker), as the preparation does but without stopping play. shader-warm.txt beside // the caches says where that stands: "warming" while it runs, so a start after a run that ended meanwhile goes on, // and "done" with the driver and a count of the times the cache was cold again right after a finished warm. At two in // a row the driver does not keep stage libraries at all and warming stops being tried (NX2NAVI_LIB_WARM=0 turns it off; // =force warms at every start). #define LW_SAMPLE 12 #define LW_COLD_MS 15.0 #define LW_MAX_THREADS 6 static struct { GfxWarmEnt *e; size_t n; volatile int64_t next, done; volatile int32_t running, stop, started; PlatThread *th[LW_MAX_THREADS + 1]; int threads, strikes; uint32_t vendor; // the GPU's vendor id, for what a cold cache's log line adds (vk_warm_hint.h) double t0; char stamp[PLAT_PATH_MAX * 2 + 32], id[64]; } LW; static void lw_stamp(const char *state, int strikes) { char line[128]; int n = snprintf(line, sizeof line, "%s %s %d\n", state, LW.id, strikes); if (!plat_file_write_atomic(LW.stamp, line, (size_t)n)) rt_log("vk_pipeline: cannot write %s", LW.stamp); } // One prepared vertex or fragment variant's library into the driver's cache; its time in ms, negative when the // variant is on the skip list, not in the shader cache or of another stage. static double lw_build(const GfxWarmEnt *e) { if (e->stage != GFX_STAGE_VS && e->stage != GFX_STAGE_FS) return -1; if (skip_listed(e->iid)) return -1; int owned = 0; const GfxShader *sh = gfx_shader_load_private(e->iid, &owned); if (!sh) return -1; double t0 = now_ms(); int st = gfx_shader_stage(sh); if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) lb_library(sh, st == GFX_STAGE_VS, 0); double ms = now_ms() - t0; if (owned) gfx_shader_release_private(sh); return ms; } static void lw_worker(void *arg) { (void)arg; plat_thread_priority(PLAT_PRIO_IDLE); for (;;) { size_t i = (size_t)plat_inc64(&LW.next) - 1; if (i >= LW.n || LW.stop) break; lw_build(&LW.e[i]); int64_t d = plat_inc64(&LW.done); if (d % 8000 == 0) rt_log("vk_pipeline: library warm: %lld of %zu prepared variants in %.0f s", (long long)d, LW.n, (now_ms() - LW.t0) / 1000); } if (!plat_dec32(&LW.running) && !LW.stop) { lw_stamp("done", LW.strikes); rt_log("vk_pipeline: library warm: done, %zu prepared variants in %.0f s on %d idle threads; new pipelines " "link from the driver's cache from here on", LW.n, (now_ms() - LW.t0) / 1000, LW.threads); } } // The median time of LW_SAMPLE prepared fragment libraries spread over the list; negative without that many. static double lw_probe(void) { double ms[LW_SAMPLE]; int got = 0; size_t step = LW.n / LW_SAMPLE ? LW.n / LW_SAMPLE : 1, at = (size_t)(plat_now_ns() / 1000) % step; for (size_t i = at; i < LW.n && got < LW_SAMPLE && !LW.stop; i += step) for (size_t j = i; j < i + step && j < LW.n; j++) { // the next fragment variant from here if (LW.e[j].stage != GFX_STAGE_FS) continue; double t = lw_build(&LW.e[j]); if (t >= 0) ms[got++] = t; break; } if (got < LW_SAMPLE / 2) return -1; for (int i = 1; i < got; i++) for (int j = i; j > 0 && ms[j] < ms[j - 1]; j--) { double t = ms[j]; ms[j] = ms[j - 1], ms[j - 1] = t; } return ms[got / 2]; } static void lw_main(void *arg) { (void)arg; plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); char was[16] = "", id[64] = ""; int strikes = 0; uint64_t size = 0; char *f = (char *)plat_file_read(LW.stamp, &size); if (f && size < 200) { char line[200]; memcpy(line, f, size), line[size] = 0; if (sscanf(line, "%15s %63s %d", was, id, &strikes) < 2) was[0] = 0; } free(f); int same = !strcmp(id, LW.id), resume = same && !strcmp(was, "warming"); const char *mode = cfg_value("NX2NAVI_LIB_WARM"); double median = resume ? 0 : lw_probe(); if (LW.stop) return; int cold = median > LW_COLD_MS || !strcmp(mode, "force"); if (!resume && !cold) { if (same && strikes) lw_stamp("done", 0); // the driver kept them this time rt_log("vk_pipeline: library warm: the driver's cache has the prepared libraries (a sample's median %.1f ms)", median); return; } if (!resume && same && strikes >= 2 && strcmp(mode, "force")) { rt_log("vk_pipeline: library warm: the prepared libraries are cold again (a sample's median %.0f ms) after two " "finished warms on this driver: it does not keep them, so pipelines compile when a draw first needs " "them (NX2NAVI_LIB_WARM=force warms anyway).%s", median, lw_cold_hint(LW.vendor, LW_COLD_GAVE_UP, LW_ON_WINDOWS)); return; } LW.strikes = resume ? strikes : same && !strcmp(was, "done") ? strikes + 1 : 0; lw_stamp("warming", LW.strikes); int cpus = (int)plat_cpu_usable(), t = cpus / 4; t = t < 2 ? 2 : t > LW_MAX_THREADS ? LW_MAX_THREADS : t; if (resume) rt_log("vk_pipeline: library warm: going on with the warm an earlier run began: %zu prepared variants on %d " "idle threads", LW.n, t); else // cold right after a finished warm on this driver says more than a first cold (vk_warm_hint.h) rt_log("vk_pipeline: library warm: the driver's cache does not have the prepared libraries (a sample's median " "%.0f ms; a driver update or a change of what they are built with): building %zu prepared variants " "into it on %d idle threads while the game runs. Until it is done, a pipeline a draw first needs may " "compile, which skips that draw for some frames or makes it wait.%s", median, LW.n, t, lw_cold_hint(LW.vendor, median <= LW_COLD_MS ? LW_WARM_FORCED // warm, but lib_warm = force : same && !strcmp(was, "done") ? LW_COLD_AFTER_WARM : LW_COLD_FIRST, LW_ON_WINDOWS)); LW.t0 = now_ms(); LW.running = t; int started = 0; for (int i = 0; i < t; i++) if ((LW.th[1 + started] = plat_thread_new(lw_worker, NULL, "lib warm"))) started++; LW.threads = started; if (plat_add32(&LW.running, started - t) == 0 && !started) rt_log("vk_pipeline: library warm: no thread started"); } // Boot, once the library state is known (lb_state): the prepared list is read and the check starts on its own thread. static void lw_start(void) { const char *mode = cfg_value("NX2NAVI_LIB_WARM"); if (!P.gpl || !strcmp(mode, "0") || plat_xchg32(&LW.started, 1)) return; char dir[PLAT_PATH_MAX * 2], path[PLAT_PATH_MAX * 2 + 32]; if (!cache_dir(dir, sizeof dir)) return; snprintf(path, sizeof path, "%s/shader-prepared.bin", dir); snprintf(LW.stamp, sizeof LW.stamp, "%s/shader-warm.txt", dir); uint64_t size = 0; uint8_t *f = plat_file_read(path, &size); size_t n = f && size >= 16 ? (size_t)(size - 16) / 36 : 0; LW.e = n ? calloc(n, sizeof *LW.e) : NULL; for (size_t i = 0; LW.e && i < n; i++) memcpy(LW.e[i].iid, f + 16 + 36 * i + 4, 32), LW.e[i].stage = f[16 + 36 * i], LW.e[i].prio = f[16 + 36 * i + 1]; free(f); if (!LW.e) return; // nothing prepared (a build without the preparation step) LW.n = n; VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties(vk_ctx()->phys, &pp); snprintf(LW.id, sizeof LW.id, "%04x:%04x:%08x", pp.vendorID, pp.deviceID, pp.driverVersion); LW.vendor = pp.vendorID; LW.th[0] = plat_thread_new(lw_main, NULL, "lib warm check"); } static void lw_stop(void) { plat_xchg32(&LW.stop, 1); for (int i = 0; i <= LW_MAX_THREADS; i++) if (LW.th[i]) plat_thread_join(LW.th[i]), LW.th[i] = NULL; } // Boot: compute shaders of the warm list (recorded and prepared), those whose pipeline stays first, then the first // KEEP recorded graphics variants. static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) { const char *e = cfg_value("NX2NAVI_LIB_PREBUILD"), *ke = cfg_get("NX2NAVI_LIB_PREBUILD_KEEP"); const char *te = cfg_get("NX2NAVI_LIB_PREBUILD_THREADS"); // no fixed default: 2 when unset (1 under 4 CPUs) const char *be = cfg_get("NX2NAVI_LIB_PREBUILD_BIG"); // SPIR-V words; LB_BIG_WORDS when unset, 0 no limit if (!A.on || !cfg_bool("NX2NAVI_LIB_PREBUILD")) return; lb_state(pr_hist, fs_hist); // the libraries' state, also for lw_start lw_start(); LB.big_words = be && *be ? strtoull(be, NULL, 10) : LB_BIG_WORDS; char dir[PLAT_PATH_MAX * 2]; if (cache_dir(dir, sizeof dir)) skip_init(dir); GfxWarmEnt *all = NULL; size_t total = gfx_shader_warm_list(&all), n = 0, n_cs = 0; LB.keep_n = ke && *ke ? atoi(ke) : 0; LB.e = malloc((total + 1) * sizeof *LB.e); // the compute shaders whose pipeline stays come first: they are the ones the game is about to dispatch uint8_t *kf = total ? malloc(total) : NULL; for (size_t i = 0; LB.e && kf && i < total; i++) if (all[i].stage == GFX_STAGE_CS && (kf[i] = (uint8_t)cu_keep(all[i].iid))) LB.e[n++] = all[i]; LB.n_keep = n; for (size_t i = 0; LB.e && i < total; i++) if (all[i].stage == GFX_STAGE_CS && !(kf && kf[i])) LB.e[n++] = all[i]; free(kf); n_cs = n; for (size_t i = 0; LB.e && i < total && (int32_t)(n - n_cs) < LB.keep_n; i++) if (all[i].stage != GFX_STAGE_CS && all[i].prio == 0) LB.e[n++] = all[i]; free(all); if (atoi(e) > 1 && (size_t)atoi(e) < n) n = (size_t)atoi(e); // a count above 1 caps the list; 1 is plain on if (LB.n_keep > n) LB.n_keep = n; if (!n) { free(LB.e), LB.e = NULL; return; } LB.n = n; LB.idle = 1; int cpus = (int)plat_cpu_usable(); // Two threads by default, not half the CPUs. On a 32-thread CPU with NVIDIA's driver, 16 builders kept the driver's // lock busy: 77% of the process's running time was a spin loop in nvoglv64.dll (about 8 cores), the prebuild had not // finished after 34,000 frames, and lib_prebuild = 0 took it to 5% and the game to its frame limit (RTX 4080 SUPER, // driver 617.14, 2026-11-08). With the driver's disk cache warm (--prepare-shaders, --warm-shaders) a build takes // well under a millisecond, so two threads still finish the list quickly. int t = te && *te ? atoi(te) : (cpus < 4 ? 1 : 2); // idle priority; a cold driver takes ~2 s per large compute shader t = t < 1 ? 1 : t > LB_MAX_THREADS ? LB_MAX_THREADS : t; if ((size_t)t > n) t = (int)n; LB.t0 = now_ms(); LB.running = t; int started = 0; for (int i = 0; i < t; i++) if ((LB.th[started] = plat_thread_new(lb_worker, (void *)(intptr_t)started, "lib prebuild"))) started++; plat_add32(&LB.running, started - t); LB.threads = started; char big[128]; if (!LB.big_words) snprintf(big, sizeof big, "compute shaders of every size"); else snprintf(big, sizeof big, "compute shaders over %llu SPIR-V words %s", (unsigned long long)LB.big_words, P.cache_control ? "only from the pipeline caches" : "left to their first dispatch"); rt_log("vk_pipeline: stage library prebuild: %zu compute pipelines, of which %zu stay (the shaders dispatched most " "recently, NX2NAVI_COMPUTE_KEEP) and the others are built into the pipeline cache and destroyed, to be created " "at their first dispatch; %zu recorded variants' libraries (NX2NAVI_LIB_PREBUILD_KEEP %d); on %d idle-priority " "threads; library state clamp %u, polygon mode %u, depth -w..w %u, samples %u; %s (NX2NAVI_LIB_PREBUILD_BIG)", n_cs < n ? n_cs : n, LB.n_keep, n > n_cs ? n - n_cs : 0, LB.keep_n, started, LB.clamp, LB.poly, LB.minus_w, LB.samples, big); } static void lb_stop(void) { lw_stop(); plat_xchg32(&LB.stop, 1); for (int i = 0; i < LB.threads; i++) if (LB.th[i]) plat_thread_join(LB.th[i]), LB.th[i] = NULL; LB.threads = 0; if (BG.th) plat_thread_join(BG.th), BG.th = NULL; // after its current build, if one runs } // Preparation: one prepared variant at a time into the driver's disk cache (private shader copy, nothing kept). arg is // the worker's journal slot. static void lb_warm_worker(void *arg) { int slot = (int)(intptr_t)arg; for (;;) { size_t i = (size_t)plat_inc64(&LB.next) - 1; if (i >= LB.n) return; int owned = 0, skip = skip_listed(LB.e[i].iid); const GfxShader *sh = skip ? NULL : gfx_shader_load_private(LB.e[i].iid, &owned); int st = sh ? gfx_shader_stage(sh) : -1; if (sh) journal_mark(slot, &LB.e[i]); // from here to its clearing below, a crash names this variant if (skip) { skip_count(); } else if (!sh) { plat_mutex_lock(&LB.stat); LB.noshader++; plat_mutex_unlock(&LB.stat); } else if (st == GFX_STAGE_CS) { Bindings B = {0}; const char *why = NULL; lb_lock(); Layout *l = compute_layout(sh, &B, &why); lb_unlock(); double t0 = now_ms(); VkPipeline pipe = l ? compute_create(sh, l, 0) : VK_NULL_HANDLE; if (pipe) vkDestroyPipeline(P.device, pipe, NULL); lb_count(2, pipe ? 3 : 2, now_ms() - t0); } else if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) { if (P.gpl) lb_library(sh, st == GFX_STAGE_VS, 0); else lb_skip_lib(); } if (sh) journal_mark(slot, NULL); if (owned) gfx_shader_release_private(sh); plat_mutex_lock(&LB.stat); uint64_t d = ++LB.done; plat_mutex_unlock(&LB.stat); if (d % 4000 == 0) lb_report("preparation progress"); } } // Preparation: every variant of /shader-prepared.bin on every core at normal priority. static void lb_warm_prepared(const char *dir) { char path[PLAT_PATH_MAX * 2 + 32]; snprintf(path, sizeof path, "%s/shader-prepared.bin", dir); uint64_t size = 0; uint8_t *f = plat_file_read(path, &size); size_t n = f && size >= 16 ? (size_t)(size - 16) / 36 : 0; LB.e = n ? calloc(n, sizeof *LB.e) : NULL; for (size_t i = 0; LB.e && i < n; i++) memcpy(LB.e[i].iid, f + 16 + 36 * i + 4, 32), LB.e[i].stage = f[16 + 36 * i], LB.e[i].prio = f[16 + 36 * i + 1]; free(f); if (!LB.e) return; skip_init(dir); char jp[PLAT_PATH_MAX * 2 + 48]; snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir); SK.journal = plat_fs_create(jp, NULL); // skip_init removed a leftover one if (!SK.journal) rt_log("vk_pipeline: cannot create %s: a driver crash in this run will not be recorded", jp); LB.n = n, LB.idle = 0, LB.t0 = now_ms(); static const uint32_t no_pr[16], no_fs[4]; lb_state(no_pr, no_fs); int cpus = (int)plat_cpu_usable(), k = 0; PlatThread *t[JR_SLOTS]; for (int i = 0; i < cpus && i < JR_SLOTS; i++) if ((t[k] = plat_thread_new(lb_warm_worker, (void *)(intptr_t)k, "lib warm"))) k++; if (!k) lb_warm_worker((void *)(intptr_t)0); // every 2 s a count on stdout for tools/make_game.py (step_shaders), as gfx_shader.c prepare_files prints for the // translation: a driver building for a long time still shows the step moving for (int i = 0; i < k; i++) while (!plat_thread_join_ns(t[i], 2000000000u)) { plat_mutex_lock(&LB.stat); uint64_t d = LB.done; plat_mutex_unlock(&LB.stat); printf("prepare-shaders: progress build %llu %zu\n", (unsigned long long)d, n); fflush(stdout); } printf("prepare-shaders: progress build %zu %zu\n", n, n); fflush(stdout); if (SK.journal) plat_file_close(SK.journal), SK.journal = NULL, plat_file_delete(jp); // every build returned lb_report("preparation done"); // with gpl off only compute variants are built: say so, rather than count the skipped ones as built unsigned long long skipped = LB.skipped; size_t tried = n - (size_t)skipped; if (LB.nolib) printf("prepare-shaders: %zu variants on %d threads in %.0f ms, %llu vertex/fragment variants skipped (graphics " "pipeline library off, they build at their first draw)\n", tried, k, now_ms() - LB.t0, (unsigned long long)LB.nolib); else printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", tried, k, now_ms() - LB.t0); if (skipped) // installer/steps.py Prepare reads this line printf("prepare-shaders: %llu variants skipped: the graphics driver crashed while building them in an earlier " "run (shader-prepare-skip.txt); they build at their first draw\n", skipped); free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0, LB.skipped = 0; } // nx2navi.exe --prepare-shaders (tools/prepare_shaders.py): the device sets the translator's host answers (they are in every // cache key), the seed next to nx2navi.exe is translated on every core, then every prepared variant's stage library or // compute pipeline is built once into the driver's disk cache (lb_warm_prepared). No game runs. int vk_pipeline_prepare_shaders(void) { if (!ensure_init()) { fprintf(stderr, "prepare-shaders: no Vulkan device\n"); return 1; } char dir[PLAT_PATH_MAX * 2]; if (!cache_dir(dir, sizeof dir)) return 1; int cpus = (int)plat_cpu_usable(); const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2; int rc = gfx_shader_prepare_seed(dir, cpus, procs < 1 ? 1 : procs); if (!rc) lb_warm_prepared(dir); return rc; } // nx2navi.exe --warm-shaders: every variant of shader-prepared.bin built into the driver's disk cache again from the shader // cache, without translating the seed. The AMD cache is keyed to the exe path (a relink keeps it: 29,587 variants in // 1.8 s) and holds a limited amount: preparing another build dir evicted this one's libraries, and a library a draw // needed then took 40-220 ms in play instead of 0.2 ms. Seconds when the libraries are still there. int vk_pipeline_warm_shaders(void) { if (!ensure_init()) { fprintf(stderr, "warm-shaders: no Vulkan device\n"); return 1; } char dir[PLAT_PATH_MAX * 2]; if (!cache_dir(dir, sizeof dir)) return 1; char prp[PLAT_PATH_MAX * 2 + 32]; snprintf(prp, sizeof prp, "%s/shader-prepared.bin", dir); if (!plat_file_exists(prp)) { fprintf(stderr, "warm-shaders: no %s (tools/prepare_shaders.py writes it)\n", prp); return 1; } lb_warm_prepared(dir); return 0; } // nx2navi.exe --probe-prewarm: the boot's shader and pipeline stages without the game, from the caches next to // nx2navi.exe (pipeline-cache.bin, pipeline-keys.bin, shader-answers.bin, the shader cache): the pipeline cache load, // the shader prewarm, the recorded pipelines and the compute pipeline prebuild, each followed by vk_mem_stage_log's // line, so the log tells what every stage leaves in device memory and in the process's commit. Then up to // PROBE_DISPATCHES compute shaders whose pipeline the boot did not keep are asked for as a dispatch would // (gfx_shader_load_input for the shader, then vk_pipeline_compute), which gives what a first dispatch costs on this // machine and how many of them the pipeline cache held. Fails when one of those pipelines is not created, when the // prebuild kept a pipeline it should not have, when a large compute shader went to the background builder although no // run dispatched it, or when the record of dispatched shaders, written to a file of the probe's own and read back, // does not hold the probe's dispatches. compute-used.bin itself stays as it is; the pipeline cache is saved as after // a boot. #define PROBE_DISPATCHES 256 int vk_pipeline_probe_prewarm(void) { vk_pipeline_async_enable(); CU.quiet = 1; vk_pipeline_prewarm(); // brings the device up itself: it has to own the key file before ensure_init runs if (!P.ready) { printf("probe-prewarm: SKIP (the boot prewarm brought no Vulkan device up: none on this machine, NX2NAVI_GPU=null " "or NX2NAVI_PIPE_SYNC=1)\n"); return 0; } while (__atomic_load_n(&LB.running, __ATOMIC_ACQUIRE) > 0) plat_sleep_ms(50); if (!LB.n) { printf("probe-prewarm: SKIP (no recorded compute shaders next to the executable: %s)\n", P.cache_path); return 0; } VkPipeStats st; vk_pipeline_stats(&st); plat_mutex_lock_shared(&LB.stat); uint64_t kept = LB.built[2], cache_only = LB.warmed[2], failed = LB.fail[2], there = LB.have[2]; uint64_t big_unused = LB.big_unused; size_t queued = BG.n; plat_mutex_unlock_shared(&LB.stat); printf("probe-prewarm: %zu compute shaders, %zu to keep: %llu kept, %llu built into the pipeline cache only, %llu " "there already, %llu failed, %llu large ones not built (no run dispatched them), %zu large ones for the " "background builder; %llu graphics pipelines\n", LB.n, LB.n_keep, (unsigned long long)kept, (unsigned long long)cache_only, (unsigned long long)there, (unsigned long long)failed, (unsigned long long)big_unused, queued, (unsigned long long)st.graphics); int fails = kept > LB.n_keep; // a dispatch never waits in this mode, so nothing else may have been kept if (fails) printf("probe-prewarm: FAIL: %llu pipelines kept, %zu expected at most\n", (unsigned long long)kept, LB.n_keep); if (queued > LB.n_keep) // only a shader a run dispatched may be built ahead when it is large fails++, printf("probe-prewarm: FAIL: %zu large compute shaders queued for the background builder, %zu kept " "shaders at most could be among them\n", queued, LB.n_keep); uint32_t asked = 0, got = 0; double t_all = 0, t_max = 0, t_load = 0; for (size_t i = LB.n_keep; i < LB.n && asked < PROBE_DISPATCHES; i++) { if (LB.e[i].stage != GFX_STAGE_CS) continue; double l0 = now_ms(); const GfxShader *sh = gfx_shader_load_input(LB.e[i].iid, 0); // from the disk: the boot released its copy double load = now_ms() - l0; uint64_t words = 0; if (!sh || gfx_shader_stage(sh) != GFX_STAGE_CS) continue; gfx_shader_spirv(sh, &words); if (LB.big_words && words > LB.big_words) continue; // a first dispatch of one compiles it: not what is timed here const VkPipeInfo *info = NULL; double t0 = now_ms(); VkPipeline p = vk_pipeline_compute(sh, &info); double ms = now_ms() - t0; asked++, got += p != VK_NULL_HANDLE; t_all += ms, t_load += load; if (ms > t_max) t_max = ms; } vk_mem_stage_log("the probe's first dispatches"); plat_mutex_lock_shared(&P.lock); uint64_t hit = CF.hit; plat_mutex_unlock_shared(&P.lock); printf("probe-prewarm: %u first dispatches of pipelines the boot did not keep: %u created, avg %.2f ms, longest %.2f " "ms, the pipeline cache held %llu of them; reading their shaders took avg %.2f ms\n", asked, got, asked ? t_all / asked : 0.0, t_max, (unsigned long long)hit, asked ? t_load / asked : 0.0); if (got != asked) fails++, printf("probe-prewarm: FAIL: %u pipelines were not created\n", asked - got); if (CU.opened) { char rp[sizeof CU.path + 8]; snprintf(rp, sizeof rp, "%s.probe", CU.path); uint64_t rsize = 0; CuHeader rh = {0}; uint32_t mine = 0; int wrote = cu_write(rp); uint8_t *rf = wrote ? plat_file_read(rp, &rsize) : NULL; long rn = cu_parse(rf, rsize, &rh); for (long i = 0; i < rn; i++) { CuRec r; memcpy(&r, rf + sizeof rh + (size_t)i * sizeof r, sizeof r); mine += r.run == CU.run; } free(rf); plat_file_delete(rp); printf("probe-prewarm: the record of dispatched compute shaders: %ld entries, %u of this run (run %u)\n", rn, mine, CU.run); if (!wrote || rn < 0 || rh.run != CU.run || mine != got) fails++, printf("probe-prewarm: FAIL: the record does not hold the %u shaders dispatched here\n", got); } lb_stop(); vk_pipeline_save(); printf("probe-prewarm: %s\n", fails ? "FAILED" : "passed"); return fails ? 1 : 0; } // ---- dynamic state void vk_pipeline_set_dynamic(VkCommandBuffer cb, const VkPipeState *s) { VkDynShadow sh; uint64_t skipped = 0; sh.have = 0; // nothing known: every call is recorded vk_pipeline_set_dynamic_delta(cb, s, &sh, 0, &skipped); } // Every graphics pipeline the replay binds lists the same dynamic states (P.dyn; mesh pipelines leave out topology, // primitive restart and vertex input), so a value recorded into a command buffer holds across pipeline binds until it // is set again. Each call is recorded only when its value differs from the one *sh says the command buffer has: a new // pipeline usually changes the shaders and a field or two, not the twenty states this used to record with every bind. int vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s, VkDynShadow *sh, int mesh, uint64_t *skipped) { if (!ensure_init()) return 0; const poly_st *pl = &s->poly; const ds_st *d = &s->ds; int core = (sh->have & VK_DYN_CORE) != 0, vtx = !mesh && (sh->have & VK_DYN_VTX), blend = (sh->have & VK_DYN_BLEND) != 0; int n = 0, left = 0; #define DYN(known, field, val, call) \ do { \ if ((known) && sh->field == (val)) left++; \ else sh->field = (val), call, n++; \ } while (0) // The front face maps unchanged: NX2NAVI sets the upper-left window origin, and the y-flipped viewport (y_flip) puts // NDC +y at row 0 as NVN does, so the framebuffer winding Vulkan tests is the one NVN tests // (a lower-left origin would need it mirrored). int front_cw = pl->front == NVN_FRONT_FACE_CW; VkCullModeFlags cull = (VkCullModeFlags)(pl->cull & 3); // NVN_FACE_* has VkCullModeFlagBits' values VkFrontFace front = front_cw ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE; DYN(core, cull, cull, vkCmdSetCullMode(cb, cull)); DYN(core, front, front, vkCmdSetFrontFace(cb, front)); if (!mesh) { VkPrimitiveTopology topo = vk_pipeline_topology(s->prim); DYN(vtx, topo, topo, vkCmdSetPrimitiveTopology(cb, topo)); } DYN(core, dt, (VkBool32)d->dt, vkCmdSetDepthTestEnable(cb, d->dt)); DYN(core, dw, (VkBool32)d->dw, vkCmdSetDepthWriteEnable(cb, d->dw)); DYN(core, dop, compare_op(d->df), vkCmdSetDepthCompareOp(cb, compare_op(d->df))); DYN(core, st, (VkBool32)d->st, vkCmdSetStencilTestEnable(cb, d->st)); for (int f = 0; f < 2; f++) { VkStencilOp o[3] = {stencil_op(f ? d->fail_b : d->fail_f), stencil_op(f ? d->zpass_b : d->zpass_f), stencil_op(f ? d->zfail_b : d->zfail_f)}; VkCompareOp c = compare_op(f ? d->sf_b : d->sf_f); if (core && sh->scmp[f] == c && !memcmp(sh->sop[f], o, sizeof o)) { left++; continue; } vkCmdSetStencilOp(cb, f ? VK_STENCIL_FACE_BACK_BIT : VK_STENCIL_FACE_FRONT_BIT, o[0], o[1], o[2], c); sh->scmp[f] = c, memcpy(sh->sop[f], o, sizeof o), n++; } // NVN_POLYGON_OFFSET_ENABLE_* bit of the polygon mode in use: point 1, line 2, fill 4 unsigned mode = pl->mode <= NVN_POLYGON_MODE_FILL ? pl->mode : NVN_POLYGON_MODE_FILL; VkBool32 bias_en = (pl->offset_en >> mode & 1) != 0, discard = s->raster_discard != 0, restart = s->prim_restart != 0; DYN(core, bias_en, bias_en, vkCmdSetDepthBiasEnable(cb, bias_en)); DYN(core, discard, discard, vkCmdSetRasterizerDiscardEnable(cb, discard)); if (!mesh) DYN(vtx, restart, restart, vkCmdSetPrimitiveRestartEnable(cb, restart)); if (!mesh && P.dyn_vi && s->vs) { VtxInput v; vertex_input(s, &v); if (vtx && sh->vi.n_attr == v.n_attr && sh->vi.n_bind == v.n_bind && !memcmp(sh->vi.a, v.a, sizeof v.a[0] * v.n_attr) && !memcmp(sh->vi.b, v.b, sizeof v.b[0] * v.n_bind)) { left++; } else { rec_vertex_input(P.set_vi, cb, v.n_bind, v.b, v.n_attr, v.a); sh->vi = v, n++; } } else if (!mesh) { sh->vi.n_attr = UINT32_MAX; // not recorded: unknown to the next draw } if (P.dyn_clamp) { VkBool32 clamp = s->depth_clamp != 0; DYN(core, clamp, clamp, rec_bool(P.set_clamp, cb, clamp)); } if (P.dyn_poly) { VkPolygonMode pm = polygon_mode(pl->mode); DYN(core, poly, pm, rec_polygon_mode(P.set_poly, cb, pm)); } if (P.dyn_a2c) { // the same signature as the depth clamp call VkBool32 a2c = s->ms.a2c != 0; DYN(core, a2c, a2c, rec_bool((PFN_vkCmdSetDepthClampEnableEXT)P.set_a2c, cb, a2c)); } #undef DYN sh->have |= VK_DYN_CORE; if (mesh) sh->have &= ~(uint32_t)VK_DYN_VTX; else sh->have |= VK_DYN_VTX; int nt = target_count(s); if (nt) { VkBool32 en[8]; VkColorBlendEquationEXT eq[8]; VkColorComponentFlags wm[8]; for (int i = 0; i < nt; i++) { const blend_st *b = &s->blend[i]; en[i] = (s->color.blend_en >> i & 1) && blendable(target_format(s->color_fmt[i], NULL, NULL)); eq[i] = (VkColorBlendEquationEXT){blend_factor(b->src_rgb), blend_factor(b->dst_rgb), blend_op(b->eq_rgb), blend_factor(b->src_a), blend_factor(b->dst_a), blend_op(b->eq_a)}; wm[i] = target_wmask(s, i); } // The calls write targets [0, nt); targets past nt keep what the command buffer had. int known = blend && (uint32_t)nt <= sh->nt; if (P.dyn_blend_en) { if (known && !memcmp(sh->en, en, sizeof en[0] * nt)) left++; else rec_blend_enable(P.set_blend_en, cb, 0, (uint32_t)nt, en), n++; } if (P.dyn_blend_eq) { if (known && !memcmp(sh->eq, eq, sizeof eq[0] * nt)) left++; else rec_blend_equation(P.set_blend_eq, cb, 0, (uint32_t)nt, eq), n++; } if (P.dyn_wmask) { if (known && !memcmp(sh->wm, wm, sizeof wm[0] * nt)) left++; else rec_write_mask(P.set_wmask, cb, 0, (uint32_t)nt, wm), n++; } memcpy(sh->en, en, sizeof en[0] * nt), memcpy(sh->eq, eq, sizeof eq[0] * nt), memcpy(sh->wm, wm, sizeof wm[0] * nt); sh->nt = blend && sh->nt > (uint32_t)nt ? sh->nt : (uint32_t)nt; sh->have |= VK_DYN_BLEND; } *skipped += (uint64_t)left; return n; } // ---- statistics, persistence, teardown void vk_pipeline_stats(VkPipeStats *s) { plat_mutex_lock_shared(&P.lock); *s = P.stats; plat_mutex_unlock_shared(&P.lock); } static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache // Copies P.cache's data into pipeline-cache.bin, with cache_rw held exclusively for the copy only (not the file write). // The lock is only ever tried, never waited for: a waiting exclusive request would hold back every new build // (a shared request queues behind it), so a pending save could stall the replay. wait_ms: how long to keep trying // while builds run (0: one try). Returns 0 when no build-free moment came, 1 otherwise (saved, or nothing to save). static int cache_save(uint32_t wait_ms) { if (!P.ready || !P.cache || !P.cache_path[0]) return 1; // Not under P.lock: holding P.lock through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped // every pipeline lookup of the replay, a freeze at each 20 s autosave. plat_mutex_lock(&save_lock); if (!P.cache) { // vk_pipeline_shutdown destroyed it plat_mutex_unlock(&save_lock); return 1; } uint64_t until = plat_now_ms() + wait_ms; while (!plat_mutex_trylock(&cache_rw)) { if (plat_now_ms() >= until) { plat_mutex_unlock(&save_lock); return 0; } plat_sleep_ms(2); } size_t size = 0; void *data = NULL; int got = vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) && vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS; plat_mutex_unlock(&cache_rw); if (got) { if (!plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path); else if (P.feedback) { char when[96]; snprintf(when, sizeof when, "at a save of %.1f MB (%+.1f MB since the boot)", size / 1048576.0, ((double)size - (double)P.cache_loaded) / 1048576.0); fb_log(when); } } free(data); plat_mutex_unlock(&save_lock); return 1; } // Builds still running after this long at a save that waits (exit, the shader preparation) leave the file as it was. #define SAVE_WAIT_MS 5000 static void save_with(uint32_t wait_ms) { if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log(); if (P.ready && P.gpl) { rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; " "%llu of pipelines with a feedback loop; avg %.2f ms, " "max %.2f ms, %llu failed), %llu draws " "lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued " "low after their fast link), %llu drawing-thread links over 5 ms; %llu waiting draws built their libraries " "(avg %.2f ms, max %.2f ms)", (unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, (unsigned long long)P.g.flinks, (unsigned long long)P.g.fb_links, P.g.links ? P.g.link_ms / P.g.links : 0.0, P.g.link_max, (unsigned long long)P.g.link_fail, (unsigned long long)P.g.defer, (unsigned long long)P.g.swaps, (unsigned long long)P.g.requeued, (unsigned long long)P.g.slow, (unsigned long long)P.g.dlinks, P.g.dlinks ? P.g.dlink_ms / P.g.dlinks : 0.0, P.g.dlink_max); uint64_t rn = P.g.ready_n[0] + P.g.ready_n[1] + P.g.ready_n[2] + P.g.ready_n[3] + P.g.ready_n[4] + P.g.ready_n[5]; rt_log("vk_pipeline: gpl: pipelines a draw missed were usable after under 4 ms %llu, 4-8 ms %llu, 8-17 ms %llu, " "17-50 ms %llu, 50-150 ms %llu, longer %llu (avg %.1f ms, longest %.0f ms); %llu of them had no fast link", (unsigned long long)P.g.ready_n[0], (unsigned long long)P.g.ready_n[1], (unsigned long long)P.g.ready_n[2], (unsigned long long)P.g.ready_n[3], (unsigned long long)P.g.ready_n[4], (unsigned long long)P.g.ready_n[5], rn ? P.g.ready_ms / (double)rn : 0.0, P.g.ready_max, (unsigned long long)P.g.ready_nofast); if (P.g.hints) rt_log("vk_pipeline: gpl: looking ahead named %llu pipeline states not known yet and queued %llu stage libraries " "for them", (unsigned long long)P.g.hints, (unsigned long long)P.g.hint_libs); if (U.waits || U.queued) rt_log("vk_pipeline: gpl: library jobs for waiting draws: %llu queued, %llu built by a worker, %llu by the " "thread that needed them (no worker free), %llu waits ended at their limit; %llu fast links on the %d " "library workers", (unsigned long long)U.queued, (unsigned long long)U.by_worker, (unsigned long long)U.stolen, (unsigned long long)U.timeouts, (unsigned long long)U.links, U.lib_workers); static const char *const kn[4] = {"vertex input", "pre-rasterization", "fragment shader", "fragment output"}; for (int i = 0; i < 4; i++) rt_log("vk_pipeline: gpl: %s libraries %llu (avg %.2f ms, max %.2f ms; %llu of 25 ms or more, %.0f ms in all)", kn[i], (unsigned long long)P.g.lib_n[i], P.g.lib_n[i] ? P.g.lib_ms[i] / P.g.lib_n[i] : 0.0, P.g.lib_max[i], (unsigned long long)P.g.lib_slow[i], P.g.lib_slow_ms[i]); if (P.g.pr_added) rt_log("vk_pipeline: gpl: %llu of the pre-rasterization libraries had outputs added for their fragment shaders " "(avg %.1f ms): another module than the one the preparation built, so the driver's cache has none of them", (unsigned long long)P.g.pr_added, P.g.pr_added_ms / (double)P.g.pr_added); if (LB.n) rt_log("vk_pipeline: gpl: prebuilt libraries a draw used: pre-rasterization %llu, fragment shader %llu; " "compute pipelines built on the replay thread %llu", (unsigned long long)P.g.pre_used[LIB_PR], (unsigned long long)P.g.pre_used[LIB_FS], (unsigned long long)LB.cs_draw); } gfx_shader_record_flush(1); if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below if (!cache_save(wait_ms) && wait_ms) rt_log("vk_pipeline: %s not saved: pipeline builds kept running for %u ms", P.cache_path, (unsigned)wait_ms); } // The save at exit and after the shader preparation: it waits up to SAVE_WAIT_MS for a moment with no build running. void vk_pipeline_save(void) { save_with(SAVE_WAIT_MS); } // The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it // runs: at most every 20 s, and only after new pipelines were built. New pipeline keys are appended every 2 s. static void autosave_once(void) { static uint64_t last_n, last_ms, last_keys; if (!P.ready) return; uint64_t now = plat_now_ms(); if (now - last_keys >= 2000) { last_keys = now; if (*(volatile uint32_t *)&K.pend_n) keys_flush(); pb_flush(); cu_flush(); } gfx_shader_record_flush(0); if (now - last_ms < 20000) return; VkPipeStats s; vk_pipeline_stats(&s); uint64_t n = s.graphics + s.compute + cache_merges; // a merge comes after its pipeline was counted (bg_main) if (n == last_n) return; // Only at a moment with no build running (cache_save): while one runs the save waits for the next tick, 500 ms // on, before it logs or flushes anything, so a busy stretch does not repeat the statistics lines every tick. if (P.cache && P.cache_path[0]) { if (!plat_mutex_trylock(&cache_rw)) return; plat_mutex_unlock(&cache_rw); } last_n = n, last_ms = now; save_with(0); } // On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to // the disk each) ran on the present thread, and the render thread waited for it to take the next frame. static void autosave_main(void *arg) { (void)arg; plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); for (;;) { plat_sleep_ms(500); autosave_once(); } } void vk_pipeline_autosave(void) { static int32_t started; if (!P.ready || plat_xchg32(&started, 1)) return; if (!plat_thread_spawn(autosave_main, NULL, "pipe autosave", 0)) rt_log("vk_pipeline: cannot start the autosave thread; the cache is saved at exit only"); } void vk_pipeline_shutdown(void) { if (!P.ready) return; lb_stop(); workers_drain(); // jobs point at layouts and Pipe entries freed below vk_pipeline_save(); plat_mutex_lock(&P.lock); vkDeviceWaitIdle(P.device); for (int i = 0; i < PIPE_BUCKETS; i++) { for (Pipe *p = pipes[i], *n; p; p = n) { n = p->next; if (p->info.pipeline) vkDestroyPipeline(P.device, p->info.pipeline, NULL); if (p->fast && p->fast != p->info.pipeline) vkDestroyPipeline(P.device, p->fast, NULL); free(p->bindings), free(p); } pipes[i] = NULL; } libs_destroy(); for (int i = 0; i < LAYOUT_BUCKETS; i++) { for (Layout *l = layouts[i], *n; l; l = n) { n = l->next; for (int k = 0; k < 2; k++) if (l->tpl[k]) P.tpl_destroy(P.device, l->tpl[k], NULL); vkDestroyPipelineLayout(P.device, l->layout, NULL); for (int h = 0; h < 2; h++) if (l->half[h]) vkDestroyPipelineLayout(P.device, l->half[h], NULL); vkDestroyDescriptorSetLayout(P.device, l->set, NULL); if (l->set1) vkDestroyDescriptorSetLayout(P.device, l->set1, NULL); free(l); } layouts[i] = NULL; } plat_mutex_lock(&save_lock); // not while the autosave thread copies it if (P.cache) vkDestroyPipelineCache(P.device, P.cache, NULL); P.cache = VK_NULL_HANDLE; plat_mutex_unlock(&save_lock); memset(&P.stats, 0, sizeof P.stats); plat_xchg32(&P.ready, 0); plat_mutex_unlock(&P.lock); } // ---- self-test: census shaders (probe_census_dir, tools/shader_census.py) through gfx_shader_get into pipelines. // Graphics pairs are a fragment shader and a vertex shader of the same source file, spread over the census; compute // shaders are spread the same way. Every pipeline is built twice (the second call must hit the cache) and the // dynamic state of the first graphics pipeline is recorded into a throwaway command buffer. typedef struct { const char *key, *c1, *src; size_t c1_len, src_len; int stage; uint32_t local[3]; } CensusEntry; static const char *st_field(const char *from, const char *to, const char *name, size_t *len) { char pat[32]; snprintf(pat, sizeof pat, "\"%s\": \"", name); const char *p = strstr(from, pat); if (!p || p >= to) return NULL; p += strlen(pat); const char *e = memchr(p, '"', (size_t)(to - p)); if (!e) return NULL; *len = (size_t)(e - p); return p; } static int census_cmp(const void *x, const void *y) { const CensusEntry *a = x, *b = y; size_t n = a->src_len < b->src_len ? a->src_len : b->src_len; int c = memcmp(a->src, b->src, n); if (c) return c; if (a->src_len != b->src_len) return a->src_len < b->src_len ? -1 : 1; return a->stage - b->stage; } static char *census_read(const char *root, size_t *size) { char path[PLAT_PATH_MAX * 2 + 64]; snprintf(path, sizeof path, "%s/jobs.json", root); FILE *f = fopen(path, "rb"); if (!f) return NULL; fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET); char *b = malloc((size_t)n + 1); if (b && fread(b, 1, (size_t)n, f) != (size_t)n) free(b), b = NULL; fclose(f); if (b) b[n] = 0, *size = (size_t)n; return b; } static int census_parse(const char *jobs, CensusEntry *out, int max) { int n = 0; for (const char *p = jobs; n < max && (p = strstr(p, "\"job\": {")); p += 8) { const char *end = strstr(p, "\"origin\": "); if (!end) break; size_t kl, sl, cl, rl; const char *k = st_field(p, end, "key", &kl), *s = st_field(p, end, "stage", &sl), *c = st_field(p, end, "c1", &cl), *r = st_field(p, end, "source", &rl); if (!k || !s || !c || !r || kl != 64) continue; CensusEntry *e = &out[n++]; e->key = k, e->c1 = c, e->c1_len = cl / 2, e->src = r, e->src_len = rl; e->stage = !strncmp(s, "vs", sl) ? GFX_STAGE_VS : !strncmp(s, "ps", sl) ? GFX_STAGE_FS : GFX_STAGE_CS; e->local[0] = e->local[1] = e->local[2] = 1; const char *l = strstr(p, "\"local\": ["); if (l && l < end) { l += 10; for (int i = 0; i < 3; i++) { while (*l == ' ' || *l == '\n' || *l == '\r' || *l == ',') l++; e->local[i] = (uint32_t)strtoul(l, (char **)&l, 10); } } } return n; } static int hexv(char c) { return c <= '9' ? c - '0' : (c | 32) - 'a' + 10; } // The program the game would hand NVN for a census entry, translated with default answers (as gfx_shader_probe.c). static const GfxShader *census_shader(const char *root, const CensusEntry *e) { char path[PLAT_PATH_MAX * 2 + 128]; static uint8_t c1[8192]; uint8_t control[0x880] = {0}; if (e->c1_len > sizeof c1) return NULL; for (size_t b = 0; b < e->c1_len; b++) c1[b] = (uint8_t)(hexv(e->c1[b * 2]) << 4 | hexv(e->c1[b * 2 + 1])); snprintf(path, sizeof path, "%s/code/%.64s.code", root, e->key); FILE *f = fopen(path, "rb"); if (!f) return NULL; fseek(f, 0, SEEK_END); long body = ftell(f); fseek(f, 0, SEEK_SET); size_t skip = e->stage == GFX_STAGE_CS ? 0 : 0x30, total = skip + (size_t)body; uint8_t *code = calloc(1, total); int ok = code && fread(code + skip, 1, (size_t)body, f) == (size_t)body; fclose(f); if (!ok) return free(code), NULL; uint32_t c1_off = (uint32_t)(total - e->c1_len), length = (uint32_t)total; memcpy(control + 0x700, &c1_off, 4), memcpy(control + 0x704, &length, 4); GfxShaderAnswers a = {0}; if (e->stage == GFX_STAGE_CS) for (int k = 0; k < 3; k++) a.local[k] = (uint16_t)e->local[k]; const GfxShader *s = gfx_shader_get(e->stage, control, code, total, c1, &a); free(code); return s; } // A state for a census pair: one target per fragment output (RGBA8 of the output's type), D32S8 depth with the test on, // alpha blending on target 0, one RGBA32 attribute per vertex input, interleaved in stream 0. static void selftest_state(VkPipeState *s, const GfxShader *vs, const GfxShader *fs) { vk_pipe_state_defaults(s); s->vs = vs, s->fs = fs; GfxShaderIo io[32]; uint32_t outs = gfx_shader_outputs(fs, io); for (int i = 0; i < 8; i++) if (outs >> i & 1) s->color_fmt[i] = io[i].type == GFX_TYPE_UINT ? NVN_FORMAT_RGBA8UI : io[i].type == GFX_TYPE_SINT ? NVN_FORMAT_RGBA8I : NVN_FORMAT_RGBA8; s->depth_fmt = NVN_FORMAT_DEPTH32F_STENCIL8; s->ds.dt = 1; s->color.blend_en = 1; s->blend[0].src_rgb = NVN_BLEND_FUNC_SRC_ALPHA, s->blend[0].dst_rgb = NVN_BLEND_FUNC_ONE_MINUS_SRC_ALPHA; s->poly.cull = NVN_FACE_BACK; uint32_t ins = gfx_shader_inputs(vs, io), off = 0; for (int l = 0; l < 16; l++) if (ins >> l & 1) { va_st a = {io[l].type == GFX_TYPE_UINT ? NVN_FORMAT_RGBA32UI : io[l].type == GFX_TYPE_SINT ? NVN_FORMAT_RGBA32I : NVN_FORMAT_RGBA32F, off, 0}; memcpy(&s->vattr[l], &a, sizeof a); off += 16; } s->vstream[0].stride = off ? off : 16; } // Records the dynamic state of a pipeline into a command buffer that is never submitted (validation checks the calls). static int selftest_record(const VkPipeState *s, VkPipeline pipe) { VkCtx *c = vk_ctx(); VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .queueFamilyIndex = c->gfx_family}; VkCommandPool pool; if (vkCreateCommandPool(c->device, &pi, NULL, &pool) != VK_SUCCESS) return 0; VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = pool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; VkCommandBuffer cb; int ok = vkAllocateCommandBuffers(c->device, &ai, &cb) == VK_SUCCESS; if (ok) { VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(cb, &bi); vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, pipe); vk_pipeline_set_dynamic(cb, s); ok = vkEndCommandBuffer(cb) == VK_SUCCESS; } vkDestroyCommandPool(c->device, pool, NULL); return ok; } int vk_pipeline_selftest(void) { enum { MAXN = 20000, WANT_PAIRS = 64, WANT_CS = 16 }; char root[PLAT_PATH_MAX * 2]; if (!vk_init() || !ensure_init()) { printf("vk_pipeline selftest: FAILED: no Vulkan device\n"); return 1; } if (!probe_census_dir(root, sizeof root)) { // the census folder, build//shaders printf("vk_pipeline selftest: FAILED: no build/v%d/shaders/jobs.json above the exe\n", rt_game_version); return 1; } size_t size; char *jobs = census_read(root, &size); CensusEntry *e = malloc(sizeof *e * MAXN); int n = jobs && e ? census_parse(jobs, e, MAXN) : 0; if (!n) { printf("vk_pipeline selftest: FAILED: no census entries\n"); free(jobs), free(e); return 1; } qsort(e, (size_t)n, sizeof *e, census_cmp); long err0 = vk_validation_errors_total(); VkPipeStats st0; vk_pipeline_stats(&st0); // candidate pairs: each fragment shader of a source file with one of the file's vertex shaders (round robin) int (*pairs)[2] = malloc(sizeof *pairs * (size_t)n), np = 0, *cs = malloc(sizeof *cs * (size_t)n), nc = 0; for (int i = 0; i < n;) { int j = i, v0 = -1, nv = 0, nf = 0; for (; j < n && e[j].src_len == e[i].src_len && !memcmp(e[j].src, e[i].src, e[i].src_len); j++) { if (e[j].stage == GFX_STAGE_VS && v0 < 0) v0 = j; // sorted by stage within a file: vertex shaders first nv += e[j].stage == GFX_STAGE_VS; if (e[j].stage == GFX_STAGE_CS) cs[nc++] = j; } for (int k = i; k < j && nv; k++) if (e[k].stage == GFX_STAGE_FS) pairs[np][0] = v0 + nf++ % nv, pairs[np][1] = k, np++; i = j; } int tried = 0, built = 0, cached = 1, shader_fail = 0, ctried = 0, cbuilt = 0, recorded = 0, varying_fix = 0; for (int t = 0; t < WANT_PAIRS && t < np; t++) { const int *pr = pairs[(long long)t * np / (np < WANT_PAIRS ? np : WANT_PAIRS)]; tried++; const GfxShader *vs = census_shader(root, &e[pr[0]]), *fs = census_shader(root, &e[pr[1]]); if (!vs || !fs) { shader_fail++; continue; } if (gfx_shader_inputs(fs, NULL) & ~gfx_shader_outputs(vs, NULL)) varying_fix++; VkPipeState s; selftest_state(&s, vs, fs); GfxShaderAnswers fa = {0}; vk_pipeline_answers(&s, NULL, &fa); const VkPipeInfo *info; VkPipeline p = vk_pipeline_graphics(&s, &info); if (!p) continue; built++; const VkPipeInfo *again; if (vk_pipeline_graphics(&s, &again) != p || again != info) cached = 0; if (!recorded) recorded = selftest_record(&s, p); } for (int t = 0; t < WANT_CS && t < nc; t++) { const CensusEntry *c = &e[cs[(long long)t * nc / (nc < WANT_CS ? nc : WANT_CS)]]; ctried++; const GfxShader *s = census_shader(root, c); if (!s) { shader_fail++; continue; } const VkPipeInfo *info; VkPipeline p = vk_pipeline_compute(s, &info); if (!p) continue; cbuilt++; const VkPipeInfo *again; if (vk_pipeline_compute(s, &again) != p || again != info) cached = 0; } vk_pipeline_save(); VkPipeStats st; vk_pipeline_stats(&st); long errors = vk_validation_errors_total() - err0; printf("vk_pipeline selftest: graphics %d/%d census pairs (%d needed vertex outputs added), compute %d/%d, " "%d shaders failed to translate; layouts %llu (%llu push); cache hits %llu; dynamic state recorded %d; " "validation errors %ld\n", built, tried, varying_fix, cbuilt, ctried, shader_fail, (unsigned long long)(st.layouts - st0.layouts), (unsigned long long)(st.push_layouts - st0.push_layouts), (unsigned long long)(st.hits - st0.hits), recorded, errors); free(pairs), free(cs), free(e), free(jobs); int ok = built > 0 && cbuilt > 0 && cached && recorded && errors == 0; printf("vk_pipeline selftest: %s\n", ok ? "passed" : "FAILED"); return ok ? 0 : 1; }