From 3573a96f46319608d8c29c5180ffff767e15e0a3 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Fri, 13 Nov 2026 07:48:06 -0500 Subject: [PATCH] Keep the SPIR-V of shaders nobody read for 30 seconds compressed in memory. The shader memory cache held 777 MiB of SPIR-V for 7115 shaders for the whole session. A background thread now compresses the SPIR-V of each shader no reader pinned for shader_compress_idle_s seconds (zstd level 1, 26% of the bytes on the test install's 44,151 modules) and frees the plain copy; readers pin it through gfx_shader_spirv_pin and get it decoded back. What the drawing thread compares at pipeline misses and draws, the module's FNV-1a and its Binding decorations, is kept with the shader, so module_binding_missing, the water corrections and the DLSS, crumbs and skip lists decode nothing. A compute pipeline created at a dispatch and a mesh shader's first pipeline key still decode on that thread, and the memory report counts those decodes apart, with the SPIR-V as held against uncompressed and the shaders held compressed. shader_compress = 0 keeps it plain. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 13 +- nx2navi.ini.example | 6 + runtime/cfg.c | 7 +- runtime/gfx.h | 23 ++- runtime/gfx_shader.c | 285 ++++++++++++++++++++++++++---- runtime/gfx_shader_guess.c | 3 +- runtime/gfx_shader_probe.c | 12 +- runtime/vk_crumbs.c | 4 +- runtime/vk_dlaa.c | 4 +- runtime/vk_dump.c | 7 +- runtime/vk_exec_draw.c | 23 +-- runtime/vk_memory.c | 19 +- runtime/vk_pipeline.c | 133 ++++++++------ tests/compute_warm_reuse_check.py | 6 +- 14 files changed, 411 insertions(+), 134 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index ee4951ed..0fc28f19 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -278,8 +278,9 @@ keeps per-core data without locks and relies on that, so the host models it (`ru those none did, stage libraries by kind, compute pipelines, layouts, descriptor pools) beside the swapchain's pixels and DLSS's and FSR's figures; `mem: system` splits the process's commit into the game's heap and physical maps, the device-local memory (Windows charges it to the process), VMA's host-visible blocks, the C runtime heap with the shares - the runtime can name (shader caches, pipeline tables, the texture cache and its decode temporaries, the log queue, the - flight recorder), the data of the driver's pipeline caches (asked once a minute on the autosave thread) and the rest; + the runtime can name (shader caches, with their SPIR-V as held against uncompressed and the decompressions, pipeline + tables, the texture cache and its decode temporaries, the log queue, the flight recorder), the data of the driver's + pipeline caches (asked once a minute on the autosave thread) and the rest; `mem: growth` names what grew by more than 64 MiB since the report before and since the first report in play. Every ten minutes of play and at exit, `mem: growth ranking` ranks the categories by their growth since play began and marks those that only climbed (`mem_growth.h`). @@ -291,6 +292,14 @@ keeps per-core data without locks and relies on that, so the host models it (`ru pass it, up to twice it, while the card and the system's commit have room. Times are wall seconds from a clock of the presents, not present counts. What a switched-off upscaler or SMAA held is freed after it has gone unused for a while (`vk_dlaa.c` `idle_release`, `vk_fsr.c` `idle_note`, `vk_aa.c` `work_idle`). +- Translated shaders at rest: the shader memory cache keeps every shader for the whole session, but a shader's SPIR-V + is read only while modules are made of it. A background thread compresses (zstd, checked by decoding it) the SPIR-V + of each shader no reader used for `shader_compress_idle_s` seconds (30) and frees the plain copy; a reader pins it + and gets it decoded back, about 60 us for a mean module (`runtime/gfx_shader.c` "SPIR-V at rest", + `runtime/spv_store.h`). The SPIR-V keeps about a quarter of its size. What the drawing thread compares at pipeline + misses and draws (the module's hash and its Binding decorations) is kept apart and needs no decode; a compute + pipeline created at a dispatch and a mesh shader's first pipeline key decode on that thread, which the memory report + counts. `shader_compress = 0` keeps it plain. - Allocation: Vulkan Memory Allocator. Debugging: Vulkan validation layers, per-pass frame dumps (`NX2NAVI_DUMP`, `vk_dump.c`), a sampling profiler (`NX2NAVI_PROFILE`, `tools/prof_report.py`) and a flight recorder (`NX2NAVI_FLIGHT`). RenderDoc cannot capture, because it hides `VK_EXT_external_memory_host` from the application. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index ba58f1a0..fea73c2e 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -298,6 +298,12 @@ # the new translator cannot translate keeps its old translation. 0: shaders the new translator has not translated yet # are translated before they are used (at start or when the game first draws them), as earlier versions did. # shader_carry = 1 +# 1 keeps a shader's translation (its SPIR-V) compressed in memory once no pipeline was made of it for +# shader_compress_idle_s seconds (1 to 3600), and decompresses it when one is made again (about 0.06 ms for most +# shaders): about 570 MiB less memory with 7,000 shaders in memory, the picture unchanged. 0 keeps every translation +# as it is. +# shader_compress = 1 +# shader_compress_idle_s = 30 # Vulkan graphics pipeline library, 0 or 1. Default: on, except with AMD's Windows driver for RX 6000 and older cards # (RX 400 to RX 6000), where the game ended at every enemy killed with it on. There it is off: a pipeline needed for # the first time is built whole while its draw is left out for some frames, and the start waits until every pipeline diff --git a/runtime/cfg.c b/runtime/cfg.c index ff6f455e..aa8a31b0 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -68,7 +68,7 @@ static const char *const known[] = { "PRESNAP_PLACE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_HD", "RUMBLE_HD_MS", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", - "SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_BULK_WRITE", "SHADOW_CHECK", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHADOW_TABLE_STORES", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", + "SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_COMPRESS", "SHADER_COMPRESS_IDLE_S", "SHADER_MEMO", "SHADOW_BULK_WRITE", "SHADOW_CHECK", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHADOW_TABLE_STORES", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", "SVC_SLOW_MS", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", @@ -151,6 +151,7 @@ static const struct { const char *key, *section; } sections[] = { {"COMPUTE_KEEP", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_SPLIT_FILL", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_RELEASED_HOLD", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"WATER_PROJECTIVE", "performance"}, + {"SHADER_COMPRESS", "performance"}, {"SHADER_COMPRESS_IDLE_S", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"PIPELINE_CACHE_PLAY", "performance"}, {"PIPELINE_CACHE_PLAY_MB", "performance"}, {"FFX_PIPE_CACHE", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"TEX_CLOBBER", "performance"}, {"LOW_END_GPU", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, @@ -353,6 +354,10 @@ static const CfgMeta metas[] = { // all|: the compute pipelines the boot keeps, those dispatched most recently (vk_pipeline.c, compute-used.bin) {"COMPUTE_KEEP", "512", NULL, RESTART}, {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again + // gfx_shader.c "SPIR-V at rest": a shader's SPIR-V is kept zstd-compressed in memory once no pipeline was made of + // it for the idle seconds; 0 keeps every shader's SPIR-V plain + {"SHADER_COMPRESS", "1", "0|1", RESTART}, + {"SHADER_COMPRESS_IDLE_S", "30", "1..3600", RESTART}, // the low-end GPU mode's rows in the menu's Game tab (ui/menu.cpp rows_low_end) {"TEX_RELEASED_MAX", "", NULL, RESTART}, // vk_texture.c released_max: "" by the video memory budget, else 1..16384 {"MIRROR_HOT", "0", "0..1000", RESTART}, {"FREEZE_IDLE", "1800", "0..100000", RESTART}, diff --git a/runtime/gfx.h b/runtime/gfx.h index e21a6d78..d12709fc 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -63,8 +63,15 @@ void gfx_shader_disk_counts(uint64_t *sync, uint64_t *sync_ns, uint64_t *bg); // thread (count, ns) and by the loaders (gfx_shader.c load_budget) // The shader caches' host memory for the memory report (vk_memory.c system_log), at report time, in bytes: the memory // cache of translated shaders (shaders entries), the use masks, the cache pack's index, the play records' sets and -// pending appends. -typedef struct { uint64_t shaders, cache, use, pack, records; } GfxShaderRam; +// pending appends. The memory cache's SPIR-V (gfx_shader.c "SPIR-V at rest"): spv_held bytes held, plain copies and +// compressed frames together, against spv_full uncompressed; spv_packed shaders held compressed alone; compress 1 on, +// -1 off, 0 no shader yet. The decodes readers' pins made, and those on the drawing threads (count, time, longest). +typedef struct { + uint64_t shaders, cache, use, pack, records; + uint64_t spv_held, spv_full, spv_packed; + uint64_t decodes, decode_ns, draw_decodes, draw_decode_ns, draw_decode_max_ns; + int compress; +} GfxShaderRam; void gfx_shader_ram(GfxShaderRam *r); // Translator answers for this device (vk_caps.c): shaderFloat64, storage offset rounding, subgroup size per vs/fs/cs. void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals, @@ -108,7 +115,17 @@ size_t gfx_shader_warm_list(GfxWarmEnt **out); // malloc'd; recorde // Disk-cache load the caller owns (*owned 1: release it), or the memory cache's entry (*owned 0). NULL if not cached. const GfxShader *gfx_shader_load_private(const uint8_t iid[32], int *owned); void gfx_shader_release_private(const GfxShader *s); -const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words); +// A shader's SPIR-V may be held compressed while nobody reads it (gfx_shader.c "SPIR-V at rest"). A reader pins it: +// gfx_shader_spirv_pin returns the plain SPIR-V (decoded first if need be), valid until the matching +// gfx_shader_spirv_unpin; every pin needs its unpin, and the pointer is not used after it. Any thread. +const uint32_t *gfx_shader_spirv_pin(const GfxShader *s, uint64_t *words); +void gfx_shader_spirv_unpin(const GfxShader *s); +uint64_t gfx_shader_words(const GfxShader *s); // the SPIR-V's length in words, without reading it +// FNV-1a (64-bit, over the bytes) of the SPIR-V: worked out once per shader and kept, so it decodes nothing on a +// compressed shader (NX2NAVI_SKIP_SHADERS, vk_dump.c and vk_dlaa.c name shaders by it). +uint64_t gfx_shader_spirv_fnv(const GfxShader *s); +// The values of the module's OpDecorate Binding instructions, in module order, in *b (kept with the shader). +uint32_t gfx_shader_bindings(const GfxShader *s, const uint32_t **b); const char *gfx_shader_res_json(const GfxShader *s); // the translator's .res.json text int gfx_shader_write_code(const GfxShader *s, const char *path); // the guest program, from the stored inputs // A shader of the runtime's own (SPIR-V and a .res.json text as the translator writes them), outside the cache and diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 6c9ec67b..d6ff1fab 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -66,7 +66,13 @@ // 36,000 shaders: flushing these files too held its translation at half speed on an NVMe drive. // // Reflection: every shader that enters the cache has its .res.json parsed into GfxShaderRes arrays and its SPIR-V -// scanned for interface Locations and the workgroup size (gfx.h accessors at the end of the file). +// scanned for interface Locations, the workgroup size and its Binding decorations (gfx.h accessors at the end of the +// file). +// +// SPIR-V at rest (NX2NAVI_SHADER_COMPRESS, default 1; section "SPIR-V at rest"): the memory cache keeps every shader +// for the life of the process, but its SPIR-V is read only while modules are made of it. A background thread keeps the +// SPIR-V of a shader nobody read for NX2NAVI_SHADER_COMPRESS_IDLE_S seconds as a zstd frame alone, and a reader that +// pins it (gfx_shader_spirv_pin) gets it decoded back (runtime/spv_store.h). // // The c2 handle ranges of handle_slot are the translator's handle layout. // The code layout the translator sees (graphics: code[0x30..], compute: code[0..]; c1 = code[c1..length] @@ -80,7 +86,9 @@ #include "gfx.h" #include "host.h" #include "platform/platform.h" +#include "prof.h" #include "prp_file.h" +#include "spv_store.h" #define CONTROL_SIZE 0x880 #define CODE_SKIP 0x30 // graphics code starts after the first 0x30 bytes of the program @@ -114,9 +122,12 @@ struct GfxShader { struct GfxShader *next; uint8_t key[32]; uint8_t iid[32]; // input id (see the keys at the top) - uint32_t *spirv; - uint64_t words; - char *reflection; // the service's .res.json text, parsed into the fields below + // The SPIR-V: plain while the shader is built (spv.plain, spv.words), then pinned by readers and compressed when + // idle (section "SPIR-V at rest"). spv_lock guards what spv_store.h says. + SpvStore spv; + uint32_t *bind; // its OpDecorate Binding values in module order (spirv_seal) + uint32_t nbind; + char *reflection; // the service's .res.json text, parsed into the fields below uint64_t reflection_size; // Reflection (reflect(), once per shader before it enters the cache) int stage; // GFX_STAGE_* @@ -139,6 +150,7 @@ struct GfxShader { }; static PlatMutex cache_lock = PLAT_MUTEX_INIT; // memory cache +static PlatMutex spv_lock = PLAT_MUTEX_INIT; // every shader's SpvStore (spv_store.h) static GfxShader *buckets[BUCKETS]; static UseEnt *use_buckets[BUCKETS]; // under cache_lock static volatile int64_t n_requests, n_hits, n_failures, n_keys, n_xlated; // n_keys: memory cache entries @@ -694,7 +706,8 @@ static GfxShader *mem_find(const uint8_t key[32]) { static void shader_free(GfxShader *s) { for (int k = 0; k < GFX_RES_KINDS; k++) free(s->res[k]); - free(s->spirv), free(s->reflection), free(s); + spv_store_free(&s->spv); + free(s->bind), free(s->reflection), free(s); } static void reflect(GfxShader *s); @@ -708,6 +721,8 @@ static void carry_served(const GfxShader *s) { "next start", id[0], id[1], id[2], id[3]); } +static void zpass_kick(void); + // Inserts s, or frees it and returns the entry another thread inserted first. static GfxShader *mem_insert(GfxShader *s) { int added = 0; @@ -726,6 +741,7 @@ static GfxShader *mem_insert(GfxShader *s) { plat_mutex_unlock(&cache_lock); if (added && s->carried) carry_served(s); else if (added) ids_note(); + if (added) zpass_kick(); return s; } @@ -766,6 +782,31 @@ static int spirv_complete(const uint32_t *w, uint64_t n) { static void reflect_spirv(GfxShader *s); static void reflect_stage(GfxShader *s); +// The last step of making a shader, once its in-memory SPIR-V is final (after reflect and the HDR swap): the module's +// Binding decorations are listed (gfx_shader_bindings, which vk_pipeline.c module_binding_missing and vk_exec_draw.c +// mfb_binding_base read in place of the SPIR-V: the first runs on the drawing thread at every pipeline miss, and must +// not decode a compressed module there), and the store gets its first use stamp, so that a shader just loaded is not +// compressed before its first pipelines are made. +static void spirv_seal(GfxShader *s) { + const uint32_t *w = s->spv.plain; + uint64_t n = s->spv.words; + uint32_t cap = 0; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (!len || i + len > n) break; + if (op == 71 && len >= 4 && w[i + 2] == 33) { // OpDecorate Binding + if (s->nbind == cap) { + uint32_t *b = realloc(s->bind, (cap = cap ? cap * 2 : 16) * sizeof *b); + if (!b) rt_fatal("gfx_shader: out of memory"); + s->bind = b; + } + s->bind[s->nbind++] = w[i + 3]; + } + i += len; + } + s->spv.used_ms = plat_now_ms(); +} + // Reads .spv and .res.json from the pack (packed 1) or the loose files; NULL if either is missing or // malformed. iid: the input id, NULL when the caller sets it. full 0: only the stage and the interface Locations are // reflected and the SPIR-V is left as the translator wrote it (the carry pass checks a translation it does not keep). @@ -799,8 +840,7 @@ static GfxShader *disk_load_from(const uint8_t key[32], const uint8_t *iid, int GfxShader *s = calloc(1, sizeof *s); memcpy(s->key, key, 32); if (iid) memcpy(s->iid, iid, 32); - s->spirv = (uint32_t *)spv; - s->words = spv_size / 4; + spv_store_fill(&s->spv, (uint32_t *)spv, spv_size / 4, 0); s->reflection = (char *)res; s->reflection_size = res_size; if (!full) { @@ -808,7 +848,8 @@ static GfxShader *disk_load_from(const uint8_t key[32], const uint8_t *iid, int return s; } reflect(s); - if (s->stage == GFX_STAGE_FS) vk_hdr_shader_swap(&s->spirv, &s->words); // the in-memory copy; the cache keeps its own + if (s->stage == GFX_STAGE_FS) vk_hdr_shader_swap(&s->spv.plain, &s->spv.words); // the in-memory copy; the cache keeps its own + spirv_seal(s); return s; } @@ -820,14 +861,16 @@ const GfxShader *gfx_shader_own(const uint32_t *spirv, uint64_t words, const cha if (!s) return NULL; size_t rn = strlen(res_json); uint8_t *buf = malloc(words * 4 + rn); - s->spirv = malloc(words * 4), s->reflection = malloc(rn + 1); - if (!buf || !s->spirv || !s->reflection) rt_fatal("gfx_shader: out of memory"); + spv_store_fill(&s->spv, malloc(words * 4), words, 0); + s->reflection = malloc(rn + 1); + if (!buf || !s->spv.plain || !s->reflection) rt_fatal("gfx_shader: out of memory"); memcpy(buf, spirv, words * 4), memcpy(buf + words * 4, res_json, rn); rt_sha256(buf, words * 4 + rn, s->key); free(buf); - memcpy(s->spirv, spirv, words * 4), memcpy(s->reflection, res_json, rn + 1); - s->words = words, s->reflection_size = rn; + memcpy(s->spv.plain, spirv, words * 4), memcpy(s->reflection, res_json, rn + 1); + s->reflection_size = rn; reflect(s); + spirv_seal(s); return s; } @@ -965,8 +1008,8 @@ enum { OP_ENTRY_POINT = 15, OP_EXECUTION_MODE = 16, OP_TYPE_INT = 21, OP_TYPE_FL // Interface Locations and the compute workgroup size from the module. Ids beyond 2^20 are ignored (translated modules stay // far below). static void reflect_spirv(GfxShader *s) { - const uint32_t *w = s->spirv; - uint64_t n = s->words; + const uint32_t *w = s->spv.plain; + uint64_t n = s->spv.words; s->local[0] = s->local[1] = s->local[2] = 1; if (n < 5) return; uint32_t bound = w[3] < (1u << 20) ? w[3] : (1u << 20); @@ -1040,8 +1083,8 @@ static void patch_txd_transpose(GfxShader *s) { const char *e = cfg_get("NX2NAVI_TXD_TRANSPOSE"); on = e && *e ? *e != '0' : rt_nvn_api_version[0] < 200; } - uint32_t *w = s->spirv; - uint64_t n = s->words; + uint32_t *w = s->spv.plain; + uint64_t n = s->spv.words; if (!on || n < 5) return; uint32_t bound = w[3] < (1u << 20) ? w[3] : (1u << 20); uint64_t *cc = calloc(bound, sizeof *cc); // word index of a 2-constituent OpCompositeConstruct defining the id @@ -1078,8 +1121,8 @@ static void patch_txd_transpose(GfxShader *s) { i += len; } o[3] = next; - free(s->spirv); - s->spirv = o, s->words = k; + free(s->spv.plain); + s->spv.plain = o, s->spv.words = k; if (plat_inc32(&n_logged) <= 4) rt_log("gfx_shader: %d gradient samples now read their derivatives per coordinate", found); out: free(cc); @@ -1105,8 +1148,8 @@ static void reflect_exact(GfxShader *s) { X_IN_BOUNDS_ACCESS_CHAIN = 66, X_COMPOSITE_EXTRACT = 81, X_COPY_OBJECT = 83, X_SAMPLED_IMAGE = 86, X_FETCH = 95, X_IMAGE = 100, X_QUERY_SIZE_LOD = 103, X_QUERY_SIZE = 104, X_QUERY_LEVELS = 106, X_DEC_BINDING = 33 }; static volatile int32_t n_pool_logged; - const uint32_t *w = s->spirv; - uint64_t n = s->words, fn = 0; + const uint32_t *w = s->spv.plain; + uint64_t n = s->spv.words, fn = 0; if (!s->nres[GFX_RES_TEX] || n < 5) return; int any = 0; // nine shaders in ten have no such read: they cost this walk over the instruction headers for (uint64_t i = 5; i < n && !any;) { @@ -1203,8 +1246,8 @@ static void patch_vec_index(GfxShader *s) { if (on) rt_log("gfx_shader: uniform vector components at computed indices are read whole and selected " "(NX2NAVI_VEC_INDEX_FIX=0 turns it off)"); } - uint32_t *w = s->spirv; - uint64_t n = s->words; + uint32_t *w = s->spv.plain; + uint64_t n = s->spv.words; if (!on || n < 5) return; uint32_t bound = w[3]; if (bound > (1u << 22)) return; @@ -1350,8 +1393,8 @@ static void patch_vec_index(GfxShader *s) { i += len; } o[3] = next; - free(s->spirv); - s->spirv = o, s->words = k; + free(s->spv.plain); + s->spv.plain = o, s->spv.words = k; if (plat_inc32(&n_logged) <= 4) rt_log("gfx_shader: %d uniform vector component reads at computed indices now select the component", found); #undef VX_OP @@ -1381,8 +1424,8 @@ static void reflect_palette(GfxShader *s) { P_IN_BOUNDS_ACCESS_CHAIN = 66, P_UCONVERT = 113, P_SCONVERT = 114, P_BITCAST = 124, P_IADD = 128, P_ISUB = 130, P_IMUL = 132, P_SHR = 194, P_SAR = 195, P_SHL = 196, P_OR = 197, P_AND = 199, P_FUNCTION = 54, P_DEC_BINDING = 33, P_SC_UNIFORM = 2, P_SC_STORAGE = 12 }; - const uint32_t *w = s->spirv; - uint64_t n = s->words, fn = 0; + const uint32_t *w = s->spv.plain; + uint64_t n = s->spv.words, fn = 0; s->pal_stride = 0, s->pal_slot = -1; if (s->stage != GFX_STAGE_VS || n < 5) return; int any = 0; // most vertex shaders read no integer input and stop here @@ -1981,6 +2024,131 @@ static int carry_has_iid(const uint8_t iid[32]) { return 0; } +// ---- SPIR-V at rest (NX2NAVI_SHADER_COMPRESS, default 1) +// The memory cache keeps every shader for the life of the process: 7115 shaders and 777 MiB of the C runtime heap in +// a session of 2026-11-13, nearly all of it SPIR-V (a .res.json averages 1.8 KB). The SPIR-V is read where modules +// are made of it: the pipeline workers and the boot's builds (vk_pipeline.c create_graphics, lib_create, emu_get), a +// compute pipeline the drawing thread creates at a dispatch (vk_pipeline.c compute_create), a mesh shader's outputs at +// its first pipeline key (mesh_outs), the mesh fallback's workers (vk_exec_draw.c mfb_module), an emulated mesh draw's +// primitive (mesh_emu_draw_prim), the shader preparation (gfx_shader_guess.c) and the diagnostics (vk_dump.c, +// gfx_shader_probe.c). What the drawing thread compares at every pipeline miss or draw comes from values kept apart: +// the module's FNV-1a (gfx_shader_spirv_fnv: vk_dlaa.c, vk_crumbs.c, the water corrections, NX2NAVI_SKIP_SHADERS) and +// its Binding decorations (gfx_shader_bindings: vk_pipeline.c module_binding_missing), so those decode nothing. +// Every reader pins the SPIR-V (gfx_shader_spirv_pin, gfx_shader_spirv_unpin; spv_store.h has the lock rules). One +// background thread (zpass_main) walks the cache every ZPASS_PERIOD_MS and compresses the SPIR-V of each shader no +// reader pinned for NX2NAVI_SHADER_COMPRESS_IDLE_S seconds (zstd level ZPASS_LEVEL, checked by decoding it before the +// plain copy goes), then frees the plain copy. The frame stays with the shader, so a shader read again keeps its +// decoded copy until it is idle once more, and then only loses that copy. The thread takes spv_lock for a few loads +// and stores per shader at normal priority, so that no reader waits for the scheduler to run a lowered thread inside +// the lock, and encodes at the lowest priority with no lock held. +// Measured on the shader cache of a test install (44,151 modules, 3,835 MiB, mean 91 KB, largest 4.8 MB): level 1 +// keeps 26.2% of the bytes (3.8 times smaller), encodes at 630 MB/s and decodes at 1.4 GB/s, 60 us for a mean module +// and about 3.4 ms for the largest compute shader; level 3 kept 26.5% and level 6 25.1% at a third of the speed, and a +// trained dictionary saved 1% more. The memory report shows the SPIR-V held, what it would be uncompressed, the +// shaders held compressed and the decodes, those on the drawing threads apart (gfx_shader_ram). +#define ZPASS_PERIOD_MS 2000 +#define ZPASS_LEVEL 1 + +static struct { + PlatOnce once; + PlatMutex pass; // held over a pass; gfx_shader_drop_memory takes it before it frees entries + volatile int32_t state; // 0 no shader yet, 1 compressing, -1 off (NX2NAVI_SHADER_COMPRESS=0) + uint32_t idle_s; // NX2NAVI_SHADER_COMPRESS_IDLE_S + volatile int64_t frames, frame_fail; // frames made, frames the encoder or its check refused + volatile int64_t dec, dec_ns; // decodes in pins (gfx_shader_spirv_pin, gfx_shader_spirv_fnv) + volatile int64_t dec_draw, dec_draw_ns, dec_draw_max_ns; // those on the drawing threads (prof.h) +} Z = {PLAT_ONCE_INIT, PLAT_MUTEX_INIT}; + +static void zpass_count_decode(uint64_t ns) { + if (!ns) return; + plat_inc64(&Z.dec), plat_add64(&Z.dec_ns, (int64_t)ns); + uint32_t tid = plat_thread_id(); + if (tid != prof_render_tid && tid != prof_replay_tid) return; + plat_inc64(&Z.dec_draw), plat_add64(&Z.dec_draw_ns, (int64_t)ns); + int64_t m = __atomic_load_n(&Z.dec_draw_max_ns, __ATOMIC_RELAXED); + while ((int64_t)ns > m && !__atomic_compare_exchange_n(&Z.dec_draw_max_ns, &m, (int64_t)ns, 0, __ATOMIC_RELAXED, + __ATOMIC_RELAXED)) {} +} + +// The bytes of a shader's plain SPIR-V and of its frame, as the memory report counts them (no lock: spv_store.h +// writes both pointers and the frame's size atomically). +static void spv_held(const GfxShader *g, uint64_t *plain, uint64_t *z) { + *plain = __atomic_load_n(&g->spv.plain, __ATOMIC_RELAXED) ? g->spv.words * 4 : 0; + *z = __atomic_load_n(&g->spv.zsize, __ATOMIC_RELAXED); +} + +// How long a shader goes unpinned before its SPIR-V is compressed. +static uint64_t zpass_idle_ms(void) { return (uint64_t)Z.idle_s * 1000; } + +static void zpass_main(void *arg) { + (void)arg; + SpvPacker pk = {0}; + int logged = 0; + for (;;) { + plat_sleep_ms(ZPASS_PERIOD_MS); + uint64_t idle_ms = zpass_idle_ms(), t0 = plat_now_ns(), raw = 0, packed = 0, freed = 0; + uint32_t made = 0, dropped = 0; + plat_mutex_lock(&Z.pass); + for (int i = 0; i < BUCKETS; i++) { + // Entries are inserted at a bucket's head and never unlinked while Z.pass is held: the chain read from the + // head stays valid without the lock, and an entry inserted meanwhile waits for the next pass. + plat_mutex_lock_shared(&cache_lock); + GfxShader *g = buckets[i]; + plat_mutex_unlock_shared(&cache_lock); + for (; g; g = g->next) { + uint64_t now = plat_now_ms(); + int step = spv_pack_begin(&g->spv, &spv_lock, now, idle_ms); + if (step == SPV_PACK_NONE) continue; + void *z = NULL; + uint32_t zn = 0; + if (step == SPV_PACK_ENCODE) { + plat_thread_priority(PLAT_PRIO_LOWEST); + z = spv_pack_encode(&g->spv, &pk, ZPASS_LEVEL, &zn); + plat_thread_priority(PLAT_PRIO_NORMAL); + if (z) { + plat_inc64(&Z.frames), made++, raw += g->spv.words * 4, packed += zn; + } else if (plat_inc64(&Z.frame_fail) <= 4) { + rt_log("gfx_shader: SPIR-V at rest: a shader's SPIR-V (%llu words) was not compressed (the encoder " + "failed or its frame did not decode to the same words); it stays as it is", + (unsigned long long)g->spv.words); + } + } + uint64_t f = spv_pack_end(&g->spv, &spv_lock, step, z, zn, now, idle_ms); + freed += f, dropped += f != 0; + } + } + plat_mutex_unlock(&Z.pass); + if (made && !logged) { + logged = 1; + rt_log("gfx_shader: SPIR-V at rest: the first pass compressed %u shaders unused for %u s, %.1f MiB to %.1f MiB " + "(%.1f%%), and freed %.1f MiB of plain SPIR-V in %u shaders, in %.0f ms (NX2NAVI_SHADER_COMPRESS=0 keeps " + "it plain)", made, (unsigned)(idle_ms / 1000), raw / 1048576.0, packed / 1048576.0, + raw ? 100.0 * (double)packed / (double)raw : 0.0, freed / 1048576.0, dropped, + (double)(plat_now_ns() - t0) / 1e6); + } + } +} + +static void zpass_start(void *arg) { + (void)arg; + if (!cfg_bool("NX2NAVI_SHADER_COMPRESS")) { + __atomic_store_n(&Z.state, -1, __ATOMIC_RELEASE); + rt_log("gfx_shader: SPIR-V at rest: off (NX2NAVI_SHADER_COMPRESS=0): every shader keeps its SPIR-V plain"); + return; + } + int idle = cfg_int("NX2NAVI_SHADER_COMPRESS_IDLE_S"); + Z.idle_s = idle < 1 ? 1 : (uint32_t)idle; + if (!plat_thread_spawn(zpass_main, NULL, "shader compress", 0)) { + __atomic_store_n(&Z.state, -1, __ATOMIC_RELEASE); + rt_log("gfx_shader: SPIR-V at rest: cannot start its thread; every shader keeps its SPIR-V plain"); + return; + } + __atomic_store_n(&Z.state, 1, __ATOMIC_RELEASE); +} + +// The first shader that enters the memory cache starts the background thread. +static void zpass_kick(void) { plat_once(&Z.once, zpass_start, NULL); } + // ---- public // Raw line exchange for the probe: one line to the service, its reply in `reply`. Starts the service if needed. @@ -1998,8 +2166,10 @@ uint64_t gfx_shader_failures(void) { return (uint64_t)n_failures; } uint64_t gfx_shader_requests(void) { return (uint64_t)n_requests; } // lines sent to the service uint64_t gfx_shader_cache_hits(void) { return (uint64_t)n_hits; } // gets answered from memory or disk -// Drops the memory cache (the probe uses it to prove the disk cache). Pointers handed out before are invalid. +// Drops the memory cache (the probe uses it to prove the disk cache). Pointers handed out before are invalid. A pass of +// the SPIR-V thread ends first: it walks the entries without cache_lock (zpass_main). void gfx_shader_drop_memory(void) { + plat_mutex_lock(&Z.pass); plat_mutex_lock(&cache_lock); for (int i = 0; i < BUCKETS; i++) { for (GfxShader *s = buckets[i], *n; s; s = n) { @@ -2018,6 +2188,7 @@ void gfx_shader_drop_memory(void) { use_files[i] = NULL; } plat_mutex_unlock(&cache_lock); + plat_mutex_unlock(&Z.pass); } static const GfxShader *fail(const char *why, const char *hex) { @@ -2833,9 +3004,35 @@ void gfx_shader_prewarm(const uint8_t (*iids)[32], size_t n, const GfxShader **o (double)plat_now_ms() - t0); free(w.other); } -const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *words) { - if (words) *words = s->words; - return s->spirv; +// Section "SPIR-V at rest": the plain SPIR-V, decoded first when the shader holds it compressed; valid until the +// matching unpin. A frame that does not decode (the packer checked it when it made it) or a decode buffer that cannot +// be had ends the process, as every other allocation of this file does. +const uint32_t *gfx_shader_spirv_pin(const GfxShader *cs, uint64_t *words) { + GfxShader *s = (GfxShader *)cs; + uint64_t ns; + const uint32_t *w = spv_pin(&s->spv, &spv_lock, &ns); + if (!w) rt_fatal("gfx_shader: a shader's compressed SPIR-V (%llu words) cannot be decoded", (unsigned long long)s->spv.words); + zpass_count_decode(ns); + if (words) *words = s->spv.words; + return w; +} + +void gfx_shader_spirv_unpin(const GfxShader *s) { spv_unpin(&((GfxShader *)s)->spv, &spv_lock, plat_now_ms()); } + +uint64_t gfx_shader_words(const GfxShader *s) { return s->spv.words; } + +// Worked out once per shader, by the first caller or by the SPIR-V thread before it compresses the module, so that a +// caller on the drawing thread finds it ready for any compressed shader. +uint64_t gfx_shader_spirv_fnv(const GfxShader *s) { + uint64_t ns; + uint64_t h = spv_fnv(&((GfxShader *)s)->spv, &spv_lock, plat_now_ms(), &ns); + zpass_count_decode(ns); + return h; +} + +uint32_t gfx_shader_bindings(const GfxShader *s, const uint32_t **b) { + *b = s->bind; + return s->nbind; } const char *gfx_shader_res_json(const GfxShader *s) { return s->reflection; } @@ -3805,19 +4002,30 @@ void gfx_shader_carry_start(void) { // memory cache's translated shaders (entries with their SPIR-V, reflection text and resource lists, kept for the life of // the process), the use masks (those read at the start, use_preload, and those found since), the cache pack's index, // and the records of programs and answers met in play (their sets and what waits to be appended). The walk holds -// cache_lock shared, as gfx_shader_sizes does: a translator's insert waits for it. +// cache_lock shared, as gfx_shader_sizes does: a translator's insert waits for it. The SPIR-V is counted as it is +// held (section "SPIR-V at rest": a plain copy, a compressed frame or both), read without spv_lock: a store the SPIR-V +// thread changes meanwhile is counted as it was or as it is. void gfx_shader_ram(GfxShaderRam *r) { memset(r, 0, sizeof *r); plat_mutex_lock_shared(&cache_lock); for (int i = 0; i < BUCKETS; i++) { for (const GfxShader *g = buckets[i]; g; g = g->next) { + uint64_t plain, z; + spv_held(g, &plain, &z); r->shaders++; - r->cache += sizeof *g + g->words * 4 + g->reflection_size + 1; + r->cache += sizeof *g + plain + z + (uint64_t)g->nbind * sizeof *g->bind + g->reflection_size + 1; for (int k = 0; k < GFX_RES_KINDS; k++) r->cache += (uint64_t)g->nres[k] * sizeof *g->res[k]; + r->spv_held += plain + z, r->spv_full += g->spv.words * 4, r->spv_packed += !plain && z; } for (const UseEnt *u = use_buckets[i]; u; u = u->next) r->use += sizeof *u; } plat_mutex_unlock_shared(&cache_lock); + r->compress = __atomic_load_n(&Z.state, __ATOMIC_ACQUIRE); + r->decodes = (uint64_t)__atomic_load_n(&Z.dec, __ATOMIC_RELAXED); + r->decode_ns = (uint64_t)__atomic_load_n(&Z.dec_ns, __ATOMIC_RELAXED); + r->draw_decodes = (uint64_t)__atomic_load_n(&Z.dec_draw, __ATOMIC_RELAXED); + r->draw_decode_ns = (uint64_t)__atomic_load_n(&Z.dec_draw_ns, __ATOMIC_RELAXED); + r->draw_decode_max_ns = (uint64_t)__atomic_load_n(&Z.dec_draw_max_ns, __ATOMIC_RELAXED); r->use += (uint64_t)__atomic_load_n(&use_files_n, __ATOMIC_RELAXED) * sizeof(UseFile) + sizeof use_files; r->pack = (uint64_t)pack_n * sizeof(PackEnt); plat_mutex_lock(&RC.lock); @@ -3825,14 +4033,19 @@ void gfx_shader_ram(GfxShaderRam *r) { plat_mutex_unlock(&RC.lock); } -// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): the shader memory cache's entries with their SPIR-V and reflection bytes -// (kept for the life of the process), the remembered failures, the translation and load queues, and the translator -// processes. cache_lock is held shared: lookups go on, a translator's insert waits for the walk. +// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): the shader memory cache's entries with their SPIR-V as held (plain and +// compressed, gfx_shader_ram) and reflection bytes (kept for the life of the process), the remembered failures, the +// translation and load queues, and the translator processes. cache_lock is held shared: lookups go on, a translator's +// insert waits for the walk. void gfx_shader_sizes(PerfSizes *s) { uint64_t n = 0, spv = 0, refl = 0, failed = 0, xq = 0, lq = 0, procs = 0, busy = 0; plat_mutex_lock_shared(&cache_lock); for (int i = 0; i < BUCKETS; i++) - for (const GfxShader *g = buckets[i]; g; g = g->next) n++, spv += g->words * 4, refl += g->reflection_size; + for (const GfxShader *g = buckets[i]; g; g = g->next) { + uint64_t plain, z; + spv_held(g, &plain, &z); + n++, spv += plain + z, refl += g->reflection_size; + } plat_mutex_unlock_shared(&cache_lock); plat_mutex_lock(&failed_lock); for (int i = 0; i < FAILED_N; i++) failed += failed_used[i] != 0; diff --git a/runtime/gfx_shader_guess.c b/runtime/gfx_shader_guess.c index 5b3c73ee..55c6e222 100644 --- a/runtime/gfx_shader_guess.c +++ b/runtime/gfx_shader_guess.c @@ -75,9 +75,10 @@ int gfx_shader_guess_refine(const GfxShader *s, int stage, const GfxShaderAnswer if (!s) return 0; if (stage == GFX_STAGE_VS) { uint64_t words = 0; - const uint32_t *w = gfx_shader_spirv(s, &words); + const uint32_t *w = gfx_shader_spirv_pin(s, &words); GfxShaderIo io[32]; uint32_t ints = w ? int_read_inputs(w, words) & gfx_shader_inputs(s, io) : 0; + gfx_shader_spirv_unpin(s); for (int l = 0; l < 32; l++) if (ints >> l & 1 && out->attr_type[l] == GFX_TYPE_FLOAT) out->attr_type[l] = GFX_TYPE_UINT; } else if (stage == GFX_STAGE_CS) { diff --git a/runtime/gfx_shader_probe.c b/runtime/gfx_shader_probe.c index 1ae4d43d..88f3fa14 100644 --- a/runtime/gfx_shader_probe.c +++ b/runtime/gfx_shader_probe.c @@ -171,8 +171,10 @@ int probe_shader(void) { first[i] = translate(&entries[i], &err); FAIL_IF(!first[i], "shader %s did not translate", entries[i].key); uint64_t words; - const uint32_t *w = gfx_shader_spirv(first[i], &words); - FAIL_IF(words < 5 || w[0] != 0x07230203, "shader %s: bad SPIR-V magic", entries[i].key); + const uint32_t *w = gfx_shader_spirv_pin(first[i], &words); + uint32_t magic = w[0]; + gfx_shader_spirv_unpin(first[i]); + FAIL_IF(words < 5 || magic != 0x07230203, "shader %s: bad SPIR-V magic", entries[i].key); } FAIL_IF(gfx_shader_requests() - req0 != WANT, "pass 1 sent %llu requests, expected %d", (unsigned long long)(gfx_shader_requests() - req0), WANT); @@ -187,8 +189,10 @@ int probe_shader(void) { second[i] = translate(&entries[i], &err); FAIL_IF(!second[i], "shader %s missing from the disk cache", entries[i].key); uint64_t words; - const uint32_t *w = gfx_shader_spirv(second[i], &words); - FAIL_IF(words < 5 || w[0] != 0x07230203, "shader %s: bad SPIR-V magic from disk", entries[i].key); + const uint32_t *w = gfx_shader_spirv_pin(second[i], &words); + uint32_t magic = w[0]; + gfx_shader_spirv_unpin(second[i]); + FAIL_IF(words < 5 || magic != 0x07230203, "shader %s: bad SPIR-V magic from disk", entries[i].key); } int hits = (int)(gfx_shader_cache_hits() - hits0); FAIL_IF(gfx_shader_requests() != req0, "pass 2 contacted the service"); diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c index 0ecbc715..8c7974ab 100644 --- a/runtime/vk_crumbs.c +++ b/runtime/vk_crumbs.c @@ -227,9 +227,7 @@ static uint64_t gp_shader_hash(const GfxShader *s) { if (!s) return 0; uint32_t i = (uint32_t)(((uintptr_t)s * 0x9E3779B97F4A7C15ull) >> 54); if (memo[i].s == s) return memo[i].h; - uint64_t words = 0, h = 0xcbf29ce484222325ull; - const uint8_t *b = (const uint8_t *)gfx_shader_spirv(s, &words); - for (uint64_t k = 0; b && k < words * 4; k++) h = (h ^ b[k]) * 0x100000001b3ull; + uint64_t h = gfx_shader_spirv_fnv(s); // kept by the shader: nothing is decoded here memo[i].s = s, memo[i].h = h; return h; } diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index a8d7ed0a..1eb62c00 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -3290,9 +3290,7 @@ static uint64_t spirv_hash(const GfxShader *s) { for (uint32_t i = 0, j = (uint32_t)(k >> 51); i < 8192; i++, j = (j + 1) & 8191) { if (tab[j].s == s) return tab[j].h; if (!tab[j].s) { - uint64_t words = 0, h = 0xcbf29ce484222325ull; - const uint8_t *b = (const uint8_t *)gfx_shader_spirv(s, &words); - for (uint64_t n = 0; b && n < words * 4; n++) h = (h ^ b[n]) * 0x100000001b3ull; + uint64_t h = gfx_shader_spirv_fnv(s); // kept by the shader: a compressed one is not decoded on this thread tab[j].s = s, tab[j].h = h; return h; } diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index b43263ad..5a58ea58 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -534,14 +534,13 @@ static uint64_t shader_hash(const GfxShader *s) { if (!s) return 0; for (int i = 0; i < n; i++) if (tab[i].s == s) return tab[i].h; - uint64_t words = 0, h = 0xcbf29ce484222325ull; - const uint32_t *w = gfx_shader_spirv(s, &words); - const uint8_t *b = (const uint8_t *)w; - for (uint64_t i = 0; i < words * 4; i++) h = (h ^ b[i]) * 0x100000001b3ull; + uint64_t words = 0, h = gfx_shader_spirv_fnv(s); + const uint32_t *w = gfx_shader_spirv_pin(s, &words); char path[600]; snprintf(path, sizeof path, "%s/sh_%016llx.spv", dir, (unsigned long long)h); FILE *f = fopen(path, "wb"); if (f) fwrite(w, 4, (size_t)words, f), fclose(f); + gfx_shader_spirv_unpin(s); snprintf(path, sizeof path, "%s/sh_%016llx.code", dir, (unsigned long long)h); // the guest program it came from gfx_shader_write_code(s, path); if (n < 1024) tab[n].s = s, tab[n].h = h, n++; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index f3406b2e..74dd0bd0 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -2475,9 +2475,7 @@ static int shader_skipped(const GfxShader *sh) { if (!n || !sh) return 0; uint32_t i = (uint32_t)(((uintptr_t)sh * 0x9E3779B97F4A7C15ull) >> 56); if (memo[i].s == sh) return memo[i].skip; - uint64_t words = 0, h = 0xcbf29ce484222325ull; // as vk_dump.c shader_hash - const uint8_t *b = (const uint8_t *)gfx_shader_spirv(sh, &words); - for (uint64_t k = 0; b && k < words * 4; k++) h = (h ^ b[k]) * 0x100000001b3ull; + uint64_t h = gfx_shader_spirv_fnv(sh); // as vk_dump.c shader_hash uint8_t skip = 0; for (int k = 0; k < n; k++) skip |= list[k] == h; memo[i].s = sh, memo[i].skip = skip; @@ -5839,15 +5837,10 @@ static struct { } MFB = {.lock = PLAT_MUTEX_INIT}; static uint32_t mfb_binding_base(const GfxShader *s) { // one past the highest binding the module declares - uint64_t n; - const uint32_t *w = gfx_shader_spirv(s, &n); - uint32_t top = 0; - for (uint64_t i = 5; i < n;) { - uint32_t op = w[i] & 0xffff, len = w[i] >> 16; - if (!len) break; - if (op == 71 && len == 4 && w[i + 2] == 33 && w[i + 3] > top) top = w[i + 3]; // OpDecorate Binding - i += len; - } + const uint32_t *b; + uint32_t n = gfx_shader_bindings(s, &b), top = 0; + for (uint32_t i = 0; i < n; i++) + if (b[i] > top) top = b[i]; return top + 1; } @@ -5856,10 +5849,11 @@ static uint32_t mfb_binding_base(const GfxShader *s) { // one past the highest static const GfxShader *mfb_module(const MfbCs *e, int stage, char *err, size_t cap) { const GfxShader *src = stage == 6 ? e->ts : e->ms; uint64_t n, on = 0; - const uint32_t *w = gfx_shader_spirv(src, &n); + const uint32_t *w = gfx_shader_spirv_pin(src, &n); uint32_t base = mfb_binding_base(src); uint32_t *out = stage == 6 ? meshfb_task_cs(w, n, &e->ti, base, &on, err, cap) : meshfb_mesh_cs(w, n, &e->mi, base, e->ti.payload_bytes / 4, &on, err, cap); + gfx_shader_spirv_unpin(src); if (!out) return NULL; static const int task_slots[1] = {MESHFB_CB_TASK}, mesh_slots[3] = {MESHFB_CB_TASK, MESHFB_CB_MESH, MESHFB_CB_LIST}; char *res = stage == 6 ? meshfb_res_json(gfx_shader_res_json(src), base, task_slots, 1) @@ -6086,8 +6080,9 @@ static int mesh_emu_draw_prim(VkExec *ex) { unsigned h = (unsigned)((uintptr_t)ms >> 4) & 255; if (memo[h].s != ms) { uint64_t n = 0; - const uint32_t *w = gfx_shader_spirv(ms, &n); + const uint32_t *w = gfx_shader_spirv_pin(ms, &n); // once per mesh shader the memo meets memo[h].s = ms, memo[h].prim = w && mesh_emu_prim(w, (size_t)n) == 1 ? NVN_DRAW_PRIMITIVE_POINTS : NVN_DRAW_PRIMITIVE_TRIANGLES; + gfx_shader_spirv_unpin(ms); } return memo[h].prim; } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index a96d0704..ef800fd7 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -5485,6 +5485,17 @@ static void system_log(uint64_t frame, const ReportFig *f, const VkPipeMem *pm, flight_mem(&frec, &fthumb); uint64_t lq = logq_bytes(); uint64_t shaders = sr.cache + sr.use + sr.pack + sr.records; + // The shaders' SPIR-V as held (gfx_shader.c "SPIR-V at rest"): against what it would be uncompressed, how many + // shaders hold it compressed alone, and the decodes readers' pins made, those on the drawing threads apart. + char spv[400]; + if (sr.compress > 0) + snprintf(spv, sizeof spv, " (their SPIR-V %.0f, uncompressed %.0f; %llu shaders compressed; %llu decompressions " + "in %.0f ms, %llu of them on the drawing threads in %.1f ms, the longest %.2f ms)", sr.spv_held / M, + sr.spv_full / M, (unsigned long long)sr.spv_packed, (unsigned long long)sr.decodes, sr.decode_ns / 1e6, + (unsigned long long)sr.draw_decodes, sr.draw_decode_ns / 1e6, sr.draw_decode_max_ns / 1e6); + else + snprintf(spv, sizeof spv, " (their SPIR-V %.0f, uncompressed%s)", sr.spv_held / M, + sr.compress < 0 ? ": shader_compress = 0" : ""); uint64_t pipes = pm->ram_pipes + pm->ram_keys + pm->ram_seed + pm->ram_binaries; uint64_t texhost = th.bytes + th.tmp, known = shaders + pipes + texhost + lq + frec; uint64_t pcache = pm->cache[0] + pm->cache[1]; @@ -5503,15 +5514,15 @@ static void system_log(uint64_t frame, const ReportFig *f, const VkPipeMem *pm, f->hblocks / M, fthumb / M); if (hi.committed) mg_put(line, sizeof line, &k, " + the C runtime heap %.0f (%.0f of it handed out; of it known: shader caches %.0f in " - "%llu shaders, pipeline tables %.0f, the texture cache %.0f with decode temporaries %.0f now and %.0f at most " + "%llu shaders%s, pipeline tables %.0f, the texture cache %.0f with decode temporaries %.0f now and %.0f at most " "since the last report, the log queue %.0f, the flight recorder's records %.0f; the rest %.0f)", - hi.committed / M, hi.used / M, shaders / M, (unsigned long long)sr.shaders, pipes / M, texhost / M, + hi.committed / M, hi.used / M, shaders / M, (unsigned long long)sr.shaders, spv, pipes / M, texhost / M, th.tmp / M, th.tmp_peak / M, lq / M, frec / M, ((double)hi.committed - (double)known) / M); else mg_put(line, sizeof line, &k, " + what the runtime knows it allocated %.0f (the C runtime heap's size is not " - "known: shader caches %.0f in %llu shaders, pipeline tables %.0f, the texture cache %.0f with decode " + "known: shader caches %.0f in %llu shaders%s, pipeline tables %.0f, the texture cache %.0f with decode " "temporaries %.0f now and %.0f at most since the last report, the log queue %.0f, the flight recorder's " - "records %.0f)", known / M, shaders / M, (unsigned long long)sr.shaders, pipes / M, texhost / M, + "records %.0f)", known / M, shaders / M, (unsigned long long)sr.shaders, spv, pipes / M, texhost / M, th.tmp / M, th.tmp_peak / M, lq / M, frec / M); mg_put(line, sizeof line, &k, " + the driver's pipeline cache data %.0f (pipeline-cache.bin's %.0f, the play cache's " "%.0f, %s) + other %.0f (the driver's own memory, thread stacks, the program's data; %u other heaps)", diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 3f9dbf03..bbe62ad2 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2879,24 +2879,22 @@ static int add_stage(Bindings *B, const GfxShader *s, uint32_t off) { } // The first binding a module's resource variables use that the binding list b[0..n) lacks (module bindings >= 1 moved -// by off), or -1. A reflection that misses a resource would otherwise hand the driver an invalid layout. +// by off), or -1. A reflection that misses a resource would otherwise hand the driver an invalid layout. The module's +// Binding decorations come from the list the shader keeps (gfx_shader_bindings), not from its SPIR-V: this runs on the +// drawing thread at every pipeline miss (pipe_miss), where a compressed module would be decoded. static int module_binding_missing(const GfxShader *s, const VkDescriptorSetLayoutBinding *b, uint32_t n, uint32_t off) { - uint64_t words = 0; - const uint32_t *w = gfx_shader_spirv(s, &words); + const uint32_t *bind; + uint32_t nb = gfx_shader_bindings(s, &bind); const GfxShaderRes *tr; int nt = gfx_shader_resources(s, GFX_RES_TEX, &tr); - for (uint64_t i = 5; w && i < words;) { - uint32_t op = w[i] & 0xffff, len = w[i] >> 16; - if (!len || i + len > words) break; + for (uint32_t i = 0; i < nb; i++) { int pooled = 0; // a pool array: set 2, not in this layout (spirv_pool_rebind) - for (int t = 0; t < nt && op == SPV_OP_DECORATE && len >= 4; t++) - if (tr[t].pool && tr[t].binding == (w[i + 3] & 0xffff)) pooled = 1; // GfxShaderRes.binding is 16 bits - if (op == SPV_OP_DECORATE && len >= 4 && w[i + 2] == SPV_DEC_BINDING && !pooled) { - uint32_t want = w[i + 3] + (w[i + 3] >= 1 ? off : 0), j = 0; - while (j < n && b[j].binding != want) j++; - if (j == n) return (int)w[i + 3]; - } - i += len; + for (int t = 0; t < nt; t++) + if (tr[t].pool && tr[t].binding == (bind[i] & 0xffff)) pooled = 1; // GfxShaderRes.binding is 16 bits + if (pooled) continue; + uint32_t want = bind[i] + (bind[i] >= 1 ? off : 0), j = 0; + while (j < n && b[j].binding != want) j++; + if (j == n) return (int)bind[i]; } return -1; } @@ -3109,15 +3107,17 @@ static const MeshEmuEnt *emu_get(const GfxShader *ms, const GfxShader *ts, VkPip const char *why = NULL; uint64_t n = 0; size_t cn = 0; - const uint32_t *w = gfx_shader_spirv(ms, &n); + const uint32_t *w = gfx_shader_spirv_pin(ms, &n); uint32_t *c = w ? mesh_emu_compute(w, (size_t)n, &e->mi, &cn, &why) : NULL; + gfx_shader_spirv_unpin(ms); if (c && (e->mesh = emu_compute(c, cn, layout))) e->ok = 1; else if (!why) why = "the mesh stage's compute pipeline was not created"; free(c); if (e->ok && e->mi.payload_words && !ts) e->ok = 0, why = "a mesh stage that reads a task payload without a task stage"; if (e->ok && ts) { - w = gfx_shader_spirv(ts, &n); + w = gfx_shader_spirv_pin(ts, &n); c = w ? mesh_emu_compute(w, (size_t)n, &e->ti, &cn, &why) : NULL; + gfx_shader_spirv_unpin(ts); uint32_t *r = c && toff ? spirv_rebind(c, cn, toff) : NULL; // the task stage's bindings follow the fragment stage's if (c && (e->task = emu_compute(r ? r : c, cn, layout))) { if (e->ti.payload_words != e->mi.payload_words) e->ok = 0, why = "task and mesh payload sizes differ"; @@ -3277,9 +3277,10 @@ static uint32_t mesh_outs(const GfxShader *ms, int *points) { } plat_mutex_unlock(&MSO.lock); uint64_t n = 0; - const uint32_t *w = gfx_shader_spirv(ms, &n); + const uint32_t *w = gfx_shader_spirv_pin(ms, &n); // once per mesh shader: decoded here if it is compressed *points = 0; mask = w ? spirv_mesh_outputs(w, n, points) : ~0u; // as create_graphics takes a module it cannot read + gfx_shader_spirv_unpin(ms); plat_mutex_lock(&MSO.lock); for (j = (uint32_t)(((uintptr_t)ms >> 4) * 0x9e3779b1u) & (MESH_OUTS_CAP - 1); MSO.ms[j] && MSO.ms[j] != ms;) j = (j + 1) & (MESH_OUTS_CAP - 1); @@ -3583,23 +3584,24 @@ enum { WATER_VERTEX, WATER_FRAGMENT, WATER_ABSORPTION }; static struct { uint64_t key; uint32_t *code; uint64_t words; int state; } water_mods[8]; static PlatMutex water_lock = PLAT_MUTEX_INIT; -// The edited module that stands for the module of hash `key`, or NULL. w: the translation it is made from at the -// first call (NULL: only look); a fragment translation is first put into its set 1 form, which must be the module -// `key` names, as it is the form the pipeline builds. -static const uint32_t *water_module(uint64_t key, int kind, const WaterProfile *p, const uint32_t *w, uint64_t n, - uint64_t *words) { +// The edited module that stands for the module of hash `key`, or NULL. src: the translation it is made from at the +// first call (NULL: only look), whose SPIR-V is pinned for the edit only; a fragment translation is first put into its +// set 1 form, which must be the module `key` names, as it is the form the pipeline builds. +static const uint32_t *water_module(uint64_t key, int kind, const WaterProfile *p, const GfxShader *src, uint64_t *words) { const uint32_t *code = NULL; plat_mutex_lock(&water_lock); int i = 0; while (i < 8 && water_mods[i].state && water_mods[i].key != key) i++; - if (i < 8 && !water_mods[i].state && w) { - uint64_t wn = 0; + if (i < 8 && !water_mods[i].state && src) { + uint64_t wn = 0, n = 0; + const uint32_t *w = gfx_shader_spirv_pin(src, &n); uint32_t *c = NULL, *set1 = kind == WATER_VERTEX ? NULL : spirv_set(w, n, 1); if (kind == WATER_VERTEX) c = spirv_water_vertex(w, n, &wn); else if (set1 && fnv(FNV0, set1, (size_t)n * 4) == key) c = kind == WATER_FRAGMENT ? spirv_water_fragment(set1, n, &wn) : spirv_water_absorption(set1, n, p->mat_pointer, p->mat_distance, &wn); free(set1); + gfx_shader_spirv_unpin(src); water_mods[i].key = key, water_mods[i].code = c, water_mods[i].words = wn, water_mods[i].state = c ? 1 : -1; static const char *const names[] = {"vertex stage of the coordinate pair", "fragment stage of the coordinate pair", "absorption material"}; @@ -3614,23 +3616,21 @@ static const uint32_t *water_module(uint64_t key, int kind, const WaterProfile * } // The water mode of a pipeline's shaders: 1 a coordinate pair whose two edited modules exist, 2 the absorption -// material whose edited module exists, else 0. +// material whose edited module exists, else 0. The drawing thread asks it for every pipeline it links (lib_keys): the +// modules' hashes are the ones the shaders keep (gfx_shader_spirv_fnv), and the SPIR-V is read only to make an edited +// module, once per profile. static int water_projective_pair(const VkPipeState *s) { static int on = -1; if (on < 0) on = cfg_bool("NX2NAVI_WATER_PROJECTIVE"); if (!on || !P.indep || !s->fs) return 0; - uint64_t vn = 0, fn = 0, wn; - const uint32_t *f = gfx_shader_spirv(s->fs, &fn); - if (!f) return 0; - uint64_t fh = fnv(FNV0, f, (size_t)fn * 4); + uint64_t wn, fh = gfx_shader_spirv_fnv(s->fs); for (size_t i = 0; i < sizeof water_profiles / sizeof *water_profiles; i++) { const WaterProfile *p = &water_profiles[i]; // The absorption correction is independent of the coordinate pair. - if (fh == p->mat) return water_module(p->mat_set1, WATER_ABSORPTION, p, f, fn, &wn) ? 2 : 0; + if (fh == p->mat) return water_module(p->mat_set1, WATER_ABSORPTION, p, s->fs, &wn) ? 2 : 0; if (fh != p->fs) continue; - const uint32_t *v = s->vs ? gfx_shader_spirv(s->vs, &vn) : NULL; - if (!v || fnv(FNV0, v, (size_t)vn * 4) != p->vs) return 0; - return water_module(p->vs, WATER_VERTEX, p, v, vn, &wn) && water_module(p->fs_set1, WATER_FRAGMENT, p, f, fn, &wn); + if (!s->vs || gfx_shader_spirv_fnv(s->vs) != p->vs) return 0; + return water_module(p->vs, WATER_VERTEX, p, s->vs, &wn) && water_module(p->fs_set1, WATER_FRAGMENT, p, s->fs, &wn); } return 0; } @@ -3639,7 +3639,7 @@ static int water_projective_pair(const VkPipeState *s) { // (in its set 1 form), else the module itself. static VkShaderModule water_pair_module(const uint32_t *w, uint64_t n, int early, int mode) { uint64_t words = 0; - const uint32_t *code = mode ? water_module(fnv(FNV0, w, (size_t)n * 4), 0, NULL, NULL, 0, &words) : NULL; + const uint32_t *code = mode ? water_module(fnv(FNV0, w, (size_t)n * 4), 0, NULL, NULL, &words) : NULL; if (mode == 1 && !code) LOG_ONCE("vk_pipeline: water: a coordinate pair's fragment module is not in its known form; built as it is"); return code ? make_module(code, words, early) : make_module(w, n, early); } @@ -3648,7 +3648,7 @@ static VkShaderModule water_pair_module(const uint32_t *w, uint64_t n, int early // outputs the vertex shader's union adds (spirv_add_outputs), which differ from one install to the next and are added // to the edited module as to any other. NULL in every other case. The module stays the table's. static const uint32_t *water_pair_vertex(const uint32_t *v, uint64_t n, int mode, uint64_t *words) { - return mode == 1 && v ? water_module(fnv(FNV0, v, (size_t)n * 4), 0, NULL, NULL, 0, words) : NULL; + return mode == 1 && v ? water_module(fnv(FNV0, v, (size_t)n * 4), 0, NULL, NULL, words) : NULL; } // water_projective_pair for a pipeline of key k. A coordinate pair blended with dual-source factors keeps its original @@ -3703,7 +3703,12 @@ static uint32_t mesh_dyn(VkDynamicState *out) { static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Layout *l, uint32_t fs_off) { VkShaderModule vm = VK_NULL_HANDLE, fm = VK_NULL_HANDLE; uint64_t vw, fw, vm_hash = 0, fm_hash = 0; // the hashes: of the final modules, for the PIPE_FEEDBACK 2 line - const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw); + // The stages' translations, pinned until the pipeline is made (gfx.h gfx_shader_spirv_pin): vcode and fcode become + // patched copies on the way, vplain and fplain stay the translations. + uint64_t vw0 = 0, fw0 = 0; + const uint32_t *vplain = gfx_shader_spirv_pin(s->vs, &vw0), *fplain = s->fs ? gfx_shader_spirv_pin(s->fs, &fw0) : NULL; + const uint32_t *vcode = vplain; + vw = vw0; uint32_t *vpatched = NULL, *fpatched = NULL; int mesh = is_mesh(s), emu = mesh && l->emu; // the water pair's vertex edit is for a translated vertex stage: an emulated mesh draw's generated one has none @@ -3714,7 +3719,11 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay if (emu) { // NX2NAVI_MESH_EMULATION: the task and mesh stages as compute pipelines, a generated vertex shader here uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0); const MeshEmuEnt *me = emu_get(s->vs, s->ts, l->layout, toff, 1); - if (!me || !me->ok) return VK_NULL_HANDLE; + if (!me || !me->ok) { + gfx_shader_spirv_unpin(s->vs); + if (s->fs) gfx_shader_spirv_unpin(s->fs); + return VK_NULL_HANDLE; + } GfxShaderIo fin[32]; uint8_t ty[32], cp[32]; uint32_t need = s->fs ? gfx_shader_inputs(s->fs, fin) : 0; @@ -3723,21 +3732,24 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay cp[i] = need >> i & 1 && !fin[i].packed && fin[i].comps ? (uint8_t)fin[i].comps : 4; } size_t vn = 0; - uint64_t fsw = 0; - const uint32_t *fcode0 = s->fs ? gfx_shader_spirv(s->fs, &fsw) : NULL; - vpatched = mesh_emu_vs(&me->mi, need, ty, cp, fcode0, (size_t)fsw, &vn); - if (!vpatched) return VK_NULL_HANDLE; + vpatched = mesh_emu_vs(&me->mi, need, ty, cp, fplain, (size_t)fw0, &vn); + if (!vpatched) { + gfx_shader_spirv_unpin(s->vs); + if (s->fs) gfx_shader_spirv_unpin(s->fs); + return VK_NULL_HANDLE; + } vcode = vpatched, vw = vn; mesh = 0; // a vertex pipeline from here on } VkShaderModule tm = VK_NULL_HANDLE; if (mesh && s->ts) { // the task stage's bindings follow another stage's in its set (task_off) uint64_t tw; - const uint32_t *tcode = gfx_shader_spirv(s->ts, &tw); + const uint32_t *tcode = gfx_shader_spirv_pin(s->ts, &tw); uint32_t toff = task_off(s, l, fs_off); uint32_t *t = tcode && toff ? spirv_rebind(tcode, tw, toff) : NULL; tm = make_module(t ? t : tcode, tw, 0); free(t); + gfx_shader_spirv_unpin(s->ts); } if (l->set1 && !mesh) { // independent layouts: outputs from the vertex shader's union, not from this fragment shader GfxShaderIo fin[32]; @@ -3768,7 +3780,8 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay if (!vpatched) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", missing); else vcode = vpatched, vw = n2; } - const uint32_t *fcode = gfx_shader_spirv(s->fs, &fw); + const uint32_t *fcode = fplain; + fw = fw0; if (missing && mesh && fcode) { // inputs no mesh output feeds read as Maxwell gives them (spirv_fs_fill_inputs) uint64_t n2 = 0; uint32_t *f = spirv_fs_fill_inputs(fcode, fw, missing, points, &n2); @@ -3806,10 +3819,8 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay n_stages++; } VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqv, rqf; - uint64_t sw; - const uint32_t *scode = gfx_shader_spirv(s->vs, &sw); // vcode may be a patched copy that is freed by now - if (!emu) pin_subgroup32(&stages[0], &rqv, scode, sw); - if (s->fs) pin_subgroup32(&stages[1], &rqf, gfx_shader_spirv(s->fs, &sw), sw); + if (!emu) pin_subgroup32(&stages[0], &rqv, vplain, vw0); // vcode may be a patched copy that is freed by now + if (s->fs) pin_subgroup32(&stages[1], &rqf, fplain, fw0); VkVertexInputAttributeDescription va[32]; VkVertexInputBindingDescription vb[17]; uint32_t na = 0, nb = 0; @@ -3918,6 +3929,8 @@ out: if (vm) vkDestroyShaderModule(P.device, vm, NULL); if (fm) vkDestroyShaderModule(P.device, fm, NULL); if (tm) vkDestroyShaderModule(P.device, tm, NULL); + gfx_shader_spirv_unpin(s->vs); + if (s->fs) gfx_shader_spirv_unpin(s->fs); return pipe; } @@ -4025,6 +4038,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi VkPipelineColorBlendStateCreateInfo cb = {.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; uint32_t *patched = NULL; VkPipeline pipe = VK_NULL_HANDLE; + const GfxShader *pinned[2] = {NULL, NULL}; // the shaders whose SPIR-V is pinned until out (gfx_shader_spirv_pin) if (kind == LIB_VI) { if (!P.dyn_vi) { VtxInput v; @@ -4043,13 +4057,13 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi // takes the subgroup pin there, so here too. No vertex outputs are added: what a fragment shader reads and // the mesh stage does not write is filled in the fragment module (FsKey.fill). uint64_t vw, tw = 0; - const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw); + const uint32_t *vcode = gfx_shader_spirv_pin(pinned[0] = s->vs, &vw); if (!vcode || !(m = make_module(vcode, vw, 0))) goto out; mst[0] = st, mst[0].stage = VK_SHADER_STAGE_MESH_BIT_EXT, mst[0].module = m; pin_subgroup32(&mst[0], &rq, vcode, vw); pi.stageCount = 1; if (s->ts) { - const uint32_t *tcode = gfx_shader_spirv(s->ts, &tw); + const uint32_t *tcode = gfx_shader_spirv_pin(pinned[1] = s->ts, &tw); uint32_t toff = binding_span(s->vs) - 1; patched = tcode && toff ? spirv_rebind(tcode, tw, toff) : NULL; if (!tcode || !(tm = make_module(patched ? patched : tcode, tw, 0))) goto out; @@ -4061,7 +4075,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi } else if (kind == LIB_PR) { const PrKey *pk = key; uint64_t vw, sw, n2 = 0; - const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw), *scode = vcode; + const uint32_t *vcode = gfx_shader_spirv_pin(pinned[0] = s->vs, &vw), *scode = vcode; sw = vw; uint64_t wn = 0; const uint32_t *vwater = water_pair_vertex(vcode, vw, pk->water_projective, &wn); @@ -4077,7 +4091,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi if (s->fs) { const FsKey *fk = key; uint64_t fw, sw, n2 = 0; - const uint32_t *fcode = gfx_shader_spirv(s->fs, &fw), *scode = fcode; + const uint32_t *fcode = gfx_shader_spirv_pin(pinned[0] = s->fs, &fw), *scode = fcode; sw = fw; uint32_t *d = NULL, *filled = NULL; if (fk->fill && fcode) { // a mesh pipeline's: before the other rewrites, as create_graphics @@ -4132,6 +4146,8 @@ out: if (m) vkDestroyShaderModule(P.device, m, NULL); if (tm) vkDestroyShaderModule(P.device, tm, NULL); free(patched); + for (int i = 0; i < 2; i++) + if (pinned[i]) gfx_shader_spirv_unpin(pinned[i]); return pipe; } @@ -6379,7 +6395,10 @@ static void compute_warm_clear(void) { static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cached_only, int *already_warm) { if (already_warm) *already_warm = 0; uint64_t words; - const uint32_t *code = gfx_shader_spirv(cs, &words); + // pinned until the pipeline is made: a dispatch on the drawing thread that creates its pipeline here decodes a + // compressed shader first, about 60 us for a mean module against the creation's milliseconds (gfx_shader.c + // "SPIR-V at rest", counted in the memory report) + const uint32_t *code = gfx_shader_spirv_pin(cs, &words); uint64_t ow = 0; uint32_t *over = spv_override(code, words, &ow); if (over) code = over, words = ow; @@ -6409,11 +6428,12 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache *already_warm = 1; __atomic_fetch_add(&compute_warm_reused, 1, __ATOMIC_RELAXED); free(over); + gfx_shader_spirv_unpin(cs); return VK_NULL_HANDLE; } } VkShaderModule m = make_module(code, words, 0); - if (!m) return free(over), VK_NULL_HANDLE; + if (!m) return free(over), gfx_shader_spirv_unpin(cs), VK_NULL_HANDLE; ci.stage.module = m; VkPipeline pipe = VK_NULL_HANDLE; VkPipelineCreationFeedbackCreateInfo fci; @@ -6434,6 +6454,7 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache (double)fb.duration / 1e6); vkDestroyShaderModule(P.device, m, NULL); free(over); + gfx_shader_spirv_unpin(cs); return pipe; } @@ -7349,8 +7370,7 @@ static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep, int uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; const char *why = NULL; - uint64_t words = 0; - gfx_shader_spirv(cs, &words); + uint64_t words = gfx_shader_words(cs); int big = LB.big_words && words > LB.big_words; // only from the caches (LB_BIG_WORDS) if (big && !keep) { // no run dispatched it: a pipeline of it would hold about 100 MiB of device memory unused plat_mutex_lock(&LB.stat); @@ -8420,9 +8440,8 @@ int vk_pipeline_probe_prewarm(void) { double l0 = now_ms(); const GfxShader *sh = gfx_shader_load_input(LB.e[i].iid, 0); // from the disk: the boot released its copy double load = now_ms() - l0; - uint64_t words = 0; if (!sh || gfx_shader_stage(sh) != GFX_STAGE_CS) continue; - gfx_shader_spirv(sh, &words); + uint64_t words = gfx_shader_words(sh); if (LB.big_words && words > LB.big_words) continue; // a first dispatch of one compiles it: not what is timed here const VkPipeInfo *info = NULL; double t0 = now_ms(); diff --git a/tests/compute_warm_reuse_check.py b/tests/compute_warm_reuse_check.py index 1ed2986c..5aa7f030 100644 --- a/tests/compute_warm_reuse_check.py +++ b/tests/compute_warm_reuse_check.py @@ -35,7 +35,9 @@ static uint64_t fnv(uint64_t h, const void *p, size_t n) { static void plat_mutex_lock(PlatMutex *m) { AcquireSRWLockExclusive(m); } static void plat_mutex_unlock(PlatMutex *m) { ReleaseSRWLockExclusive(m); } static void rt_sha256(const void *p, size_t n, uint8_t out[32]) { sha256(p,n,out); } -static const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *n) { *n=8;return s->code; } +static int pins; // gfx_shader_spirv_pin minus gfx_shader_spirv_unpin: every exit of compute_create unpins +static const uint32_t *gfx_shader_spirv_pin(const GfxShader *s, uint64_t *n) { pins++;*n=8;return s->code; } +static void gfx_shader_spirv_unpin(const GfxShader *s) { pins--; } static uint32_t *spv_override(const uint32_t *code, uint64_t n, uint64_t *out) { if (!override_word) return NULL; uint32_t *p=malloc(n*4);memcpy(p,code,n*4);p[7]=override_word;*out=n;return p; @@ -97,7 +99,7 @@ int main(void) { compute_warm_clear(); assert(compute_create(&a,&l,0,&reused) && !reused && calls==12); compute_warm_clear(); - assert(modules==0); + assert(modules==0 && pins==0); puts("compute warm reuse: equivalent requests, live handles, layout/flags/code separation, failures and cleanup passed"); return 0; }