mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
a1fcbb0b07
The driver keeps the prepared shaders' libraries in its shader cache, and under Windows that cache has a size limit set in NVIDIA's settings. A limit smaller than the set drops the oldest libraries as new ones come in (a 2048 MB cache could not hold them on the reporter's RTX 5070 Ti), so every start builds them again in the background and play stutters until that is done. The cold lines of the library warm now say so on NVIDIA under Windows, most plainly when the cache is cold right after a warm that finished on the same driver; the build kit's notes and the example file say it too. tests/warm_hint_check.py checks which hint each case gets. From pull request #403 by tarballz; two conflicts resolved (an include and the build kit's notes). Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
6096 lines
324 KiB
C
6096 lines
324 KiB
C
// Graphics and compute pipelines and descriptor layouts of the Vulkan replay. The op-stream replay (vk_exec.c)
|
|
// collects NVN state into a VkPipeState and asks for the pipeline of each draw; this file maps that state to Vulkan,
|
|
// numbers the descriptor bindings and keeps the caches.
|
|
//
|
|
// Binding model. The translator (tools/xlate/Program.cs, a SPIR-V back end) decorates every resource of a
|
|
// shader with DescriptorSet 0 and numbers the bindings of each shader from 1 in the order it meets them (Accessor
|
|
// Create*Binding); binding 0 is its support buffer (ResourceManager: SupportBuffer.Binding, a std140 uniform
|
|
// buffer every shader declares). The numbering is per shader, so the vertex and fragment stages of one program collide.
|
|
// The scheme here:
|
|
// - one descriptor set, set 0, per pipeline; no push constants (the translator's SPIR-V uses none);
|
|
// - binding 0: the support buffer, one uniform buffer shared by all stages of the pipeline;
|
|
// - vertex stage: its own bindings 1..n unchanged; fragment stage: its bindings b >= 1 move to b + n (the SPIR-V
|
|
// Binding decorations are rewritten when the module is created); compute: its own bindings;
|
|
// - descriptor types from the reflection: constant buffers UNIFORM_BUFFER, storage buffers STORAGE_BUFFER, textures
|
|
// COMBINED_IMAGE_SAMPLER (UNIFORM_TEXEL_BUFFER for buffer textures, SAMPLED_IMAGE for separate ones), images
|
|
// STORAGE_IMAGE (STORAGE_TEXEL_BUFFER for buffer images), array lengths as descriptor counts;
|
|
// - set 0 is a push-descriptor set (VK_KHR_push_descriptor) when its descriptors fit maxPushDescriptors (32 on the RX
|
|
// 9070; a pipeline layout may hold one push set only, which rules out a set per stage); larger ones (a vertex and a
|
|
// fragment stage near the census maxima of 15 and 27) are ordinary sets the replay allocates (VkPipeInfo.push 0).
|
|
// A fixed layout per stage combination (a binding per constant buffer, storage buffer, texture
|
|
// and image slot per stage) would hold about 150 descriptors and could never be pushed, so layouts are made per distinct
|
|
// binding list and shared by every pipeline with that list; VkPipeInfo.bindings tells the replay which NVN slot feeds
|
|
// each binding.
|
|
//
|
|
// Fragment inputs the vertex stage does not write (stages are translated separately) get
|
|
// outputs added to the vertex module, as Vulkan rejects such pairs, initialised to (0, 0, 0, 1) as Maxwell reads an
|
|
// unwritten output (docs/maxwell.md 1.2). The unwritten components of an output the vertex stage does write
|
|
// start at (0, 0, 0, 1) in the translator (tools/xlate/patches/0006).
|
|
//
|
|
// Pipeline key. Everything Vulkan 1.3 can take as dynamic state is dynamic: viewports and scissors with count, depth
|
|
// bias, blend constants, stencil masks and references, cull mode, front face, topology (within its class), depth and
|
|
// stencil tests and ops, depth bias enable, rasterizer discard; vertex input (VK_EXT_vertex_input_dynamic_state, else
|
|
// binding strides only); and, where VK_EXT_extended_dynamic_state3 offers them, depth clamp, polygon mode, blend enables,
|
|
// equations, write masks and alpha to coverage. The key holds the rest: shaders, target formats, sample count, topology
|
|
// class, depth clip mode, logic op and what is not dynamic on the GPU. The VkPipelineCache persists to
|
|
// pipeline-cache.bin next to nx2navi.exe (NX2NAVI_PIPELINE_CACHE overrides the path).
|
|
//
|
|
// The vertex format table has a 3-component fallback. The pipeline key holds the program, target formats, blend and
|
|
// write masks, vertex layout and sample count. The binding conventions are those of the shader translator.
|
|
// Contains code Copyright (c) 2026 Peiikai, Copyright (c) 2026 jhaizhou (MIT; see THIRD_PARTY_NOTICES.md).
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "nvn_hmap.h"
|
|
#include "platform/platform.h"
|
|
#include "tex.h"
|
|
#include "vk_gpa.h"
|
|
#include "vk_warm_hint.h"
|
|
|
|
#define LOG_ONCE(...) do { static volatile int32_t once_; if (!plat_xchg32(&once_, 1)) rt_log(__VA_ARGS__); } while (0)
|
|
|
|
enum { SPV_OP_ENTRY_POINT = 15, SPV_OP_TYPE_INT = 21, SPV_OP_TYPE_FLOAT = 22, SPV_OP_TYPE_VECTOR = 23,
|
|
SPV_OP_TYPE_POINTER = 32, SPV_OP_CONSTANT = 43, SPV_OP_CONSTANT_COMPOSITE = 44,
|
|
SPV_OP_CONSTANT_NULL = 46, SPV_OP_FUNCTION = 54, SPV_OP_VARIABLE = 59,
|
|
SPV_OP_DECORATE = 71, SPV_DEC_BUILTIN = 11, SPV_DEC_LOCATION = 30, SPV_DEC_INDEX = 32, SPV_DEC_BINDING = 33, SPV_DEC_DESCRIPTOR_SET = 34,
|
|
SPV_SC_OUTPUT = 3 };
|
|
|
|
#define MAX_BINDINGS 128
|
|
#define PIPE_BUCKETS 4096
|
|
#define LAYOUT_BUCKETS 256
|
|
|
|
// ---- device capabilities and the pipeline cache (set up on first use, under P.lock)
|
|
|
|
enum { NP_16 = 1, NP_32 = 2, NP_64 = 4 }; // float widths in P.nan_preserve and spirv_nan_preserve
|
|
|
|
static struct {
|
|
PlatMutex lock;
|
|
volatile int32_t ready;
|
|
VkDevice device;
|
|
VkPipelineCache cache;
|
|
uint32_t max_push;
|
|
int indep; // stage-independent layouts (vk_caps.gpl): vertex set 0, fragment set 1
|
|
int push_stage; // GFX_STAGE_* whose set is pushed with independent layouts
|
|
int gpl; // draws whose pipeline is building get a fast-linked one (vk_caps.gpl)
|
|
struct { uint64_t links, wlinks, dlinks, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g;
|
|
int vs_outs_all; // NX2NAVI_VS_OUTS: 0 union of paired fragment inputs, 1 (all) all 32 output locations
|
|
int dyn_vi, dyn_clamp, dyn_poly, dyn_blend_en, dyn_blend_eq, dyn_wmask, dyn_a2c;
|
|
int f_logic_op, f_depth_clamp, f_fill_non_solid, f_dual_src, f_full_subgroups;
|
|
int cache_control; // pipelineCreationCacheControl: a create can ask for the pipeline caches only
|
|
int feedback; // NX2NAVI_PIPE_FEEDBACK: 1 count pipeline cache hits, 2 also log each compute and graphics build
|
|
uint64_t cache_loaded; // bytes of pipeline-cache.bin the driver took at boot
|
|
int pv_last; // provokingVertexLast enabled: NVN's default is the last vertex
|
|
VkShaderStageFlags sg32_stages; // stages that accept requiredSubgroupSize = 32 (0: not available)
|
|
int reconverge; // modules with subgroup ops get MaximallyReconvergesKHR (spirv_reconverge)
|
|
uint32_t nan_preserve; // float widths that get SignedZeroInfNanPreserve (spirv_nan_preserve): NP_16|NP_32|NP_64
|
|
VkDynamicState dyn[32];
|
|
uint32_t n_dyn;
|
|
PFN_vkCmdPushDescriptorSetKHR push;
|
|
int tpl; // descriptor update templates on (make_tpl)
|
|
PFN_vkCreateDescriptorUpdateTemplate tpl_create;
|
|
PFN_vkDestroyDescriptorUpdateTemplate tpl_destroy;
|
|
PFN_vkUpdateDescriptorSetWithTemplate tpl_update;
|
|
PFN_vkCmdPushDescriptorSetWithTemplateKHR tpl_push;
|
|
PFN_vkCmdSetVertexInputEXT set_vi;
|
|
PFN_vkCmdSetDepthClampEnableEXT set_clamp;
|
|
PFN_vkCmdSetPolygonModeEXT set_poly;
|
|
PFN_vkCmdSetColorBlendEnableEXT set_blend_en;
|
|
PFN_vkCmdSetColorBlendEquationEXT set_blend_eq;
|
|
PFN_vkCmdSetColorWriteMaskEXT set_wmask;
|
|
PFN_vkCmdSetAlphaToCoverageEnableEXT set_a2c;
|
|
char cache_path[PLAT_PATH_MAX * 2];
|
|
VkPipeStats stats;
|
|
} P = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
typedef struct Layout {
|
|
struct Layout *next;
|
|
uint64_t hash;
|
|
VkDescriptorSetLayout set;
|
|
VkPipelineLayout layout;
|
|
int push;
|
|
uint32_t n;
|
|
VkDescriptorSetLayout set1; // independent layouts: the fragment set (set 1); VK_NULL_HANDLE otherwise
|
|
int push1; // set 1 is a push-descriptor set
|
|
uint32_t n0; // independent layouts: b[0..n0) are set 0, the rest set 1
|
|
VkPipelineLayout half[2]; // GPL libraries: {set 0, null}, {null, set 1} (made on first use)
|
|
VkDescriptorUpdateTemplate tpl[2]; // per set (make_tpl); VK_NULL_HANDLE: written with write structs
|
|
VkDescriptorSetLayoutBinding b[];
|
|
} Layout;
|
|
|
|
typedef struct Pipe {
|
|
struct Pipe *next;
|
|
uint64_t hash;
|
|
VkPipeInfo info;
|
|
VkPipeBinding *bindings;
|
|
int building; // a background job owns info.pipeline (under P.lock)
|
|
struct Job *job; // that job while building
|
|
const struct Layout *lay; // graphics: the layout it was entered with
|
|
VkPipeline fast; // GPL: the fast-linked pipeline (kept until shutdown: recorded draws use it)
|
|
uint8_t fast_tried;
|
|
uint8_t link_queued; // a waiting draw queued its fast link to the workers (LibJob p), not yet run
|
|
uint8_t used; // compute: a dispatch of this run asked for it (vk_pipeline_compute)
|
|
uint8_t pre; // compute: built and kept by the boot prebuild (lb_compute) or bg_main
|
|
size_t key_size;
|
|
uint8_t key[];
|
|
} Pipe;
|
|
|
|
static Layout *layouts[LAYOUT_BUCKETS];
|
|
static Pipe *pipes[PIPE_BUCKETS];
|
|
|
|
static uint64_t fnv(uint64_t h, const void *p, size_t n) {
|
|
const uint8_t *b = p;
|
|
for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 0x100000001b3ull;
|
|
return h;
|
|
}
|
|
#define FNV0 0xcbf29ce484222325ull
|
|
static __thread uint64_t mm_hash; // FNV-1a of the module make_module last created on this thread (PIPE_FEEDBACK 2)
|
|
|
|
// Hash of a pipeline table key, 8 bytes per step: a draw hashes its whole GKey, and byte-wise FNV over it cost more
|
|
// than the lookup. Only the in-memory table uses it; the key file keeps fnv.
|
|
static uint64_t key_hash(const void *k, size_t n) { return nvn_hm_hash(k, (uint32_t)n); }
|
|
|
|
// The cache file lives next to nx2navi.exe, so every build dir (the main build and each private fork) has its own:
|
|
// forks that shared build\v121's cache broke each other's runs (2026-10-22).
|
|
static void cache_path_init(void) {
|
|
const char *cfg_path = cfg_get("NX2NAVI_PIPELINE_CACHE");
|
|
if (cfg_path && *cfg_path && strlen(cfg_path) < sizeof P.cache_path) {
|
|
strcpy(P.cache_path, cfg_path);
|
|
return;
|
|
}
|
|
P.cache_path[0] = 0;
|
|
char exe[PLAT_PATH_MAX * 2];
|
|
int n = plat_exe_path(exe, sizeof exe);
|
|
if (!n || n >= sizeof exe) return;
|
|
char *cut = plat_last_sep(exe);
|
|
if (!cut) return;
|
|
*cut = 0;
|
|
snprintf(P.cache_path, sizeof P.cache_path, "%s/pipeline-cache.bin", exe);
|
|
}
|
|
|
|
// ---- pipeline binaries (VK_KHR_pipeline_binary; NX2NAVI_PIPE_BINARIES, default 0 while it is being tested; 1 on)
|
|
// pipeline-cache.bin does not carry compiled pipelines from one executable to another on NVIDIA 621.01: two
|
|
// byte-identical exes under different names, every compute module hash-identical, and the second compiled 317 of the
|
|
// 323 compute pipelines the first had read from the file (2026-11-09). A pipeline binary made from one exe's pipeline
|
|
// creates the same pipeline in another in 0.5 ms instead of 26 ms. So where the driver has the extension, every
|
|
// pipeline, stage library and compute pipeline is created through pipe_create: its driver pipeline key
|
|
// (vkGetPipelineKeyKHR, computed from the create info) is looked up in pipeline-binaries.bin next to nx2navi.exe and
|
|
// the pipeline is made from the stored binaries; a key not there is compiled with its data captured, and its binaries
|
|
// are appended to the file (autosave, every 2 s). pipeline-cache.bin is then neither read nor written: the driver's
|
|
// own disk cache serves an unchanged exe, and the binaries serve a renamed or updated one.
|
|
// Each binary is stored once under its own binary key: on NVIDIA 621.01 nearly every graphics pipeline has, besides
|
|
// its own binaries of 20-200 KB, one 2 MiB binary that is byte-identical across pipelines (2,506 copies in a 4.7 GB
|
|
// first-version file).
|
|
// File: header {"NX2NPBIN", version, the driver's global key size and bytes} (another global key: another driver or
|
|
// GPU, so the file starts empty); then records {u32 bytes after this field, u32 type, u32 key size, 32 key bytes,
|
|
// payload}: type PB_BLOB, a binary under its binary key, payload its data; type PB_PIPE, a pipeline under its pipeline
|
|
// key, payload {u32 count, count x {u32 key size, 32 key bytes}} naming its binaries. Only an index of the records
|
|
// stays in memory; a pipeline's records are read from the file when it is created. NX2NAVI_PIPE_BINARIES_MAX_MB
|
|
// (default 1024, 0 no limit): a larger file starts empty.
|
|
#define PB_VERSION 2
|
|
// 64-bit file offsets: long is 32 bits on Windows, and a file over 2 GB read as cut short.
|
|
#ifdef _WIN32
|
|
#define pb_seek _fseeki64
|
|
#define pb_tell _ftelli64
|
|
#else
|
|
#define pb_seek fseeko
|
|
#define pb_tell ftello
|
|
#endif
|
|
#define PB_MAX_BIN 8
|
|
#define PB_KEY VK_MAX_PIPELINE_BINARY_KEY_SIZE_KHR
|
|
enum { PB_BLOB = 1, PB_PIPE = 2 };
|
|
#define PB_HEAD (4 + 4 + PB_KEY) // a record's bytes after its size field and before its payload
|
|
typedef struct { char magic[8]; uint32_t version, gkey_size; uint8_t gkey[PB_KEY]; } PbHeader;
|
|
typedef struct { uint64_t h, off; uint32_t size; } PbSlot; // off: file offset of the record's size field
|
|
static struct {
|
|
int on;
|
|
PFN_vkCreatePipelineBinariesKHR create;
|
|
PFN_vkDestroyPipelineBinaryKHR destroy;
|
|
PFN_vkGetPipelineKeyKHR key;
|
|
PFN_vkGetPipelineBinaryDataKHR data;
|
|
PFN_vkReleaseCapturedPipelineDataKHR release;
|
|
char path[PLAT_PATH_MAX * 2 + 32];
|
|
PlatMutex lock; // the index, the pending records and rf
|
|
FILE *rf; // reads records
|
|
PbSlot *slot;
|
|
uint32_t cap, n, pipes, blobs;
|
|
uint8_t *pend; // records not written yet; they follow the file's end in this order
|
|
size_t pend_n, pend_cap;
|
|
uint64_t file_size; // bytes on disk
|
|
int no_capture[3]; // kind (pipe_create) whose create refused the capture flag
|
|
uint64_t from_file, load_fail, compiled, captured, capture_fail, blob_new, blob_shared;
|
|
double from_ms, compile_ms;
|
|
} PB = {.lock = PLAT_MUTEX_INIT};
|
|
static __thread int pb_last; // 1: the last pipe_create on this thread made its pipeline from binaries
|
|
static __thread int cc_hit; // compute_create's last pipeline on this thread: 1 the pipeline cache held it
|
|
// (P.cache: read from pipeline-cache.bin, or built into it earlier in this
|
|
// run) or it was made from pipeline binaries, 0 not, -1 the driver did not say
|
|
|
|
static uint64_t pb_hash(uint32_t type, const uint8_t *key, uint32_t n) {
|
|
uint64_t h = fnv(fnv(FNV0, &type, 4), key, n);
|
|
return h ? h : 1;
|
|
}
|
|
|
|
static PbSlot *pb_find(uint64_t h) { // caller holds PB.lock
|
|
if (!PB.cap) return NULL;
|
|
for (uint32_t j = (uint32_t)h & (PB.cap - 1);; j = (j + 1) & (PB.cap - 1)) {
|
|
if (!PB.slot[j].h) return NULL;
|
|
if (PB.slot[j].h == h) return &PB.slot[j];
|
|
}
|
|
}
|
|
|
|
// Enters a record (a hash match is taken as the key: 64 bits). 0 when one with the key is there already or out of
|
|
// memory. Caller holds PB.lock.
|
|
static int pb_insert(uint64_t h, uint64_t off, uint32_t size) {
|
|
if ((PB.n + 1) * 2 > PB.cap) {
|
|
uint32_t cap = PB.cap ? PB.cap * 2 : 4096;
|
|
PbSlot *t = calloc(cap, sizeof *t);
|
|
if (!t) return 0;
|
|
for (uint32_t i = 0; i < PB.cap; i++) {
|
|
if (!PB.slot[i].h) continue;
|
|
uint32_t j = (uint32_t)PB.slot[i].h & (cap - 1);
|
|
while (t[j].h) j = (j + 1) & (cap - 1);
|
|
t[j] = PB.slot[i];
|
|
}
|
|
free(PB.slot);
|
|
PB.slot = t, PB.cap = cap;
|
|
}
|
|
if (pb_find(h)) return 0;
|
|
uint32_t j = (uint32_t)h & (PB.cap - 1);
|
|
while (PB.slot[j].h) j = (j + 1) & (PB.cap - 1);
|
|
PB.slot[j] = (PbSlot){h, off, size};
|
|
PB.n++;
|
|
return 1;
|
|
}
|
|
|
|
static void pb_rewrite(const PbHeader *hd) { // an empty file with this driver's header
|
|
FILE *f = fopen(PB.path, "wb");
|
|
if (f) fwrite(hd, sizeof *hd, 1, f), fclose(f);
|
|
PB.file_size = f ? sizeof *hd : 0;
|
|
}
|
|
|
|
// Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache).
|
|
static void pb_init(VkCtx *c) {
|
|
if (!c->has_pipeline_binary || !P.cache_path[0]) return;
|
|
#define PBPROC(field, name) (PB.field = (void *)vk_dpa(c->device, name))
|
|
if (!PBPROC(create, "vkCreatePipelineBinariesKHR") || !PBPROC(destroy, "vkDestroyPipelineBinaryKHR") ||
|
|
!PBPROC(key, "vkGetPipelineKeyKHR") || !PBPROC(data, "vkGetPipelineBinaryDataKHR") ||
|
|
!PBPROC(release, "vkReleaseCapturedPipelineDataKHR")) {
|
|
rt_log("vk_pipeline: pipeline binaries: the driver lacks a VK_KHR_pipeline_binary function; using pipeline-cache.bin");
|
|
return;
|
|
}
|
|
#undef PBPROC
|
|
PbHeader want = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, PB_VERSION, 0, {0}};
|
|
VkPipelineBinaryKeyKHR gk = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
|
if (PB.key(c->device, NULL, &gk) != VK_SUCCESS || !gk.keySize || gk.keySize > PB_KEY) {
|
|
rt_log("vk_pipeline: pipeline binaries: no global key from the driver; using pipeline-cache.bin");
|
|
return;
|
|
}
|
|
want.gkey_size = gk.keySize;
|
|
memcpy(want.gkey, gk.key, gk.keySize);
|
|
snprintf(PB.path, sizeof PB.path, "%s", P.cache_path);
|
|
char *cut = plat_last_sep(PB.path);
|
|
snprintf(cut ? cut + 1 : PB.path, sizeof PB.path - (size_t)(cut ? cut + 1 - PB.path : 0), "pipeline-binaries.bin");
|
|
uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPE_BINARIES_MAX_MB"), NULL, 10) << 20;
|
|
FILE *f = fopen(PB.path, "rb");
|
|
PbHeader hd;
|
|
const char *why = NULL;
|
|
uint64_t good = 0, len = 0;
|
|
if (f) {
|
|
pb_seek(f, 0, SEEK_END);
|
|
len = (uint64_t)pb_tell(f);
|
|
pb_seek(f, 0, SEEK_SET);
|
|
if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version";
|
|
else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB";
|
|
else {
|
|
good = sizeof hd;
|
|
for (;;) { // index: per record its type, key and offset
|
|
uint32_t head[3];
|
|
uint8_t key[PB_KEY];
|
|
if (fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || head[0] < PB_HEAD ||
|
|
(head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || good + 4 + head[0] > len)
|
|
break;
|
|
if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) {
|
|
if (head[1] == PB_PIPE) PB.pipes++;
|
|
else PB.blobs++;
|
|
}
|
|
good += 4 + (uint64_t)head[0];
|
|
if (pb_seek(f, (int64_t)good, SEEK_SET)) break;
|
|
}
|
|
if (good != len) why = "cut short (a record half written)";
|
|
}
|
|
fclose(f);
|
|
}
|
|
if (why || !f) { // start empty: the index is dropped and the file rewritten
|
|
if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0);
|
|
free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
|
|
pb_rewrite(&want);
|
|
} else {
|
|
PB.file_size = len;
|
|
}
|
|
PB.rf = fopen(PB.path, "rb");
|
|
PB.on = PB.rf != NULL && PB.file_size >= sizeof want;
|
|
rt_log("vk_pipeline: pipeline binaries %s: %s, %u pipelines, %u binaries, %.1f MB (NX2NAVI_PIPE_BINARIES=0 goes back "
|
|
"to pipeline-cache.bin)", PB.on ? "on" : "off (cannot open the file)", PB.path, PB.pipes, PB.blobs,
|
|
PB.file_size / 1048576.0);
|
|
}
|
|
|
|
// The payload of the record of (type, key), malloc'd (*size bytes), or NULL. Caller holds PB.lock.
|
|
static uint8_t *pb_read(uint32_t type, const uint8_t *key, uint32_t ks, uint32_t *size) {
|
|
PbSlot *s = pb_find(pb_hash(type, key, ks));
|
|
if (!s || s->size < PB_HEAD) return NULL;
|
|
uint32_t n = s->size - PB_HEAD;
|
|
uint8_t *head = malloc(PB_HEAD + (size_t)n), *out = NULL;
|
|
if (!head) return NULL;
|
|
int ok;
|
|
if (s->off >= PB.file_size) { // still pending, at its place in pend
|
|
size_t at = (size_t)(s->off - PB.file_size) + 4;
|
|
ok = at + s->size <= PB.pend_n;
|
|
if (ok) memcpy(head, PB.pend + at, s->size);
|
|
} else {
|
|
ok = !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1;
|
|
}
|
|
uint32_t t, k;
|
|
if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4);
|
|
if (ok && t == type && k == ks && !memcmp(head + 8, key, ks)) { // the key itself, not only its hash
|
|
memmove(head, head + PB_HEAD, n);
|
|
out = head, *size = n;
|
|
} else {
|
|
free(head);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// The binaries stored for pipeline key k, created in bins (*n of them). 0 when none are stored or they fail.
|
|
static int pb_load(const VkPipelineBinaryKeyKHR *k, VkPipelineBinaryKHR bins[PB_MAX_BIN], uint32_t *n) {
|
|
VkPipelineBinaryKeyKHR keys[PB_MAX_BIN];
|
|
VkPipelineBinaryDataKHR data[PB_MAX_BIN];
|
|
uint8_t *blob[PB_MAX_BIN] = {0};
|
|
uint32_t nb = 0, size = 0;
|
|
plat_mutex_lock(&PB.lock);
|
|
uint8_t *pipe = pb_read(PB_PIPE, k->key, k->keySize, &size);
|
|
int ok = pipe && size >= 4;
|
|
if (ok) memcpy(&nb, pipe, 4);
|
|
if (!nb || nb > PB_MAX_BIN || size < 4 + nb * (4 + PB_KEY)) ok = 0;
|
|
for (uint32_t i = 0; ok && i < nb; i++) {
|
|
const uint8_t *e = pipe + 4 + i * (4 + PB_KEY);
|
|
uint32_t ks, ds = 0;
|
|
memcpy(&ks, e, 4);
|
|
keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR, .keySize = ks};
|
|
memcpy(keys[i].key, e + 4, PB_KEY);
|
|
if (ks > PB_KEY || !(blob[i] = pb_read(PB_BLOB, keys[i].key, ks, &ds))) ok = 0;
|
|
else data[i] = (VkPipelineBinaryDataKHR){.dataSize = ds, .pData = blob[i]};
|
|
}
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (ok) {
|
|
VkPipelineBinaryKeysAndDataKHR kd = {.binaryCount = nb, .pPipelineBinaryKeys = keys, .pPipelineBinaryData = data};
|
|
VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pKeysAndDataInfo = &kd};
|
|
VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR,
|
|
.pipelineBinaryCount = nb, .pPipelineBinaries = bins};
|
|
ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS;
|
|
*n = nb;
|
|
}
|
|
for (uint32_t i = 0; i < PB_MAX_BIN; i++) free(blob[i]);
|
|
free(pipe);
|
|
return ok;
|
|
}
|
|
|
|
// Appends one record to pend and the index. Caller holds PB.lock.
|
|
static int pb_put(uint32_t type, const uint8_t *key, uint32_t ks, const void *a, size_t an, const void *b, size_t bn) {
|
|
size_t size = PB_HEAD + an + bn;
|
|
if (size >= (1u << 31)) return 0;
|
|
if (PB.pend_n + 4 + size > PB.pend_cap) {
|
|
size_t cap = PB.pend_cap ? PB.pend_cap * 2 : (size_t)4 << 20;
|
|
while (cap < PB.pend_n + 4 + size) cap *= 2;
|
|
uint8_t *t = realloc(PB.pend, cap);
|
|
if (!t) return 0;
|
|
PB.pend = t, PB.pend_cap = cap;
|
|
}
|
|
if (!pb_insert(pb_hash(type, key, ks), PB.file_size + PB.pend_n, (uint32_t)size)) return 0;
|
|
uint8_t *o = PB.pend + PB.pend_n;
|
|
uint32_t s32 = (uint32_t)size;
|
|
memcpy(o, &s32, 4), memcpy(o + 4, &type, 4), memcpy(o + 8, &ks, 4);
|
|
memset(o + 12, 0, PB_KEY), memcpy(o + 12, key, ks);
|
|
if (an) memcpy(o + 4 + PB_HEAD, a, an);
|
|
if (bn) memcpy(o + 4 + PB_HEAD + an, b, bn);
|
|
PB.pend_n += 4 + size;
|
|
if (type == PB_PIPE) PB.pipes++;
|
|
else PB.blobs++;
|
|
return 1;
|
|
}
|
|
|
|
static void pb_flush(void);
|
|
|
|
// Stores the binaries of pipeline p (created with its data captured) under pipeline key k, then frees the data. A
|
|
// binary whose key is stored already is not stored again.
|
|
static void pb_capture(VkPipeline p, const VkPipelineBinaryKeyKHR *k) {
|
|
VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pipeline = p};
|
|
VkPipelineBinaryKHR bins[PB_MAX_BIN];
|
|
VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR};
|
|
uint32_t nb = 0;
|
|
int ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS && hi.pipelineBinaryCount &&
|
|
hi.pipelineBinaryCount <= PB_MAX_BIN;
|
|
if (ok) {
|
|
hi.pPipelineBinaries = bins;
|
|
ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS;
|
|
nb = ok ? hi.pipelineBinaryCount : 0;
|
|
}
|
|
VkPipelineBinaryKeyKHR keys[PB_MAX_BIN];
|
|
uint8_t list[4 + PB_MAX_BIN * (4 + PB_KEY)] = {0};
|
|
memcpy(list, &nb, 4);
|
|
for (uint32_t i = 0; ok && i < nb; i++) { // each binary's key, and its data unless stored already
|
|
VkPipelineBinaryDataInfoKHR di = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_DATA_INFO_KHR, .pipelineBinary = bins[i]};
|
|
keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
|
size_t ds = 0;
|
|
ok = PB.data(P.device, &di, &keys[i], &ds, NULL) == VK_SUCCESS && keys[i].keySize && keys[i].keySize <= PB_KEY;
|
|
if (!ok) break;
|
|
memcpy(list + 4 + i * (4 + PB_KEY), &keys[i].keySize, 4), memcpy(list + 8 + i * (4 + PB_KEY), keys[i].key, keys[i].keySize);
|
|
plat_mutex_lock(&PB.lock);
|
|
int have = pb_find(pb_hash(PB_BLOB, keys[i].key, keys[i].keySize)) != NULL;
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (have) {
|
|
PB.blob_shared++;
|
|
continue;
|
|
}
|
|
uint8_t *d = malloc(ds ? ds : 1);
|
|
ok = d && PB.data(P.device, &di, &keys[i], &ds, d) == VK_SUCCESS;
|
|
if (ok) {
|
|
plat_mutex_lock(&PB.lock);
|
|
if (pb_put(PB_BLOB, keys[i].key, keys[i].keySize, d, ds, NULL, 0)) PB.blob_new++;
|
|
plat_mutex_unlock(&PB.lock); // a 0 from pb_put: another thread stored the same binary meanwhile
|
|
}
|
|
free(d);
|
|
}
|
|
for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL);
|
|
VkReleaseCapturedPipelineDataInfoKHR ri = {.sType = VK_STRUCTURE_TYPE_RELEASE_CAPTURED_PIPELINE_DATA_INFO_KHR, .pipeline = p};
|
|
PB.release(P.device, &ri, NULL);
|
|
plat_mutex_lock(&PB.lock);
|
|
if (ok && pb_put(PB_PIPE, k->key, k->keySize, list, 4 + nb * (4 + PB_KEY), NULL, 0)) PB.captured++;
|
|
else if (!ok) PB.capture_fail++;
|
|
int full = PB.pend_n >= ((size_t)64 << 20);
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (full) pb_flush(); // a cold boot captures thousands before the autosave thread runs: keep the buffer small
|
|
}
|
|
|
|
// Appends the pending records (the autosave thread every 2 s, and the save at exit).
|
|
static void pb_flush(void) {
|
|
if (!PB.on) return;
|
|
plat_mutex_lock(&PB.lock);
|
|
if (PB.pend_n) {
|
|
FILE *f = fopen(PB.path, "ab");
|
|
int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1;
|
|
if (f) ok = !fclose(f) && ok;
|
|
if (ok) PB.file_size += PB.pend_n;
|
|
else rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory", PB.path);
|
|
if (ok) PB.pend_n = 0;
|
|
}
|
|
plat_mutex_unlock(&PB.lock);
|
|
}
|
|
|
|
static void pb_log(const char *when) {
|
|
if (!PB.on) return;
|
|
plat_mutex_lock(&PB.lock);
|
|
rt_log("vk_pipeline: pipeline binaries %s: %llu made from pipeline-binaries.bin (avg %.2f ms), %llu compiled (avg %.1f "
|
|
"ms), %llu captured, %llu could not be stored, %llu stored ones failed; binaries %llu new, %llu shared; %u pipelines and "
|
|
"%u binaries in the file, %.1f MB", when,
|
|
(unsigned long long)PB.from_file, PB.from_file ? PB.from_ms / PB.from_file : 0.0,
|
|
(unsigned long long)PB.compiled, PB.compiled ? PB.compile_ms / PB.compiled : 0.0,
|
|
(unsigned long long)PB.captured, (unsigned long long)PB.capture_fail, (unsigned long long)PB.load_fail,
|
|
(unsigned long long)PB.blob_new, (unsigned long long)PB.blob_shared, PB.pipes, PB.blobs,
|
|
(PB.file_size + PB.pend_n) / 1048576.0);
|
|
plat_mutex_unlock(&PB.lock);
|
|
}
|
|
|
|
// Creates one pipeline: kind 0 a graphics pipeline, 1 a graphics pipeline library, 2 a compute pipeline; ci its
|
|
// Vk*PipelineCreateInfo. With binaries on: from the stored binaries of its pipeline key when there are any, else
|
|
// compiled with P.cache and its binaries captured. The create info is restored before returning.
|
|
typedef struct { VkStructureType sType; const void *pNext; VkPipelineCreateFlags flags; } PipeCiHead;
|
|
|
|
// Builds into P.cache against reads of its data. Every create with P.cache holds this shared (pipe_create_in); the
|
|
// copy of the data in cache_save holds it exclusively and only takes it when no build is running. The Vulkan rules
|
|
// allow reading a cache created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT while pipelines are
|
|
// built into it, but Intel's Windows driver (101.8991 on an Arc 140V) crashed inside vkGetPipelineCacheData during the
|
|
// copy, reading through a null entry, with compute and library builds running beside the autosave.
|
|
static PlatMutex cache_rw = PLAT_MUTEX_INIT;
|
|
|
|
// A pipeline cache of this thread's own that its creates build into in place of P.cache; VK_NULL_HANDLE (every thread
|
|
// but the background builder of large compute shaders) for P.cache itself. A create into it does not hold cache_rw, so
|
|
// a build of minutes does not keep pipeline-cache.bin from being saved; bg_merge moves its content into P.cache.
|
|
static __thread VkPipelineCache own_cache;
|
|
// New content in P.cache that P.stats does not count, for autosave_once to save: bg_merge's merges, and the compiles
|
|
// of pipelines built for the cache only (lb_compute keep 0).
|
|
static volatile uint32_t cache_merges;
|
|
|
|
static VkResult pipe_create_in(int kind, void *ci, VkPipeline *pipe, VkPipelineCache cache) {
|
|
int shared = cache && !own_cache;
|
|
if (cache && own_cache) cache = own_cache;
|
|
if (shared) plat_mutex_lock_shared(&cache_rw);
|
|
VkResult r = kind == 2 ? vkCreateComputePipelines(P.device, cache, 1, ci, NULL, pipe)
|
|
: vkCreateGraphicsPipelines(P.device, cache, 1, ci, NULL, pipe);
|
|
if (shared) plat_mutex_unlock_shared(&cache_rw);
|
|
return r;
|
|
}
|
|
|
|
static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) {
|
|
PipeCiHead *hd = ci;
|
|
VkResult r;
|
|
pb_last = 0;
|
|
#define PIPE_CREATE(cache) pipe_create_in(kind, ci, pipe, cache)
|
|
if (!PB.on) return PIPE_CREATE(P.cache);
|
|
VkPipelineCreateInfoKHR kci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_INFO_KHR, .pNext = ci};
|
|
VkPipelineBinaryKeyKHR key = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
|
int keyed = PB.key(P.device, &kci, &key) == VK_SUCCESS && key.keySize && key.keySize <= PB_KEY;
|
|
const void *next = hd->pNext;
|
|
if (keyed) {
|
|
VkPipelineBinaryKHR bins[PB_MAX_BIN];
|
|
uint32_t nb = 0;
|
|
uint64_t t0 = plat_now_ns();
|
|
if (pb_load(&key, bins, &nb)) {
|
|
VkPipelineBinaryInfoKHR bi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_INFO_KHR, .pNext = next,
|
|
.binaryCount = nb, .pPipelineBinaries = bins};
|
|
hd->pNext = &bi;
|
|
r = PIPE_CREATE(VK_NULL_HANDLE); // a pipeline made from binaries takes no pipeline cache
|
|
hd->pNext = next;
|
|
for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL);
|
|
plat_mutex_lock(&PB.lock);
|
|
if (r == VK_SUCCESS) PB.from_file++, PB.from_ms += (plat_now_ns() - t0) / 1e6;
|
|
else PB.load_fail++;
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (r == VK_SUCCESS) return pb_last = 1, r;
|
|
}
|
|
}
|
|
int capture = keyed && !PB.no_capture[kind];
|
|
VkPipelineCreateFlags2CreateInfoKHR f2 = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR, .pNext = next,
|
|
.flags = (VkPipelineCreateFlags2KHR)hd->flags | VK_PIPELINE_CREATE_2_CAPTURE_DATA_BIT_KHR};
|
|
uint64_t t0 = plat_now_ns();
|
|
if (capture) hd->pNext = &f2;
|
|
r = PIPE_CREATE(P.cache);
|
|
hd->pNext = next;
|
|
if (capture && r < 0) { // the driver refuses the capture for this kind: create without it from now on
|
|
PB.no_capture[kind] = 1;
|
|
rt_log("vk_pipeline: pipeline binaries: a %s with its data captured failed (%d); such pipelines are not stored",
|
|
kind == 2 ? "compute pipeline" : kind ? "pipeline library" : "graphics pipeline", (int)r);
|
|
capture = 0;
|
|
r = PIPE_CREATE(P.cache);
|
|
}
|
|
#undef PIPE_CREATE
|
|
if (r == VK_SUCCESS && *pipe) {
|
|
plat_mutex_lock(&PB.lock);
|
|
PB.compiled++, PB.compile_ms += (plat_now_ns() - t0) / 1e6;
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (capture) pb_capture(*pipe, &key);
|
|
}
|
|
return r;
|
|
}
|
|
|
|
// Cache data whose header names another device or driver is dropped before Vulkan sees it.
|
|
static void cache_load(VkCtx *c) {
|
|
cache_path_init();
|
|
pb_init(c);
|
|
if (PB.on) { // the binaries carry compiled pipelines across runs; P.cache stays VK_NULL_HANDLE
|
|
rt_log("vk_pipeline: %s is not read or written while pipeline binaries are on", P.cache_path);
|
|
return;
|
|
}
|
|
void *data = NULL;
|
|
size_t size = 0;
|
|
FILE *f = P.cache_path[0] ? fopen(P.cache_path, "rb") : NULL;
|
|
if (f) {
|
|
fseek(f, 0, SEEK_END);
|
|
long len = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
if (len > (long)sizeof(VkPipelineCacheHeaderVersionOne) && (data = malloc((size_t)len)) &&
|
|
fread(data, 1, (size_t)len, f) == (size_t)len)
|
|
size = (size_t)len;
|
|
fclose(f);
|
|
VkPipelineCacheHeaderVersionOne h;
|
|
if (size) memcpy(&h, data, sizeof h);
|
|
if (size && (h.headerVersion != VK_PIPELINE_CACHE_HEADER_VERSION_ONE || h.vendorID != c->props.vendorID ||
|
|
h.deviceID != c->props.deviceID || memcmp(h.pipelineCacheUUID, c->props.pipelineCacheUUID, VK_UUID_SIZE))) {
|
|
rt_log("vk_pipeline: %s was written by another device or driver; starting empty", P.cache_path);
|
|
size = 0;
|
|
}
|
|
// NX2NAVI_PIPELINE_CACHE_MAX_MB (default 512; 0 no limit): a larger file starts the run empty, and the run's
|
|
// saves replace it. The file only grows: entries of earlier translators and builds stay, and on NVIDIA
|
|
// (621.01) an entry another executable wrote is neither used nor written again, so a renamed or updated exe
|
|
// compiles while the file keeps its size (2026-11-09: 650 MB, 15-18 % of the pipelines read from it). The
|
|
// whole file sits in process memory twice at the boot (this read and the driver's copy) and is copied out
|
|
// at every save, which a 16 GB machine near its commit limit cannot spare.
|
|
uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_MAX_MB"), NULL, 10) << 20;
|
|
if (size && cap && size > cap) {
|
|
rt_log("vk_pipeline: %s holds %.0f MB, over NX2NAVI_PIPELINE_CACHE_MAX_MB %llu: starting empty; this run "
|
|
"rewrites it", P.cache_path, size / 1048576.0, (unsigned long long)(cap >> 20));
|
|
size = 0;
|
|
}
|
|
}
|
|
VkPipelineCacheCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, .initialDataSize = size,
|
|
.pInitialData = size ? data : NULL};
|
|
if (vkCreatePipelineCache(c->device, &ci, NULL, &P.cache) != VK_SUCCESS) {
|
|
ci.initialDataSize = 0, ci.pInitialData = NULL;
|
|
if (vkCreatePipelineCache(c->device, &ci, NULL, &P.cache) != VK_SUCCESS) P.cache = VK_NULL_HANDLE;
|
|
}
|
|
free(data);
|
|
// What the driver kept of the file: a driver may drop initial data it does not accept without failing the create,
|
|
// which leaves every build of the run compiling while the cache file looks valid.
|
|
size_t kept = 0;
|
|
if (P.cache && vkGetPipelineCacheData(c->device, P.cache, &kept, NULL) != VK_SUCCESS) kept = 0;
|
|
P.cache_loaded = kept;
|
|
if (size)
|
|
rt_log("vk_pipeline: %s: %.1f MB read, the driver holds %.1f MB of it", P.cache_path, size / 1048576.0,
|
|
kept / 1048576.0);
|
|
}
|
|
|
|
// The float widths spirv_nan_preserve declares on this device (P.nan_preserve), from NX2NAVI_VK_NAN_PRESERVE: 0 off,
|
|
// 1 on every driver; unset, on every driver except NVIDIA's own, on which the game draws right without it. Only widths
|
|
// the driver can preserve (VkPhysicalDeviceFloatControlsProperties) are declared, and nothing without 32-bit floats.
|
|
static void nan_preserve_init(VkCtx *c) {
|
|
VkPhysicalDeviceDriverProperties drv = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES};
|
|
VkPhysicalDeviceFloatControlsProperties fc = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES,
|
|
.pNext = &drv};
|
|
VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &fc};
|
|
vkGetPhysicalDeviceProperties2(c->phys, &p2);
|
|
const char *e = cfg_get("NX2NAVI_VK_NAN_PRESERVE"); // no fixed default: it depends on the driver, see above
|
|
int set = e && *e, want = set ? strcmp(e, "0") != 0 : drv.driverID != VK_DRIVER_ID_NVIDIA_PROPRIETARY;
|
|
uint32_t can = (fc.shaderSignedZeroInfNanPreserveFloat16 ? NP_16 : 0) |
|
|
(fc.shaderSignedZeroInfNanPreserveFloat32 ? NP_32 : 0) |
|
|
(fc.shaderSignedZeroInfNanPreserveFloat64 ? NP_64 : 0);
|
|
P.nan_preserve = want && (can & NP_32) ? can : 0;
|
|
if (P.nan_preserve)
|
|
rt_log("vk_pipeline: game shaders keep NaN, infinity and signed zero (SignedZeroInfNanPreserve for %s32%s-bit "
|
|
"floats%s)", can & NP_16 ? "16, " : "", can & NP_64 ? ", 64" : "", set ? ", NX2NAVI_VK_NAN_PRESERVE=1" : "");
|
|
else
|
|
rt_log("vk_pipeline: game shaders without SignedZeroInfNanPreserve (%s)",
|
|
!want ? (set ? "NX2NAVI_VK_NAN_PRESERVE=0" : "NVIDIA's driver") : "the driver cannot keep them in 32-bit floats");
|
|
}
|
|
|
|
static void keys_open(void);
|
|
static void cu_open(void);
|
|
static void prebuild_start(void);
|
|
|
|
static int ensure_init(void) {
|
|
if (P.ready) return 1;
|
|
if (!vk_init()) return 0;
|
|
plat_mutex_lock(&P.lock);
|
|
if (P.ready) {
|
|
plat_mutex_unlock(&P.lock);
|
|
return 1;
|
|
}
|
|
VkCtx *c = vk_ctx();
|
|
P.device = c->device;
|
|
VkPhysicalDevicePushDescriptorPropertiesKHR pd = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR};
|
|
VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &pd};
|
|
vkGetPhysicalDeviceProperties2(c->phys, &p2);
|
|
P.max_push = pd.maxPushDescriptors;
|
|
P.indep = vk_caps.gpl;
|
|
P.gpl = vk_caps.gpl;
|
|
rt_log(P.gpl ? "vk_pipeline: graphics pipeline library on: a draw whose pipeline is building gets a fast-linked one, "
|
|
"the optimised build replaces it when done"
|
|
: "vk_pipeline: graphics pipeline library off: draws wait for or skip pipelines that are building");
|
|
const char *ps = cfg_get("NX2NAVI_PUSH_STAGE"), *vo = cfg_get("NX2NAVI_VS_OUTS");
|
|
P.push_stage = ps && !strcmp(ps, "fs") ? GFX_STAGE_FS : GFX_STAGE_VS;
|
|
P.vs_outs_all = vo && !strcmp(vo, "all");
|
|
if (P.indep) rt_log("vk_pipeline: stage-independent layouts: vertex set 0, fragment set 1, %s set pushed; vertex outputs: %s",
|
|
P.push_stage == GFX_STAGE_VS ? "vertex" : "fragment", P.vs_outs_all ? "all 32 locations" : "union of paired fragment inputs");
|
|
// vk_device.c enables every feature the GPU reports in this chain, so the query tells what is enabled.
|
|
VkPhysicalDeviceExtendedDynamicState3FeaturesEXT e3 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_3_FEATURES_EXT};
|
|
VkPhysicalDeviceProvokingVertexFeaturesEXT pvf = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_FEATURES_EXT,
|
|
.pNext = c->has_eds3 ? &e3 : NULL};
|
|
VkPhysicalDeviceVulkan13Features v13 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES,
|
|
.pNext = c->has_provoking_vertex ? (void *)&pvf : (c->has_eds3 ? (void *)&e3 : NULL)};
|
|
VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &v13};
|
|
vkGetPhysicalDeviceFeatures2(c->phys, &f2);
|
|
P.pv_last = c->has_provoking_vertex && pvf.provokingVertexLast;
|
|
if (!P.pv_last) rt_log("vk_pipeline: provokingVertexLast unavailable: flat outputs come from the first vertex (NVN: last)");
|
|
// Maxwell warps have 32 lanes; the translator assumes it (maxwell.md 3.4). Pin 32 where the driver allows it.
|
|
if (v13.subgroupSizeControl) {
|
|
VkPhysicalDeviceVulkan13Properties p13 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES};
|
|
VkPhysicalDeviceProperties2 q2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &p13};
|
|
vkGetPhysicalDeviceProperties2(c->phys, &q2);
|
|
P.sg32_stages = vk_caps.sg_pin_stages; // vk_caps.c: 32 unless NX2NAVI_VK_SUBGROUP; 0 without the pin
|
|
rt_log("vk_pipeline: subgroup size %u-%u, requiredSubgroupSizeStages %#x, pinning %u on %#x", p13.minSubgroupSize,
|
|
p13.maxSubgroupSize, (unsigned)p13.requiredSubgroupSizeStages, vk_caps.sg_size, (unsigned)P.sg32_stages);
|
|
}
|
|
// A warp reconverges where the guest compiler put it back together; Vulkan guarantees that only with maximal
|
|
// reconvergence (spirv_reconverge). NX2NAVI_VK_RECONVERGE=0 leaves the modules as translated.
|
|
P.reconverge = c->has_maximal_reconvergence && rt_knob("VK_RECONVERGE", "1") == 1;
|
|
rt_log("vk_pipeline: maximal reconvergence for shaders with subgroup ops: %s",
|
|
P.reconverge ? "on" : c->has_maximal_reconvergence ? "off (NX2NAVI_VK_RECONVERGE=0)" : "not supported by the driver");
|
|
nan_preserve_init(c);
|
|
P.f_full_subgroups = v13.computeFullSubgroups;
|
|
P.cache_control = v13.pipelineCreationCacheControl;
|
|
// Pipeline creation feedback is core in Vulkan 1.3; the driver reports per pipeline whether P.cache held it.
|
|
P.feedback = c->props.apiVersion >= VK_API_VERSION_1_3 ? atoi(cfg_value("NX2NAVI_PIPE_FEEDBACK")) : 0;
|
|
P.f_logic_op = f2.features.logicOp;
|
|
P.f_depth_clamp = f2.features.depthClamp;
|
|
P.f_fill_non_solid = f2.features.fillModeNonSolid;
|
|
P.f_dual_src = f2.features.dualSrcBlend;
|
|
#define PROC(field, name) (P.field = (void *)vk_dpa(c->device, name))
|
|
PROC(push, "vkCmdPushDescriptorSetKHR");
|
|
// Descriptor update templates (core 1.1; the push variant comes with VK_KHR_push_descriptor on 1.1): the driver
|
|
// walks a layout prepared when the layout is made instead of parsing write structs every draw.
|
|
// NX2NAVI_DESC_TEMPLATE=0 keeps the write structs.
|
|
if (!cfg_bool("NX2NAVI_DESC_TEMPLATE")) {
|
|
rt_log("vk_pipeline: descriptor update templates off (NX2NAVI_DESC_TEMPLATE=0)");
|
|
} else if (c->props.apiVersion >= VK_API_VERSION_1_1 && PROC(tpl_create, "vkCreateDescriptorUpdateTemplate") &&
|
|
PROC(tpl_destroy, "vkDestroyDescriptorUpdateTemplate") && PROC(tpl_update, "vkUpdateDescriptorSetWithTemplate")) {
|
|
P.tpl = 1;
|
|
if (!PROC(tpl_push, "vkCmdPushDescriptorSetWithTemplateKHR"))
|
|
rt_log("vk_pipeline: vkCmdPushDescriptorSetWithTemplateKHR missing: push sets are written with write structs");
|
|
} else {
|
|
rt_log("vk_pipeline: descriptor update templates unavailable: sets are written with write structs");
|
|
}
|
|
// NX2NAVI_VK_DYN_VERTEX=0/1 forces dynamic vertex input off or on. Default off on two drivers:
|
|
// - Intel's frozen Windows driver for its older integrated GPUs (UHD 630, Ice Lake: Vulkan 1.3.215 and older), which
|
|
// draws nothing with it.
|
|
// - AMD's driver for RDNA 3 and newer (vk_caps.amd_llpc), which compiles a pipeline with dynamic vertex input again
|
|
// at every boot: neither pipeline-cache.bin nor the driver's own disk cache gives it back. Measured on an
|
|
// RX 7900 XT (26.10.1) with NX2NAVI_PIPE_FEEDBACK=2, booting one save twice from an empty cache: the second boot
|
|
// compiled all 622 recorded non-mesh pipelines again at 83 ms each, with the module and state hashes of the first
|
|
// boot (the same with NX2NAVI_GPL=0, so it is not the pipeline libraries or their layouts), while the mesh
|
|
// pipelines, which have no vertex input, came from the file. With the vertex layout in the pipeline the second
|
|
// boot built 787 pipelines in 170 ms and the third 898 of 898 from the file in 233 ms, with no build left for the
|
|
// scene. A player's file of 3,979 records used the whole boot budget (PREWARM_PIPE_MS, 20 s) at every boot; with
|
|
// the vertex layout in the pipeline its third boot built 4,273 pipelines in 0.9 s. The price is more pipelines,
|
|
// one per vertex layout (787 records against 692 for that boot); records written with dynamic vertex input still
|
|
// prebuild, each for the layout of the draw that recorded it. The RDNA 1 and 2 branch (vk_caps.amd_sc) is not
|
|
// measured and keeps dynamic vertex input.
|
|
const char *dv = cfg_get("NX2NAVI_VK_DYN_VERTEX"); // no fixed default: it depends on the GPU, see above
|
|
int intel_legacy = c->props.vendorID == 0x8086 && c->props.apiVersion <= VK_MAKE_API_VERSION(0, 1, 3, 215);
|
|
int dv_on = dv && *dv ? strcmp(dv, "0") != 0 : !intel_legacy && !vk_caps.amd_llpc;
|
|
P.dyn_vi = dv_on && c->has_vertex_input_dynamic_state && PROC(set_vi, "vkCmdSetVertexInputEXT");
|
|
if (!dv_on && c->has_vertex_input_dynamic_state)
|
|
rt_log("vk_pipeline: dynamic vertex input off (%s)",
|
|
dv && *dv ? "NX2NAVI_VK_DYN_VERTEX=0"
|
|
: intel_legacy ? "Intel's older Windows driver draws nothing with it"
|
|
: "AMD's driver compiles pipelines that have it again at every boot");
|
|
if (c->has_eds3) {
|
|
P.dyn_clamp = e3.extendedDynamicState3DepthClampEnable && P.f_depth_clamp && PROC(set_clamp, "vkCmdSetDepthClampEnableEXT");
|
|
P.dyn_poly = e3.extendedDynamicState3PolygonMode && PROC(set_poly, "vkCmdSetPolygonModeEXT");
|
|
P.dyn_blend_en = e3.extendedDynamicState3ColorBlendEnable && PROC(set_blend_en, "vkCmdSetColorBlendEnableEXT");
|
|
P.dyn_blend_eq = e3.extendedDynamicState3ColorBlendEquation && PROC(set_blend_eq, "vkCmdSetColorBlendEquationEXT");
|
|
P.dyn_wmask = e3.extendedDynamicState3ColorWriteMask && PROC(set_wmask, "vkCmdSetColorWriteMaskEXT");
|
|
P.dyn_a2c = e3.extendedDynamicState3AlphaToCoverageEnable && PROC(set_a2c, "vkCmdSetAlphaToCoverageEnableEXT");
|
|
}
|
|
// NX2NAVI_VK_DYN_A2C=0/1 forces dynamic alpha to coverage off or on. Default off on AMD's Windows driver branch for
|
|
// RDNA 2 and older (vk_caps.amd_sc), which applies alpha to coverage while the dynamic state says off: on an RX 590
|
|
// (24.9.1) NX2navi, which never turns it on, drew Link and the NPCs with depth but no colour (black silhouettes) and
|
|
// dithered other surfaces into checkerboards (issue #67, RX 470/570/580). Taken out of the dynamic state, it goes
|
|
// into the pipeline key and the characters draw. With the texture pool marks it stays out of the dynamic state
|
|
// whatever the setting: a marked fragment shader gets early fragment tests only in a pipeline that has alpha to
|
|
// coverage off (spirv_pool_mark), so the pipeline has to know.
|
|
const char *da = cfg_get("NX2NAVI_VK_DYN_A2C"); // no fixed default: it depends on the driver, see above
|
|
int marks = vk_pool_marks_enabled();
|
|
int da_on = !marks && (da && *da ? strcmp(da, "0") != 0 : !vk_caps.amd_sc);
|
|
if (!da_on && P.dyn_a2c) {
|
|
P.dyn_a2c = 0;
|
|
rt_log("vk_pipeline: dynamic alpha to coverage off (%s)", marks ? "the texture pool marks, NX2NAVI_TEX_DISUSE"
|
|
: da && *da ? "NX2NAVI_VK_DYN_A2C=0"
|
|
: "AMD's driver for RDNA 2 and older");
|
|
}
|
|
#undef PROC
|
|
static const VkDynamicState core[] = {
|
|
VK_DYNAMIC_STATE_VIEWPORT_WITH_COUNT, VK_DYNAMIC_STATE_SCISSOR_WITH_COUNT, VK_DYNAMIC_STATE_DEPTH_BIAS,
|
|
VK_DYNAMIC_STATE_BLEND_CONSTANTS, VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK, VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
|
|
VK_DYNAMIC_STATE_STENCIL_REFERENCE, VK_DYNAMIC_STATE_CULL_MODE, VK_DYNAMIC_STATE_FRONT_FACE,
|
|
VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY, VK_DYNAMIC_STATE_DEPTH_TEST_ENABLE, VK_DYNAMIC_STATE_DEPTH_WRITE_ENABLE,
|
|
VK_DYNAMIC_STATE_DEPTH_COMPARE_OP, VK_DYNAMIC_STATE_STENCIL_TEST_ENABLE, VK_DYNAMIC_STATE_STENCIL_OP,
|
|
VK_DYNAMIC_STATE_DEPTH_BIAS_ENABLE, VK_DYNAMIC_STATE_RASTERIZER_DISCARD_ENABLE,
|
|
VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE};
|
|
P.n_dyn = 0;
|
|
for (size_t i = 0; i < sizeof core / sizeof *core; i++) P.dyn[P.n_dyn++] = core[i];
|
|
P.dyn[P.n_dyn++] = P.dyn_vi ? VK_DYNAMIC_STATE_VERTEX_INPUT_EXT : VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE;
|
|
if (P.dyn_clamp) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_DEPTH_CLAMP_ENABLE_EXT;
|
|
if (P.dyn_poly) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_POLYGON_MODE_EXT;
|
|
if (P.dyn_blend_en) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_BLEND_ENABLE_EXT;
|
|
if (P.dyn_blend_eq) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_BLEND_EQUATION_EXT;
|
|
if (P.dyn_wmask) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_WRITE_MASK_EXT;
|
|
if (P.dyn_a2c) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_ALPHA_TO_COVERAGE_ENABLE_EXT;
|
|
vk_mem_stage_log("device creation");
|
|
cache_load(c);
|
|
vk_mem_stage_log("the pipeline cache load");
|
|
rt_log("vk_pipeline: max push descriptors %u; dynamic vertex input %d, depth clamp %d, polygon mode %d, blend enable %d, "
|
|
"blend equation %d, write mask %d, alpha to coverage %d; cache %s",
|
|
P.max_push, P.dyn_vi, P.dyn_clamp, P.dyn_poly, P.dyn_blend_en, P.dyn_blend_eq, P.dyn_wmask, P.dyn_a2c,
|
|
P.cache_path[0] ? P.cache_path : "<memory only>");
|
|
keys_open();
|
|
cu_open();
|
|
plat_xchg32(&P.ready, 1);
|
|
plat_mutex_unlock(&P.lock);
|
|
prebuild_start();
|
|
return 1;
|
|
}
|
|
|
|
// ---- NVN -> Vulkan mappings
|
|
|
|
static VkBlendFactor blend_factor(unsigned f) {
|
|
switch (f) {
|
|
case 1: return VK_BLEND_FACTOR_ZERO;
|
|
case 2: return VK_BLEND_FACTOR_ONE;
|
|
case 3: return VK_BLEND_FACTOR_SRC_COLOR;
|
|
case 4: return VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR;
|
|
case 5: return VK_BLEND_FACTOR_SRC_ALPHA;
|
|
case 6: return VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
|
|
case 7: return VK_BLEND_FACTOR_DST_ALPHA;
|
|
case 8: return VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA;
|
|
case 9: return VK_BLEND_FACTOR_DST_COLOR;
|
|
case 10: return VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR;
|
|
case 11: return VK_BLEND_FACTOR_SRC_ALPHA_SATURATE;
|
|
case 0x10: return VK_BLEND_FACTOR_SRC1_COLOR;
|
|
case 0x11: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR;
|
|
case 0x12: return VK_BLEND_FACTOR_SRC1_ALPHA;
|
|
case 0x13: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA;
|
|
case 0x61: return VK_BLEND_FACTOR_CONSTANT_COLOR;
|
|
case 0x62: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR;
|
|
case 0x63: return VK_BLEND_FACTOR_CONSTANT_ALPHA;
|
|
case 0x64: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA;
|
|
default: LOG_ONCE("vk_pipeline: unknown NVN blend factor %#x, using ONE", f); return VK_BLEND_FACTOR_ONE;
|
|
}
|
|
}
|
|
|
|
static VkBlendOp blend_op(unsigned e) {
|
|
switch (e) {
|
|
case 2: return VK_BLEND_OP_SUBTRACT;
|
|
case 3: return VK_BLEND_OP_REVERSE_SUBTRACT;
|
|
case 4: return VK_BLEND_OP_MIN;
|
|
case 5: return VK_BLEND_OP_MAX;
|
|
default: return VK_BLEND_OP_ADD;
|
|
}
|
|
}
|
|
|
|
// NVN_DEPTH_FUNC_* / NVN_STENCIL_FUNC_* are NEVER = 1 .. ALWAYS = 8 in VkCompareOp order.
|
|
static VkCompareOp compare_op(unsigned f) { return f >= 1 && f <= 8 ? (VkCompareOp)(f - 1) : VK_COMPARE_OP_ALWAYS; }
|
|
// NVN_STENCIL_OP_* are KEEP = 1 .. DECR_WRAP = 8 in VkStencilOp order.
|
|
static VkStencilOp stencil_op(unsigned o) { return o >= 1 && o <= 8 ? (VkStencilOp)(o - 1) : VK_STENCIL_OP_KEEP; }
|
|
// NVN_POLYGON_MODE_POINT/LINE/FILL = 0/1/2, VkPolygonMode FILL/LINE/POINT = 0/1/2.
|
|
static VkPolygonMode polygon_mode(unsigned m) {
|
|
if (!P.f_fill_non_solid || m > 2) return VK_POLYGON_MODE_FILL;
|
|
return (VkPolygonMode)(2 - m);
|
|
}
|
|
|
|
VkPrimitiveTopology vk_pipeline_topology(int prim) {
|
|
switch (prim) {
|
|
case NVN_DRAW_PRIMITIVE_POINTS: return VK_PRIMITIVE_TOPOLOGY_POINT_LIST;
|
|
case NVN_DRAW_PRIMITIVE_LINES: return VK_PRIMITIVE_TOPOLOGY_LINE_LIST;
|
|
case NVN_DRAW_PRIMITIVE_LINE_LOOP:
|
|
case NVN_DRAW_PRIMITIVE_LINE_STRIP: return VK_PRIMITIVE_TOPOLOGY_LINE_STRIP;
|
|
case NVN_DRAW_PRIMITIVE_TRIANGLE_STRIP:
|
|
case NVN_DRAW_PRIMITIVE_QUAD_STRIP: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; // same triangles, same coverage
|
|
case NVN_DRAW_PRIMITIVE_TRIANGLE_FAN:
|
|
case NVN_DRAW_PRIMITIVE_POLYGON: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN;
|
|
case NVN_DRAW_PRIMITIVE_LINES_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY;
|
|
case NVN_DRAW_PRIMITIVE_LINE_STRIP_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY;
|
|
case NVN_DRAW_PRIMITIVE_TRIANGLES_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY;
|
|
case NVN_DRAW_PRIMITIVE_TRIANGLE_STRIP_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY;
|
|
case NVN_DRAW_PRIMITIVE_PATCHES: return VK_PRIMITIVE_TOPOLOGY_PATCH_LIST;
|
|
default: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; // TRIANGLES; QUADS after the replay's index rewrite
|
|
}
|
|
}
|
|
|
|
// Topology class (a pipeline's static topology only fixes the class): 0 point, 1 line, 2 triangle, 3 patch.
|
|
static int topology_class(VkPrimitiveTopology t) {
|
|
switch (t) {
|
|
case VK_PRIMITIVE_TOPOLOGY_POINT_LIST: return 0;
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_LIST:
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP:
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY:
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY: return 1;
|
|
case VK_PRIMITIVE_TOPOLOGY_PATCH_LIST: return 3;
|
|
default: return 2;
|
|
}
|
|
}
|
|
|
|
int vk_pipeline_tex_type(int target) {
|
|
switch (target) {
|
|
case NVN_TEXTURE_TARGET_1D: return GFX_TEX_1D;
|
|
case NVN_TEXTURE_TARGET_3D: return GFX_TEX_3D;
|
|
case NVN_TEXTURE_TARGET_1D_ARRAY: return GFX_TEX_1D_ARRAY;
|
|
case NVN_TEXTURE_TARGET_2D_ARRAY: return GFX_TEX_2D_ARRAY;
|
|
case NVN_TEXTURE_TARGET_2D_MULTISAMPLE: return GFX_TEX_2D_MS;
|
|
case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY: return GFX_TEX_2D_MS_ARRAY;
|
|
case NVN_TEXTURE_TARGET_CUBEMAP: return GFX_TEX_CUBE;
|
|
case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: return GFX_TEX_CUBE_ARRAY;
|
|
case NVN_TEXTURE_TARGET_BUFFER: return GFX_TEX_BUFFER;
|
|
default: return GFX_TEX_2D; // 2D, RECTANGLE
|
|
}
|
|
}
|
|
|
|
// Numeric type of a colour format as a shader sees it: GFX_TYPE_*.
|
|
static int format_type(VkFormat f) {
|
|
switch (f) {
|
|
case VK_FORMAT_R8_UINT: case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8B8_UINT: case VK_FORMAT_R8G8B8A8_UINT:
|
|
case VK_FORMAT_R16_UINT: case VK_FORMAT_R16G16_UINT: case VK_FORMAT_R16G16B16_UINT: case VK_FORMAT_R16G16B16A16_UINT:
|
|
case VK_FORMAT_R32_UINT: case VK_FORMAT_R32G32_UINT: case VK_FORMAT_R32G32B32_UINT: case VK_FORMAT_R32G32B32A32_UINT:
|
|
case VK_FORMAT_A2B10G10R10_UINT_PACK32: case VK_FORMAT_B8G8R8A8_UINT: case VK_FORMAT_S8_UINT:
|
|
return GFX_TYPE_UINT;
|
|
case VK_FORMAT_R8_SINT: case VK_FORMAT_R8G8_SINT: case VK_FORMAT_R8G8B8_SINT: case VK_FORMAT_R8G8B8A8_SINT:
|
|
case VK_FORMAT_R16_SINT: case VK_FORMAT_R16G16_SINT: case VK_FORMAT_R16G16B16_SINT: case VK_FORMAT_R16G16B16A16_SINT:
|
|
case VK_FORMAT_R32_SINT: case VK_FORMAT_R32G32_SINT: case VK_FORMAT_R32G32B32_SINT: case VK_FORMAT_R32G32B32A32_SINT:
|
|
case VK_FORMAT_A2B10G10R10_SINT_PACK32: case VK_FORMAT_B8G8R8A8_SINT:
|
|
return GFX_TYPE_SINT;
|
|
default: return GFX_TYPE_FLOAT;
|
|
}
|
|
}
|
|
|
|
// Optimal-tiling (o = 1) or buffer (o = 0) features of a core format, cached.
|
|
static VkFormatFeatureFlags format_features(VkFormat f, int optimal) {
|
|
static VkFormatFeatureFlags known[2][256];
|
|
static uint8_t have[2][256];
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return 0;
|
|
if ((unsigned)f < 256 && have[optimal][f]) return known[optimal][f];
|
|
VkFormatProperties p;
|
|
vkGetPhysicalDeviceFormatProperties(c->phys, f, &p);
|
|
VkFormatFeatureFlags r = optimal ? p.optimalTilingFeatures : p.bufferFeatures;
|
|
if ((unsigned)f < 256) known[optimal][f] = r, have[optimal][f] = 1;
|
|
return r;
|
|
}
|
|
|
|
// Vertex attribute formats by NVNformat.
|
|
typedef struct { VkFormat vk; uint8_t type; } VtxFmt;
|
|
#define F(v) {VK_FORMAT_##v, GFX_TYPE_FLOAT}
|
|
#define U(v) {VK_FORMAT_##v, GFX_TYPE_UINT}
|
|
#define I(v) {VK_FORMAT_##v, GFX_TYPE_SINT}
|
|
static const VtxFmt vtx_formats[0x9B] = {
|
|
[0x01] = F(R8_UNORM), [0x02] = F(R8_SNORM), [0x03] = U(R8_UINT), [0x04] = I(R8_SINT), [0x05] = F(R16_SFLOAT),
|
|
[0x06] = F(R16_UNORM), [0x07] = F(R16_SNORM), [0x08] = U(R16_UINT), [0x09] = I(R16_SINT), [0x0A] = F(R32_SFLOAT),
|
|
[0x0B] = U(R32_UINT), [0x0C] = I(R32_SINT), [0x0D] = F(R8G8_UNORM), [0x0E] = F(R8G8_SNORM), [0x0F] = U(R8G8_UINT),
|
|
[0x10] = I(R8G8_SINT), [0x11] = F(R16G16_SFLOAT), [0x12] = F(R16G16_UNORM), [0x13] = F(R16G16_SNORM),
|
|
[0x14] = U(R16G16_UINT), [0x15] = I(R16G16_SINT), [0x16] = F(R32G32_SFLOAT), [0x17] = U(R32G32_UINT),
|
|
[0x18] = I(R32G32_SINT), [0x19] = F(R8G8B8_UNORM), [0x1A] = F(R8G8B8_SNORM), [0x1B] = U(R8G8B8_UINT),
|
|
[0x1C] = I(R8G8B8_SINT), [0x1D] = F(R16G16B16_SFLOAT), [0x1E] = F(R16G16B16_UNORM), [0x1F] = F(R16G16B16_SNORM),
|
|
[0x20] = U(R16G16B16_UINT), [0x21] = I(R16G16B16_SINT), [0x22] = F(R32G32B32_SFLOAT), [0x23] = U(R32G32B32_UINT),
|
|
[0x24] = I(R32G32B32_SINT), [0x25] = F(R8G8B8A8_UNORM), [0x26] = F(R8G8B8A8_SNORM), [0x27] = U(R8G8B8A8_UINT),
|
|
[0x28] = I(R8G8B8A8_SINT), [0x29] = F(R16G16B16A16_SFLOAT), [0x2A] = F(R16G16B16A16_UNORM),
|
|
[0x2B] = F(R16G16B16A16_SNORM), [0x2C] = U(R16G16B16A16_UINT), [0x2D] = I(R16G16B16A16_SINT),
|
|
[0x2E] = F(R32G32B32A32_SFLOAT), [0x2F] = U(R32G32B32A32_UINT), [0x30] = I(R32G32B32A32_SINT),
|
|
[0x3D] = F(A2B10G10R10_UNORM_PACK32), [0x3E] = U(A2B10G10R10_UINT_PACK32), [0x3F] = F(B10G11R11_UFLOAT_PACK32),
|
|
// *_UI2F / *_I2F: integer data read as float; the 8- and 16-bit ones are Vulkan's SCALED formats
|
|
[0x51] = F(R8_USCALED), [0x52] = F(R8_SSCALED), [0x53] = F(R16_USCALED), [0x54] = F(R16_SSCALED),
|
|
[0x57] = F(R8G8_USCALED), [0x58] = F(R8G8_SSCALED), [0x59] = F(R16G16_USCALED), [0x5A] = F(R16G16_SSCALED),
|
|
[0x5D] = F(R8G8B8_USCALED), [0x5E] = F(R8G8B8_SSCALED), [0x5F] = F(R16G16B16_USCALED), [0x60] = F(R16G16B16_SSCALED),
|
|
[0x63] = F(R8G8B8A8_USCALED), [0x64] = F(R8G8B8A8_SSCALED), [0x65] = F(R16G16B16A16_USCALED),
|
|
[0x66] = F(R16G16B16A16_SSCALED),
|
|
// 32-bit ones have no SCALED format: read as integers (values are not converted; logged when used)
|
|
[0x55] = U(R32_UINT), [0x56] = I(R32_SINT), [0x5B] = U(R32G32_UINT), [0x5C] = I(R32G32_SINT),
|
|
[0x61] = U(R32G32B32_UINT), [0x62] = I(R32G32B32_SINT), [0x67] = U(R32G32B32A32_UINT), [0x68] = I(R32G32B32A32_SINT),
|
|
[0x69] = F(A2B10G10R10_SNORM_PACK32), [0x6A] = I(A2B10G10R10_SINT_PACK32), [0x6B] = F(A2B10G10R10_USCALED_PACK32),
|
|
[0x6C] = F(A2B10G10R10_SSCALED_PACK32), [0x9A] = F(B8G8R8A8_UNORM),
|
|
};
|
|
#undef F
|
|
#undef U
|
|
#undef I
|
|
|
|
// 3-component 8/16-bit formats are not vertex formats on every GPU: read 4 components (LAS vertex_format_supported).
|
|
static VkFormat widen(VkFormat f) {
|
|
switch (f) {
|
|
case VK_FORMAT_R8G8B8_UNORM: return VK_FORMAT_R8G8B8A8_UNORM;
|
|
case VK_FORMAT_R8G8B8_SNORM: return VK_FORMAT_R8G8B8A8_SNORM;
|
|
case VK_FORMAT_R8G8B8_UINT: return VK_FORMAT_R8G8B8A8_UINT;
|
|
case VK_FORMAT_R8G8B8_SINT: return VK_FORMAT_R8G8B8A8_SINT;
|
|
case VK_FORMAT_R8G8B8_USCALED: return VK_FORMAT_R8G8B8A8_USCALED;
|
|
case VK_FORMAT_R8G8B8_SSCALED: return VK_FORMAT_R8G8B8A8_SSCALED;
|
|
case VK_FORMAT_R16G16B16_SFLOAT: return VK_FORMAT_R16G16B16A16_SFLOAT;
|
|
case VK_FORMAT_R16G16B16_UNORM: return VK_FORMAT_R16G16B16A16_UNORM;
|
|
case VK_FORMAT_R16G16B16_SNORM: return VK_FORMAT_R16G16B16A16_SNORM;
|
|
case VK_FORMAT_R16G16B16_UINT: return VK_FORMAT_R16G16B16A16_UINT;
|
|
case VK_FORMAT_R16G16B16_SINT: return VK_FORMAT_R16G16B16A16_SINT;
|
|
case VK_FORMAT_R16G16B16_USCALED: return VK_FORMAT_R16G16B16A16_USCALED;
|
|
case VK_FORMAT_R16G16B16_SSCALED: return VK_FORMAT_R16G16B16A16_SSCALED;
|
|
default: return VK_FORMAT_UNDEFINED;
|
|
}
|
|
}
|
|
|
|
VkFormat vk_vertex_format(int nvn_format, int *type) {
|
|
if (type) *type = GFX_TYPE_FLOAT;
|
|
if (nvn_format <= 0 || nvn_format >= (int)(sizeof vtx_formats / sizeof *vtx_formats)) return VK_FORMAT_UNDEFINED;
|
|
const VtxFmt *v = &vtx_formats[nvn_format];
|
|
if (v->vk == VK_FORMAT_UNDEFINED) return VK_FORMAT_UNDEFINED;
|
|
if (type) *type = v->type;
|
|
if ((nvn_format >= 0x55 && nvn_format <= 0x68) && v->type != GFX_TYPE_FLOAT)
|
|
LOG_ONCE("vk_pipeline: 32-bit *_UI2F/*_I2F vertex formats are read as integers (first: %#x)", nvn_format);
|
|
VkFormat f = v->vk;
|
|
if (vk_ctx() && !(format_features(f, 0) & VK_FORMAT_FEATURE_VERTEX_BUFFER_BIT)) {
|
|
VkFormat w = widen(f);
|
|
if (w == VK_FORMAT_UNDEFINED || !(format_features(w, 0) & VK_FORMAT_FEATURE_VERTEX_BUFFER_BIT)) {
|
|
LOG_ONCE("vk_pipeline: vertex format %d (NVN %#x) unsupported", (int)f, nvn_format);
|
|
return VK_FORMAT_UNDEFINED;
|
|
}
|
|
LOG_ONCE("vk_pipeline: vertex format %d unsupported, read as 4 components (first: NVN %#x)", (int)f, nvn_format);
|
|
f = w;
|
|
}
|
|
return f;
|
|
}
|
|
|
|
// Attachment format of an NVN format (tex_vk_format: the format the images are created with).
|
|
static VkFormat target_format(int nvn_format, int *depth, int *stencil) {
|
|
TexVkFormat t;
|
|
if (depth) *depth = 0;
|
|
if (stencil) *stencil = 0;
|
|
if (!nvn_format || !tex_vk_format(nvn_format, &t)) return VK_FORMAT_UNDEFINED;
|
|
if (depth) *depth = t.depth;
|
|
if (stencil) *stencil = t.stencil;
|
|
return t.vk;
|
|
}
|
|
|
|
void vk_pipe_state_defaults(VkPipeState *s) {
|
|
memset(s, 0, sizeof *s);
|
|
s->samples = 1;
|
|
s->prim = NVN_DRAW_PRIMITIVE_TRIANGLES;
|
|
for (int i = 0; i < 8; i++)
|
|
s->blend[i] = (blend_st){(uint64_t)i, NVN_BLEND_FUNC_ONE, NVN_BLEND_FUNC_ZERO, NVN_BLEND_FUNC_ONE,
|
|
NVN_BLEND_FUNC_ZERO, NVN_BLEND_EQUATION_ADD, NVN_BLEND_EQUATION_ADD};
|
|
s->color = (color_st){0, NVN_LOGIC_OP_COPY, NVN_ALPHA_FUNC_ALWAYS};
|
|
s->chmask.mask = 0xffffffffu;
|
|
s->ms.enable = 1;
|
|
s->poly = (poly_st){NVN_FACE_NONE, NVN_FRONT_FACE_CCW, NVN_POLYGON_MODE_FILL, 0};
|
|
s->ds = (ds_st){0, 1, NVN_DEPTH_FUNC_LESS, 0, NVN_STENCIL_FUNC_ALWAYS, NVN_STENCIL_FUNC_ALWAYS,
|
|
NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP,
|
|
NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP};
|
|
}
|
|
|
|
void vk_pipeline_answers(const VkPipeState *s, GfxShaderAnswers *vs, GfxShaderAnswers *fs) {
|
|
if (vs)
|
|
for (int i = 0; i < 16; i++) {
|
|
va_st a;
|
|
memcpy(&a, &s->vattr[i], sizeof a);
|
|
int type = GFX_TYPE_FLOAT;
|
|
if (a.fmt) vk_vertex_format((int)a.fmt, &type);
|
|
vs->attr_type[i] = (uint8_t)type;
|
|
}
|
|
if (fs)
|
|
for (int i = 0; i < 8; i++) {
|
|
VkFormat f = target_format(s->color_fmt[i], NULL, NULL);
|
|
fs->frag_out_type[i] = (uint8_t)(f == VK_FORMAT_UNDEFINED ? GFX_TYPE_FLOAT : format_type(f));
|
|
}
|
|
}
|
|
|
|
// ---- vertex input (pipeline state, or vkCmdSetVertexInputEXT)
|
|
|
|
typedef VkVtxInput VtxInput;
|
|
|
|
// An attribute the game left without a format is a fixed one on Maxwell, a single 32-bit component of 0 (an unused
|
|
// attribute is bound as IsFixed 1x32 float): it reads (0, 0, 0, 1), as Vulkan expands a
|
|
// one-component format. NX2NAVI's foliage shaders scale wind sway by 1 - colour.w; shapes without a vertex colour
|
|
// (Brightbloom stem and leaves) must not sway, or they bend away from the parts that have one (the bulb).
|
|
static VkFormat zero_format(int type) {
|
|
return type == GFX_TYPE_UINT ? VK_FORMAT_R32_UINT : type == GFX_TYPE_SINT ? VK_FORMAT_R32_SINT : VK_FORMAT_R32_SFLOAT;
|
|
}
|
|
|
|
// Attributes for the Locations the vertex shader reads; one without a usable NVN attribute reads VK_PIPE_ZERO_BINDING.
|
|
static void vertex_input(const VkPipeState *s, VtxInput *v) {
|
|
GfxShaderIo io[32];
|
|
uint32_t used = gfx_shader_inputs(s->vs, io), streams = 0;
|
|
int zero = 0;
|
|
memset(v, 0, sizeof *v);
|
|
for (int l = 0; l < 32; l++) {
|
|
if (!(used >> l & 1)) continue;
|
|
VkVertexInputAttributeDescription2EXT *a = &v->a[v->n_attr++];
|
|
*a = (VkVertexInputAttributeDescription2EXT){.sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_ATTRIBUTE_DESCRIPTION_2_EXT,
|
|
.location = (uint32_t)l};
|
|
va_st va = {0};
|
|
if (l < 16) memcpy(&va, &s->vattr[l], sizeof va);
|
|
VkFormat f = va.fmt ? vk_vertex_format((int)va.fmt, NULL) : VK_FORMAT_UNDEFINED;
|
|
if (f != VK_FORMAT_UNDEFINED && va.stream < 16) {
|
|
a->binding = va.stream, a->format = f, a->offset = va.offset;
|
|
streams |= 1u << va.stream;
|
|
} else {
|
|
a->binding = VK_PIPE_ZERO_BINDING, a->format = zero_format(io[l].type);
|
|
zero = 1;
|
|
}
|
|
}
|
|
for (uint32_t b = 0; b < 16; b++) {
|
|
if (!(streams >> b & 1)) continue;
|
|
uint32_t div = s->vstream[b].divisor;
|
|
if (div > 1) LOG_ONCE("vk_pipeline: instance divisor %u treated as 1 (no VK_EXT_vertex_attribute_divisor)", div);
|
|
v->b[v->n_bind++] = (VkVertexInputBindingDescription2EXT){
|
|
.sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT, .binding = b, .stride = s->vstream[b].stride,
|
|
.inputRate = div ? VK_VERTEX_INPUT_RATE_INSTANCE : VK_VERTEX_INPUT_RATE_VERTEX, .divisor = 1};
|
|
}
|
|
if (zero)
|
|
v->b[v->n_bind++] = (VkVertexInputBindingDescription2EXT){
|
|
.sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT, .binding = VK_PIPE_ZERO_BINDING,
|
|
.stride = 0, .inputRate = VK_VERTEX_INPUT_RATE_VERTEX, .divisor = 1};
|
|
}
|
|
|
|
int vk_pipeline_vertex_input_dynamic(void) { return ensure_init() && P.dyn_vi; }
|
|
|
|
PFN_vkCmdPushDescriptorSetKHR vk_pipeline_push_fn(void) { return ensure_init() ? P.push : NULL; }
|
|
PFN_vkCmdPushDescriptorSetWithTemplateKHR vk_pipeline_push_tpl_fn(void) { return ensure_init() && P.tpl ? P.tpl_push : NULL; }
|
|
PFN_vkUpdateDescriptorSetWithTemplate vk_pipeline_update_tpl_fn(void) { return ensure_init() && P.tpl ? P.tpl_update : NULL; }
|
|
|
|
int vk_pipeline_set_indep(int on) {
|
|
if (!ensure_init()) return 0;
|
|
plat_mutex_lock(&P.lock);
|
|
int was = P.indep;
|
|
P.indep = on && vk_caps.gpl;
|
|
plat_mutex_unlock(&P.lock);
|
|
return was;
|
|
}
|
|
|
|
// ---- SPIR-V rewrites
|
|
|
|
// Binding decorations b >= 1 become b + off (the fragment stage's share of set 0).
|
|
static uint32_t *spirv_rebind(const uint32_t *w, uint64_t n, uint32_t off) {
|
|
uint32_t *o = malloc(n * 4);
|
|
if (!o) return NULL;
|
|
memcpy(o, w, n * 4);
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = o[i] & 0xffff, len = o[i] >> 16;
|
|
if (!len || i + len > n || op == SPV_OP_FUNCTION) break;
|
|
if (op == SPV_OP_DECORATE && len >= 4 && o[i + 2] == SPV_DEC_BINDING && o[i + 3] >= 1) o[i + 3] += off;
|
|
i += len;
|
|
}
|
|
return o;
|
|
}
|
|
|
|
// DescriptorSet decorations become set (the fragment stage's resources move to set 1 with independent layouts).
|
|
static uint32_t *spirv_set(const uint32_t *w, uint64_t n, uint32_t set) {
|
|
uint32_t *o = malloc(n * 4);
|
|
if (!o) return NULL;
|
|
memcpy(o, w, n * 4);
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = o[i] & 0xffff, len = o[i] >> 16;
|
|
if (!len || i + len > n || op == SPV_OP_FUNCTION) break;
|
|
if (op == SPV_OP_DECORATE && len >= 4 && o[i + 2] == SPV_DEC_DESCRIPTOR_SET) o[i + 3] = set;
|
|
i += len;
|
|
}
|
|
return o;
|
|
}
|
|
|
|
// Dual-source blending (docs/maxwell.md 5.6): SRC1 blend factors read the fragment shader's colour output 1,
|
|
// which Vulkan takes from Location 0, Index 1. The Output variable at Location 1 moves there, as the translator declares it
|
|
// with DualSourceBlendEnable (CodeGen/Spirv/Declarations.cs); the pipeline is keyed by GKey.dsb, so only pipelines
|
|
// that blend with SRC1 get this module. NULL when the module has no Output at Location 1.
|
|
static uint32_t *spirv_dual_source(const uint32_t *w, uint64_t n, uint64_t *out_n) {
|
|
uint32_t outs[64];
|
|
int nout = 0;
|
|
uint64_t at = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n) return NULL;
|
|
if (op == SPV_OP_VARIABLE && len >= 4 && w[i + 3] == SPV_SC_OUTPUT && nout < 64) outs[nout++] = w[i + 2];
|
|
i += len;
|
|
}
|
|
for (uint64_t i = 5; i < n && !at;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (op == SPV_OP_DECORATE && len == 4 && w[i + 2] == SPV_DEC_LOCATION && w[i + 3] == 1)
|
|
for (int k = 0; k < nout; k++)
|
|
if (outs[k] == w[i + 1]) at = i;
|
|
i += len;
|
|
}
|
|
if (!at) return NULL;
|
|
uint32_t *o = malloc((n + 4) * 4);
|
|
if (!o) return NULL;
|
|
memcpy(o, w, (at + 4) * 4);
|
|
o[at + 3] = 0;
|
|
o[at + 4] = 4u << 16 | SPV_OP_DECORATE, o[at + 5] = w[at + 1], o[at + 6] = SPV_DEC_INDEX, o[at + 7] = 1;
|
|
memcpy(o + at + 8, w + at + 4, (n - at - 4) * 4);
|
|
*out_n = n + 4;
|
|
return o;
|
|
}
|
|
|
|
// Adds an Output variable for each Location of `missing` (type and width from io) to a module, declared in the entry
|
|
// point interface and initialised as Maxwell reads an unwritten output: (0, 0, 0, 1), the w as the bits of 1.0f whatever
|
|
// the type (attribute memory is untyped); fewer than 4 components are all zero. Existing scalar, vector and pointer types
|
|
// are reused. NULL when the module does not have the expected layout.
|
|
static uint32_t *spirv_add_outputs(const uint32_t *w, uint64_t n, uint32_t missing, const GfxShaderIo io[32],
|
|
uint64_t *out_n) {
|
|
uint64_t ep_at = 0, ann_end = 0, fn_at = 0;
|
|
uint32_t bound = w[3], scalar[3] = {0, 0, 0}; // GFX_TYPE_FLOAT, SINT, UINT
|
|
struct { uint32_t id, comp, count; } vec[64];
|
|
struct { uint32_t id, type; } ptr[64];
|
|
int nvec = 0, nptr = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n) return NULL;
|
|
const uint32_t *a = w + i + 1;
|
|
if (op == SPV_OP_ENTRY_POINT && !ep_at) ep_at = i;
|
|
else if (op == SPV_OP_DECORATE || op == 72 || op == 73 || op == 74 || op == 75 || op == 332 || op == 5632 || op == 5633)
|
|
ann_end = i + len;
|
|
else if (op == SPV_OP_TYPE_FLOAT && len >= 3 && a[1] == 32) scalar[GFX_TYPE_FLOAT] = a[0];
|
|
else if (op == SPV_OP_TYPE_INT && len >= 4 && a[1] == 32) scalar[a[2] ? GFX_TYPE_SINT : GFX_TYPE_UINT] = a[0];
|
|
else if (op == SPV_OP_TYPE_VECTOR && len >= 4 && nvec < 64) vec[nvec].id = a[0], vec[nvec].comp = a[1], vec[nvec++].count = a[2];
|
|
else if (op == SPV_OP_TYPE_POINTER && len >= 4 && a[1] == SPV_SC_OUTPUT && nptr < 64) ptr[nptr].id = a[0], ptr[nptr++].type = a[2];
|
|
else if (op == SPV_OP_FUNCTION) { fn_at = i; break; }
|
|
i += len;
|
|
}
|
|
if (!ep_at || !ann_end || !fn_at || ann_end < ep_at || fn_at < ann_end) return NULL;
|
|
uint32_t types[32 * 40], decs[32 * 4], vars[32], zero[3] = {0, 0, 0}, one[3] = {0, 0, 0};
|
|
int nt = 0, nd = 0, nv = 0;
|
|
for (int l = 0; l < 32; l++) {
|
|
if (!(missing >> l & 1)) continue;
|
|
int t = io[l].type <= GFX_TYPE_UINT ? io[l].type : GFX_TYPE_FLOAT;
|
|
uint32_t comps = io[l].comps ? io[l].comps : 4;
|
|
if (!scalar[t]) {
|
|
scalar[t] = bound++;
|
|
if (t == GFX_TYPE_FLOAT) types[nt++] = 3u << 16 | SPV_OP_TYPE_FLOAT, types[nt++] = scalar[t], types[nt++] = 32;
|
|
else types[nt++] = 4u << 16 | SPV_OP_TYPE_INT, types[nt++] = scalar[t], types[nt++] = 32, types[nt++] = t == GFX_TYPE_SINT;
|
|
}
|
|
uint32_t type = scalar[t];
|
|
if (comps > 1) {
|
|
int k = 0;
|
|
while (k < nvec && !(vec[k].comp == type && vec[k].count == comps)) k++;
|
|
if (k == nvec) {
|
|
if (nvec == 64) return NULL;
|
|
vec[nvec].id = bound++, vec[nvec].comp = type, vec[nvec++].count = comps;
|
|
types[nt++] = 4u << 16 | SPV_OP_TYPE_VECTOR, types[nt++] = vec[k].id, types[nt++] = type, types[nt++] = comps;
|
|
}
|
|
type = vec[k].id;
|
|
}
|
|
int k = 0;
|
|
while (k < nptr && ptr[k].type != type) k++;
|
|
if (k == nptr) {
|
|
if (nptr == 64) return NULL;
|
|
ptr[nptr].id = bound++, ptr[nptr++].type = type;
|
|
types[nt++] = 4u << 16 | SPV_OP_TYPE_POINTER, types[nt++] = ptr[k].id, types[nt++] = SPV_SC_OUTPUT, types[nt++] = type;
|
|
}
|
|
uint32_t init = bound++, var = bound++;
|
|
if (comps == 4) {
|
|
if (!zero[t]) {
|
|
zero[t] = bound++, one[t] = bound++;
|
|
types[nt++] = 4u << 16 | SPV_OP_CONSTANT, types[nt++] = scalar[t], types[nt++] = zero[t], types[nt++] = 0;
|
|
types[nt++] = 4u << 16 | SPV_OP_CONSTANT, types[nt++] = scalar[t], types[nt++] = one[t], types[nt++] = 0x3f800000;
|
|
}
|
|
types[nt++] = 7u << 16 | SPV_OP_CONSTANT_COMPOSITE, types[nt++] = type, types[nt++] = init;
|
|
types[nt++] = zero[t], types[nt++] = zero[t], types[nt++] = zero[t], types[nt++] = one[t];
|
|
} else {
|
|
types[nt++] = 3u << 16 | SPV_OP_CONSTANT_NULL, types[nt++] = type, types[nt++] = init;
|
|
}
|
|
types[nt++] = 5u << 16 | SPV_OP_VARIABLE, types[nt++] = ptr[k].id, types[nt++] = var, types[nt++] = SPV_SC_OUTPUT,
|
|
types[nt++] = init;
|
|
decs[nd++] = 4u << 16 | SPV_OP_DECORATE, decs[nd++] = var, decs[nd++] = SPV_DEC_LOCATION, decs[nd++] = (uint32_t)l;
|
|
vars[nv++] = var;
|
|
}
|
|
uint64_t total = n + (uint64_t)(nt + nd + nv);
|
|
uint32_t *o = malloc(total * 4), *p = o;
|
|
if (!o) return NULL;
|
|
uint32_t ep_len = w[ep_at] >> 16;
|
|
memcpy(p, w, 5 * 4), p[3] = bound, p += 5;
|
|
memcpy(p, w + 5, (ep_at - 5) * 4), p += ep_at - 5;
|
|
*p++ = (ep_len + (uint32_t)nv) << 16 | SPV_OP_ENTRY_POINT;
|
|
memcpy(p, w + ep_at + 1, (ep_len - 1) * 4), p += ep_len - 1;
|
|
memcpy(p, vars, (size_t)nv * 4), p += nv;
|
|
memcpy(p, w + ep_at + ep_len, (ann_end - ep_at - ep_len) * 4), p += ann_end - ep_at - ep_len;
|
|
memcpy(p, decs, (size_t)nd * 4), p += nd;
|
|
memcpy(p, w + ann_end, (fn_at - ann_end) * 4), p += fn_at - ann_end;
|
|
memcpy(p, types, (size_t)nt * 4), p += nt;
|
|
memcpy(p, w + fn_at, (n - fn_at) * 4), p += n - fn_at;
|
|
*out_n = (uint64_t)(p - o);
|
|
return o;
|
|
}
|
|
|
|
// Two fixes for what the translator declares but a shader does not use (NULL: nothing to change):
|
|
// - the fragment shader interlock capability and extension, which the translator adds to every fragment shader when the host
|
|
// reports interlock support (SpirvGenerator.cs) and vk_device.c does not enable; kept when the shader interlocks;
|
|
// - the CullDistance[1] member of gl_PerVertex next to ClipDistance[8] (Declarations.cs CreatePerVertexStructType):
|
|
// 9 distances exceed maxCombinedClipAndCullDistances (8 on the RX 9070). The translator never accesses CullDistance; the
|
|
// member goes when it is the struct's last and no access chain selects it.
|
|
static uint32_t *spirv_fixup(const uint32_t *w, uint64_t n, uint64_t *out_n) {
|
|
enum { OP_EXTENSION = 10, OP_CAPABILITY = 17, OP_MEMBER_NAME = 6, OP_EXECUTION_MODE = 16, OP_TYPE_ARRAY = 28,
|
|
OP_TYPE_STRUCT = 30, OP_CONSTANT = 43, OP_MEMBER_DECORATE = 72, OP_ACCESS_CHAIN = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66,
|
|
OP_BEGIN_INTERLOCK = 5364, OP_END_INTERLOCK = 5365, CAP_SAMPLE_INTERLOCK = 5363, CAP_PIXEL_INTERLOCK = 5378,
|
|
CAP_RATE_INTERLOCK = 5372, BUILTIN_CLIP = 3, BUILTIN_CULL = 4 };
|
|
if (n < 5) return NULL;
|
|
uint32_t bound = w[3] < (1u << 22) ? w[3] : (1u << 22);
|
|
uint32_t *val = calloc(bound, sizeof *val), *elem = calloc(bound, sizeof *elem), *len = calloc(bound, sizeof *len),
|
|
*pointee = calloc(bound, sizeof *pointee), *var_type = calloc(bound, sizeof *var_type);
|
|
uint32_t *o = NULL;
|
|
int interlock_used = 0, interlock_declared = 0, cull_ok = 0;
|
|
uint32_t cull_struct = 0, cull_member = 0, clip_member = ~0u, members = 0, member_types[8] = {0};
|
|
if (!val || !elem || !len || !pointee || !var_type) goto out;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) goto out;
|
|
const uint32_t *a = w + i + 1;
|
|
if (op == OP_BEGIN_INTERLOCK || op == OP_END_INTERLOCK) interlock_used = 1;
|
|
else if (op == OP_EXECUTION_MODE && l >= 3 && a[1] >= 5366 && a[1] <= 5371) interlock_used = 1;
|
|
else if (op == OP_CAPABILITY && l >= 2 && (a[0] == CAP_PIXEL_INTERLOCK || a[0] == CAP_SAMPLE_INTERLOCK || a[0] == CAP_RATE_INTERLOCK))
|
|
interlock_declared = 1;
|
|
else if (op == OP_CONSTANT && l == 4 && a[1] < bound) val[a[1]] = a[2];
|
|
else if (op == OP_TYPE_ARRAY && l >= 4 && a[0] < bound) elem[a[0]] = a[1], len[a[0]] = a[2] < bound ? val[a[2]] : 0;
|
|
else if (op == SPV_OP_TYPE_POINTER && l >= 4 && a[0] < bound) pointee[a[0]] = a[2];
|
|
else if (op == SPV_OP_VARIABLE && l >= 4 && a[1] < bound) var_type[a[1]] = a[0] < bound ? pointee[a[0]] : 0;
|
|
else if (op == OP_MEMBER_DECORATE && l >= 5 && a[2] == SPV_DEC_BUILTIN && a[3] == BUILTIN_CULL) cull_struct = a[0], cull_member = a[1];
|
|
else if (op == OP_MEMBER_DECORATE && l >= 5 && a[2] == SPV_DEC_BUILTIN && a[3] == BUILTIN_CLIP) clip_member = a[1];
|
|
else if (op == OP_TYPE_STRUCT && cull_struct && a[0] == cull_struct && l >= 2) {
|
|
members = l - 2;
|
|
for (uint32_t m = 0; m < members && m < 8; m++) member_types[m] = a[1 + m];
|
|
} else if ((op == OP_ACCESS_CHAIN || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && cull_struct && a[2] < bound) {
|
|
uint32_t t = var_type[a[2]], k = 3;
|
|
if (t < bound && t && elem[t] == cull_struct) k = 4; // arrayed per-vertex block
|
|
if ((t == cull_struct || k == 4) && k + 1 < l && a[k] < bound && val[a[k]] == cull_member) cull_struct = 0;
|
|
}
|
|
i += l;
|
|
}
|
|
VkCtx *c = vk_ctx();
|
|
uint32_t limit = c ? c->props.limits.maxCombinedClipAndCullDistances : 8;
|
|
if (cull_struct && members && members <= 8 && cull_member == members - 1 && clip_member < members) {
|
|
uint32_t clip_t = member_types[clip_member], cull_t = member_types[cull_member];
|
|
uint32_t clip_n = clip_t < bound ? len[clip_t] : 0, cull_n = cull_t < bound ? len[cull_t] : 0;
|
|
cull_ok = clip_n + cull_n > limit && clip_n <= limit;
|
|
}
|
|
int strip = interlock_declared && !interlock_used;
|
|
if (!strip && !cull_ok) goto out;
|
|
o = malloc(n * 4);
|
|
if (!o) goto out;
|
|
uint64_t on = 5;
|
|
memcpy(o, w, 5 * 4);
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
const uint32_t *a = w + i + 1;
|
|
int drop = 0;
|
|
if (strip && op == OP_CAPABILITY && (a[0] == CAP_PIXEL_INTERLOCK || a[0] == CAP_SAMPLE_INTERLOCK || a[0] == CAP_RATE_INTERLOCK)) drop = 1;
|
|
if (strip && op == OP_EXTENSION && l >= 2 && !strncmp((const char *)a, "SPV_EXT_fragment_shader_interlock", (l - 1) * 4)) drop = 1;
|
|
if (cull_ok && (op == OP_MEMBER_DECORATE || op == OP_MEMBER_NAME) && l >= 3 && a[0] == cull_struct && a[1] == cull_member) drop = 1;
|
|
if (!drop) {
|
|
memcpy(o + on, w + i, l * 4);
|
|
if (cull_ok && op == OP_TYPE_STRUCT && a[0] == cull_struct) o[on] = (l - 1) << 16 | op; // last member gone
|
|
on += op == OP_TYPE_STRUCT && cull_ok && a[0] == cull_struct ? l - 1 : l;
|
|
}
|
|
i += l;
|
|
}
|
|
*out_n = on;
|
|
out:
|
|
free(val), free(elem), free(len), free(pointee), free(var_type);
|
|
return o;
|
|
}
|
|
|
|
// True when the module declares a GroupNonUniform* or SubgroupBallot-style capability (lane masks assume 32 lanes).
|
|
static int uses_subgroup_ops(const uint32_t *w, uint64_t words) {
|
|
for (uint64_t i = 5; i < words;) {
|
|
uint32_t op = w[i] & 0xffff, n = w[i] >> 16;
|
|
if (!n || i + n > words) return 0;
|
|
if (op != 17) return 0; // OpCapability; capabilities come first, stop at the first other instruction
|
|
uint32_t cap = w[i + 1];
|
|
if ((cap >= 61 && cap <= 68) || cap == 4423 || cap == 4431) return 1; // NonUniform*, SubgroupBallotKHR, SubgroupVoteKHR
|
|
i += n;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Guest code runs warp-wide operations (ballot, shuffle, vote) where the guest compiler had the warp reconverged, for
|
|
// example right after a loop that each lane leaves at a different iteration. Vulkan only promises the same lanes at the
|
|
// same point with the MaximallyReconvergesKHR execution mode. Without it a driver may run the lanes that left the loop
|
|
// early through the operations on their own, with a ballot or scan that sees only some of the lanes: a mesh program
|
|
// that hands out output slots from one could give two primitives the same slot. This adds the extension and the mode
|
|
// for each entry point of a module that declares subgroup capabilities. NULL: nothing to change (or no memory).
|
|
static uint32_t *spirv_reconverge(const uint32_t *w, uint64_t n, uint64_t *out_n) {
|
|
enum { OP_EXTENSION = 10, OP_EXECUTION_MODE = 16, OP_CAPABILITY = 17, MODE_MAXIMALLY_RECONVERGES = 6023 };
|
|
static const char ext[] = "SPV_KHR_maximal_reconvergence";
|
|
const uint32_t ext_words = (uint32_t)(sizeof ext + 3) / 4; // the string with its terminating zero, padded
|
|
if (n < 5 || !uses_subgroup_ops(w, n)) return NULL;
|
|
uint64_t caps_end = 0, eps_end = 0;
|
|
uint32_t eps[16], neps = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) return NULL;
|
|
if (op == OP_CAPABILITY) caps_end = i + l;
|
|
else if (op == SPV_OP_ENTRY_POINT && l >= 3) {
|
|
if (neps == 16) return NULL;
|
|
eps[neps++] = w[i + 2], eps_end = i + l;
|
|
} else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == MODE_MAXIMALLY_RECONVERGES) return NULL;
|
|
else if (op == SPV_OP_FUNCTION) break;
|
|
i += l;
|
|
}
|
|
if (!caps_end || !neps || eps_end < caps_end) return NULL;
|
|
uint64_t on = n + 1 + ext_words + 3ull * neps;
|
|
uint32_t *o = malloc(on * 4);
|
|
if (!o) return NULL;
|
|
uint64_t p = 0;
|
|
memcpy(o, w, caps_end * 4), p = caps_end;
|
|
o[p++] = (1 + ext_words) << 16 | OP_EXTENSION;
|
|
memset(o + p, 0, ext_words * 4);
|
|
memcpy(o + p, ext, sizeof ext), p += ext_words;
|
|
memcpy(o + p, w + caps_end, (eps_end - caps_end) * 4), p += eps_end - caps_end;
|
|
for (uint32_t k = 0; k < neps; k++) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = eps[k], o[p++] = MODE_MAXIMALLY_RECONVERGES;
|
|
memcpy(o + p, w + eps_end, (n - eps_end) * 4), p += n - eps_end;
|
|
*out_n = p;
|
|
return o;
|
|
}
|
|
|
|
// Maxwell computes in IEEE 754, and the game's shaders rely on it: its compiler writes an unordered comparison as
|
|
// `x > c || isnan(x)`, a normalisation of a zero-length vector gives an infinity that a later comparison sorts out, and
|
|
// integer masks such as 0xffffffff travel through float moves and selects. The translator's own guards are isnan
|
|
// tests as well (fragment outputs, min, max and clamp). Without the SignedZeroInfNanPreserve execution mode, SPIR-V
|
|
// lets the driver assume that no float is NaN or infinite, and AMD's compilers act on it: on an RX 9070 XT a select
|
|
// between 1 and the bits 0xffffffff, converted to float as a signed integer, gave 2^32 instead of -1 and blew the
|
|
// shadow casters up into huge triangles (issue #156); RX 6000 cards drew Navi, and Link's skin next to her, with black
|
|
// and white blocks (#41, #110); and AMD GPUs drew Navi's sparkles but not her orb, whose light sits at her own centre.
|
|
// NVIDIA's driver keeps the IEEE results without the mode. This adds the capability, the extension (the translator
|
|
// writes SPIR-V 1.3) and the mode for each entry point: 32-bit floats always, 16-bit and 64-bit ones where the module
|
|
// declares Float16 or Float64 and `widths` (NP_*) allows them. NULL: nothing to change (the module has the mode, or no
|
|
// 32-bit width is allowed, or no memory).
|
|
static uint32_t *spirv_nan_preserve(const uint32_t *w, uint64_t n, uint32_t widths, uint64_t *out_n) {
|
|
enum { OP_EXTENSION = 10, OP_EXECUTION_MODE = 16, OP_CAPABILITY = 17, CAP_FLOAT16 = 9, CAP_FLOAT64 = 10,
|
|
CAP_PRESERVE = 4466, MODE_PRESERVE = 4461 };
|
|
static const char ext[] = "SPV_KHR_float_controls";
|
|
const uint32_t ext_words = (uint32_t)(sizeof ext + 3) / 4; // the string with its terminating zero, padded
|
|
if (n < 5 || !(widths & NP_32)) return NULL;
|
|
uint64_t caps_end = 0, eps_end = 0;
|
|
uint32_t eps[16], neps = 0, use = NP_32;
|
|
int has_cap = 0, has_ext = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) return NULL;
|
|
if (op == OP_CAPABILITY && l >= 2) {
|
|
caps_end = i + l;
|
|
if (w[i + 1] == CAP_FLOAT16) use |= NP_16;
|
|
else if (w[i + 1] == CAP_FLOAT64) use |= NP_64;
|
|
else if (w[i + 1] == CAP_PRESERVE) has_cap = 1;
|
|
} else if (op == OP_EXTENSION && (l - 1) * 4 >= sizeof ext && !memcmp(w + i + 1, ext, sizeof ext)) {
|
|
has_ext = 1;
|
|
} else if (op == SPV_OP_ENTRY_POINT && l >= 3) {
|
|
if (neps == 16) return NULL;
|
|
eps[neps++] = w[i + 2], eps_end = i + l;
|
|
} else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == MODE_PRESERVE) {
|
|
return NULL;
|
|
} else if (op == SPV_OP_FUNCTION) {
|
|
break;
|
|
}
|
|
i += l;
|
|
}
|
|
if (!caps_end || !neps || eps_end < caps_end) return NULL;
|
|
use &= widths;
|
|
uint32_t nw = 0, width[3];
|
|
if (use & NP_16) width[nw++] = 16;
|
|
width[nw++] = 32;
|
|
if (use & NP_64) width[nw++] = 64;
|
|
uint64_t on = n + (has_cap ? 0 : 2) + (has_ext ? 0 : 1 + ext_words) + 4ull * nw * neps;
|
|
uint32_t *o = malloc(on * 4);
|
|
if (!o) return NULL;
|
|
uint64_t p = 0;
|
|
memcpy(o, w, caps_end * 4), p = caps_end;
|
|
if (!has_cap) o[p++] = 2u << 16 | OP_CAPABILITY, o[p++] = CAP_PRESERVE;
|
|
if (!has_ext) {
|
|
o[p++] = (1 + ext_words) << 16 | OP_EXTENSION;
|
|
memset(o + p, 0, ext_words * 4);
|
|
memcpy(o + p, ext, sizeof ext), p += ext_words;
|
|
}
|
|
memcpy(o + p, w + caps_end, (eps_end - caps_end) * 4), p += eps_end - caps_end;
|
|
for (uint32_t k = 0; k < neps; k++)
|
|
for (uint32_t j = 0; j < nw; j++)
|
|
o[p++] = 4u << 16 | OP_EXECUTION_MODE, o[p++] = eps[k], o[p++] = MODE_PRESERVE, o[p++] = width[j];
|
|
memcpy(o + p, w + eps_end, (n - eps_end) * 4), p += n - eps_end;
|
|
*out_n = p;
|
|
return o;
|
|
}
|
|
|
|
static int spirv_mark_list_test(const char *list); // texture pool marks, below
|
|
|
|
// --selftest (main.c): spirv_nan_preserve on a small fragment module, without a device. Returns the number of failures.
|
|
int vk_pipeline_spirv_selftest(void) {
|
|
enum { CAP = 2u << 16 | 17, MODE = 4u << 16 | 16 };
|
|
static const uint32_t head[] = {0x07230203, 0x00010300, 0, 5, 0, CAP, 1, CAP, 10}; // Shader, Float64
|
|
static const uint32_t body[] = {
|
|
3u << 16 | 14, 0, 1, // OpMemoryModel Logical GLSL450
|
|
5u << 16 | 15, 4, 1, 0x6e69616d, 0, // OpEntryPoint Fragment %1 "main"
|
|
3u << 16 | 16, 1, 7, // OpExecutionMode %1 OriginUpperLeft
|
|
2u << 16 | 19, 2, // %2 = OpTypeVoid
|
|
3u << 16 | 33, 3, 2, // %3 = OpTypeFunction %2
|
|
5u << 16 | 54, 2, 1, 0, 3, // %1 = OpFunction %2 None %3
|
|
2u << 16 | 248, 4, 1u << 16 | 253, 1u << 16 | 56}; // OpLabel, OpReturn, OpFunctionEnd
|
|
enum { NH = sizeof head / 4, NB = sizeof body / 4, EP_END = 8 }; // EP_END: body words up to the entry point's end
|
|
uint32_t in[NH + NB], want[NH + NB + 32];
|
|
memcpy(in, head, sizeof head), memcpy(in + NH, body, sizeof body);
|
|
int failures = 0;
|
|
// widths: what the device allows; f64: the module declares Float64; w64: a 64-bit mode is expected
|
|
for (int t = 0; t < 3; t++) {
|
|
uint32_t widths = t == 0 ? NP_16 | NP_32 | NP_64 : NP_32, n = NH + NB, p = 0;
|
|
int f64 = t != 2, w64 = t == 0;
|
|
if (!f64) in[NH - 1] = 0; // Matrix in place of Float64
|
|
memcpy(want, in, NH * 4), p = NH;
|
|
want[p++] = CAP, want[p++] = 4466;
|
|
want[p++] = 7u << 16 | 10;
|
|
memset(want + p, 0, 24), memcpy(want + p, "SPV_KHR_float_controls", 23), p += 6;
|
|
memcpy(want + p, body, EP_END * 4), p += EP_END;
|
|
want[p++] = MODE, want[p++] = 1, want[p++] = 4461, want[p++] = 32;
|
|
if (w64) want[p++] = MODE, want[p++] = 1, want[p++] = 4461, want[p++] = 64;
|
|
memcpy(want + p, body + EP_END, (NB - EP_END) * 4), p += NB - EP_END;
|
|
uint64_t on = 0, again_n = 0;
|
|
uint32_t *o = spirv_nan_preserve(in, n, widths, &on);
|
|
uint32_t *again = o ? spirv_nan_preserve(o, on, widths, &again_n) : NULL;
|
|
if (!o || on != p || memcmp(o, want, p * 4) || again) {
|
|
printf("spirv: SignedZeroInfNanPreserve rewrite, case %d: wrong output\n", t);
|
|
failures++;
|
|
}
|
|
free(o), free(again);
|
|
}
|
|
if (spirv_nan_preserve(in, NH + NB, NP_16 | NP_64, &(uint64_t){0})) {
|
|
printf("spirv: SignedZeroInfNanPreserve rewrite: a module changed without the 32-bit width\n");
|
|
failures++;
|
|
}
|
|
printf("spirv: SignedZeroInfNanPreserve rewrite: %s\n", failures ? "FAIL" : "ok");
|
|
const char *list = cfg_get("NX2NAVI_SPV_MARK_TEST");
|
|
if (list && *list) failures += spirv_mark_list_test(list);
|
|
return failures;
|
|
}
|
|
|
|
// Chain {size 32} into a stage create info when the stage uses subgroup ops and the driver allows the pin.
|
|
static void pin_subgroup32(VkPipelineShaderStageCreateInfo *st, VkPipelineShaderStageRequiredSubgroupSizeCreateInfo *rq,
|
|
const uint32_t *w, uint64_t words) {
|
|
if (!(P.sg32_stages & st->stage) || !w || !uses_subgroup_ops(w, words)) return;
|
|
*rq = (VkPipelineShaderStageRequiredSubgroupSizeCreateInfo){
|
|
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO, .requiredSubgroupSize = vk_caps.sg_size};
|
|
st->pNext = rq;
|
|
}
|
|
|
|
// The x of a compute module's LocalSize (OpExecutionMode 17), 0 when it has none.
|
|
static uint32_t local_size_x(const uint32_t *w, uint64_t words) {
|
|
for (uint64_t i = 5; w && i < words;) {
|
|
uint32_t op = w[i] & 0xffff, n = w[i] >> 16;
|
|
if (!n || i + n > words) return 0;
|
|
if (op == 16 && n >= 6 && w[i + 2] == 17) return w[i + 3];
|
|
if (op == 54) return 0; // OpFunction: the execution modes are all before it
|
|
i += n;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Pool arrays (the translator's bindless texture and sampler arrays of VK_POOL_TEX / VK_POOL_SAMP elements, found by
|
|
// their types) move to set 2, at the binding of their view kind (vk.h VK_POOL_KINDS) or VK_POOL_KINDS for samplers;
|
|
// the kinds used are noted in vk_pool_kinds_used so the replay fills them. NULL: nothing to move (or no memory).
|
|
static uint32_t *spirv_pool_rebind(const uint32_t *w, uint64_t n) {
|
|
enum { OP_TYPE_IMAGE = 25, OP_TYPE_SAMPLER = 26, OP_TYPE_ARRAY_ = 28 };
|
|
uint32_t bound = n >= 5 && w[3] < (1u << 22) ? w[3] : 0;
|
|
if (!bound) return NULL;
|
|
int32_t *kind = malloc(bound * sizeof *kind); // per id: image kind, 100 sampler, -1 other (types, then variables)
|
|
uint32_t *cval = calloc(bound, sizeof *cval);
|
|
if (!kind || !cval) return free(kind), free(cval), NULL;
|
|
memset(kind, 0xff, bound * sizeof *kind);
|
|
// Pool variables and their binding decorations are module declarations, all before the first function.
|
|
int any = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n || op == SPV_OP_FUNCTION) break;
|
|
if (op == OP_TYPE_IMAGE && len >= 9 && w[i + 1] < bound && w[i + 7] == 1) { // sampled image
|
|
uint32_t dim = w[i + 3], depth = w[i + 4], arrayed = w[i + 5], ms = w[i + 6];
|
|
if (dim <= 3 && !ms) kind[w[i + 1]] = (int32_t)((depth == 1 ? 8 : 0) + dim * 2 + (arrayed ? 1 : 0));
|
|
} else if (op == OP_TYPE_SAMPLER && len >= 2 && w[i + 1] < bound) {
|
|
kind[w[i + 1]] = 100;
|
|
} else if (op == SPV_OP_CONSTANT && len >= 4 && w[i + 2] < bound) {
|
|
cval[w[i + 2]] = w[i + 3];
|
|
} else if (op == OP_TYPE_ARRAY_ && len >= 4 && w[i + 1] < bound && w[i + 2] < bound && w[i + 3] < bound) {
|
|
int32_t k = kind[w[i + 2]];
|
|
uint32_t want = k == 100 ? VK_POOL_SAMP : VK_POOL_TEX;
|
|
kind[w[i + 1]] = k >= 0 && cval[w[i + 3]] == want ? k + 1000 : -1; // + 1000: a pool array
|
|
} else if (op == SPV_OP_TYPE_POINTER && len >= 4 && w[i + 1] < bound && w[i + 3] < bound) {
|
|
kind[w[i + 1]] = kind[w[i + 3]] >= 1000 ? kind[w[i + 3]] : -1;
|
|
} else if (op == SPV_OP_VARIABLE && len >= 4 && w[i + 1] < bound && w[i + 2] < bound) {
|
|
kind[w[i + 2]] = kind[w[i + 1]] >= 1000 ? kind[w[i + 1]] + 1000 : -1; // + 2000: a pool variable
|
|
if (kind[w[i + 2]] >= 2000) any = 1;
|
|
}
|
|
i += len;
|
|
}
|
|
uint32_t *o = NULL;
|
|
if (any && (o = malloc(n * 4))) {
|
|
memcpy(o, w, n * 4);
|
|
uint32_t used = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = o[i] & 0xffff, len = o[i] >> 16;
|
|
if (!len || i + len > n || op == SPV_OP_FUNCTION) break;
|
|
if (op == SPV_OP_DECORATE && len >= 4 && o[i + 1] < bound && kind[o[i + 1]] >= 2000) {
|
|
int32_t k = kind[o[i + 1]] - 2000;
|
|
if (o[i + 2] == SPV_DEC_DESCRIPTOR_SET) o[i + 3] = 2;
|
|
else if (o[i + 2] == SPV_DEC_BINDING) o[i + 3] = k == 100 ? VK_POOL_KINDS : (uint32_t)k;
|
|
if (k != 100) used |= 1u << k;
|
|
}
|
|
i += len;
|
|
}
|
|
__atomic_fetch_or(&vk_pool_kinds_used, used, __ATOMIC_RELAXED);
|
|
}
|
|
free(kind), free(cval);
|
|
return o;
|
|
}
|
|
|
|
// ---- texture pool marks (NX2NAVI_TEX_DISUSE, default on; vk_texture.c "disuse" uses them)
|
|
// Ocarina samples nearly every texture through the bindless pool: a shader reads a handle from memory and indexes the
|
|
// pool array with it, so the CPU never learns which of the registered textures a frame samples, and every registered
|
|
// texture counts as in use (vk_texture.c idle). The game keeps the textures of the areas it has loaded registered, so
|
|
// none of them could go, and their memory, which the driver charges to the system's commit, grew to 6 GiB in a session.
|
|
// This rewrite gives each module that indexes the texture pool a storage buffer at set 2, binding VK_POOL_MARK_BINDING
|
|
// (a uint per pool slot, vk_exec_draw.c "pool marks") and, after every access chain into a pool texture array, a store
|
|
// of 1 into the slot it indexes. The stores of one warp to one slot are one memory transaction. The replay copies the
|
|
// buffer out and clears it every few submissions; slots marked mean textures sampled. A module that indexes the pool
|
|
// and cannot be marked (SPIR-V before 1.3 or after 1.6, a stage whose stores the device does not allow, no memory) is
|
|
// counted in vk_pool_marks_missed, which keeps vk_texture.c from evicting by disuse: an unmarked shader could sample a
|
|
// texture the marks call unused. make_module logs the first few by stage, version, size, hash and reason.
|
|
volatile int32_t vk_pool_marks_missed;
|
|
static int marks_on = -1, marks_vs_ok, marks_fs_ok;
|
|
|
|
int vk_pool_marks_enabled(void) {
|
|
if (marks_on < 0) {
|
|
marks_on = rt_knob("TEX_DISUSE", "1") >= 1; // 2: marks and stand-ins, nothing freed (vk_texture.c "disuse")
|
|
VkCtx *c = vk_ctx();
|
|
if (c) {
|
|
VkPhysicalDeviceFeatures f;
|
|
vkGetPhysicalDeviceFeatures(c->phys, &f); // vk_device.c enables every feature the GPU reports
|
|
marks_vs_ok = f.vertexPipelineStoresAndAtomics, marks_fs_ok = f.fragmentStoresAndAtomics;
|
|
}
|
|
}
|
|
return marks_on;
|
|
}
|
|
|
|
// A fragment shader that stores to memory runs before the depth test unless it declares EarlyFragmentTests: the GPU
|
|
// must run it for fragments the test then rejects, as the stores are visible. The marks would so take the early depth
|
|
// rejection from every marked fragment shader and make overdraw cost full shading. With early_ok, a fragment shader
|
|
// whose result is the same under early tests gets EarlyFragmentTests along with its marks: one that discards, writes
|
|
// its depth, sample mask or stencil reference, or already writes memory does not, as early tests would change what it
|
|
// does. The caller passes early_ok only for a pipeline without alpha to coverage, which also drops samples after the
|
|
// shader (ensure_init keeps alpha to coverage out of the dynamic state while marks are on, so a pipeline knows).
|
|
// *status: 1 marked, 0 no pool texture access to mark, -1 one that could not be marked (*why says what stopped it).
|
|
// NULL unless marked.
|
|
static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, uint64_t *out_n, int *status,
|
|
const char **why) {
|
|
enum { OP_TYPE_RUNTIME_ARRAY = 29, OP_TYPE_STRUCT_ = 30, OP_ACCESS_CHAIN_ = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66,
|
|
OP_STORE_ = 62, OP_MEMBER_DECORATE_ = 72, OP_DECORATION_GROUP = 73, OP_GROUP_DECORATE = 74,
|
|
OP_GROUP_MEMBER_DECORATE = 75, OP_DECORATE_ID = 332, OP_DECORATE_STRING = 5632, OP_MEMBER_DECORATE_STRING = 5633,
|
|
OP_EXECUTION_MODE = 16, OP_EXECUTION_MODE_ID = 331, OP_IMAGE_WRITE = 99, OP_ATOMIC_FIRST = 227,
|
|
OP_ATOMIC_LAST = 242, OP_KILL = 252, OP_TERMINATE_INVOCATION = 4416, OP_DEMOTE_TO_HELPER = 5380,
|
|
EM_EARLY_FRAGMENT_TESTS = 9, BI_SAMPLE_MASK = 20, BI_FRAG_DEPTH = 22, BI_FRAG_STENCIL_REF = 5014,
|
|
SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35 };
|
|
*status = 0, *why = NULL;
|
|
if (n < 5) return NULL;
|
|
uint32_t bound = w[3];
|
|
if (!bound || bound > (1u << 22)) return NULL;
|
|
uint32_t *bind = calloc(bound, sizeof *bind); // per id: its Binding + 1 when its DescriptorSet is 2, else 0
|
|
uint8_t *set2 = calloc(bound, 1);
|
|
if (!bind || !set2) return free(bind), free(set2), *status = -1, *why = "no memory", NULL;
|
|
uint64_t ann_end = 0, fn_start = 0, em_end = 0; // em_end: past the entry points and execution modes
|
|
uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0;
|
|
int vertex_stage = 0, fragment_stage = 0, entries = 0, early = early_ok;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) return free(bind), free(set2), *status = -1, *why = "an instruction past the module's end", NULL;
|
|
if (op == SPV_OP_FUNCTION) { fn_start = i; break; }
|
|
if (op == SPV_OP_ENTRY_POINT || op == OP_EXECUTION_MODE || op == OP_EXECUTION_MODE_ID) em_end = i + l;
|
|
if (op == SPV_OP_ENTRY_POINT && l >= 3) {
|
|
entries++;
|
|
if (w[i + 1] <= 3) vertex_stage = 1; // Vertex, TessellationControl, TessellationEvaluation, Geometry
|
|
else if (w[i + 1] == 4) fragment_stage = 1, fs_entry = w[i + 2];
|
|
} else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == EM_EARLY_FRAGMENT_TESTS) {
|
|
early = 0; // has them already
|
|
} else if (op == SPV_OP_DECORATE && l >= 4 && w[i + 1] < bound) {
|
|
if (w[i + 2] == SPV_DEC_DESCRIPTOR_SET && w[i + 3] == 2) set2[w[i + 1]] = 1;
|
|
else if (w[i + 2] == SPV_DEC_BINDING) bind[w[i + 1]] = w[i + 3] + 1;
|
|
else if (w[i + 2] == SPV_DEC_BUILTIN &&
|
|
(w[i + 3] == BI_SAMPLE_MASK || w[i + 3] == BI_FRAG_DEPTH || w[i + 3] == BI_FRAG_STENCIL_REF))
|
|
early = 0;
|
|
} else if (op == SPV_OP_VARIABLE && l >= 4 && w[i + 3] == SC_STORAGE_BUFFER) {
|
|
early = 0; // the guest's own storage buffers: its stores would change under early tests
|
|
} else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 0) {
|
|
uint_t = w[i + 1];
|
|
} else if (op == SPV_OP_CONSTANT && l == 4 && uint_t && w[i + 1] == uint_t) {
|
|
if (w[i + 3] == 0 && !c0) c0 = w[i + 2];
|
|
if (w[i + 3] == 1 && !c1) c1 = w[i + 2];
|
|
} else if (op == SPV_OP_TYPE_POINTER && l == 4 && w[i + 2] == SC_STORAGE_BUFFER && uint_t && w[i + 3] == uint_t) {
|
|
ptr_u = w[i + 1];
|
|
}
|
|
if (op == SPV_OP_DECORATE || op == OP_MEMBER_DECORATE_ || op == OP_DECORATION_GROUP || op == OP_GROUP_DECORATE ||
|
|
op == OP_GROUP_MEMBER_DECORATE || op == OP_DECORATE_ID || op == OP_DECORATE_STRING || op == OP_MEMBER_DECORATE_STRING)
|
|
ann_end = i + l;
|
|
i += l;
|
|
}
|
|
// pool texture variables: set 2, a binding below VK_POOL_KINDS (spirv_pool_rebind); one already at the marks
|
|
// binding means the module was marked before
|
|
uint64_t sites = 0;
|
|
int marked = 0;
|
|
for (uint32_t id = 0; id < bound; id++)
|
|
if (set2[id] && bind[id] == VK_POOL_MARK_BINDING + 1) marked = 1;
|
|
for (uint64_t i = fn_start; fn_start && !marked && i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) break;
|
|
if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] &&
|
|
bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS)
|
|
sites++;
|
|
else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE ||
|
|
(op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST))
|
|
early = 0;
|
|
i += l;
|
|
}
|
|
if (marked || !sites) return free(bind), free(set2), NULL;
|
|
*why = w[1] < 0x00010300 || w[1] > 0x00010600 ? "a SPIR-V version before 1.3 or after 1.6"
|
|
: !ann_end || !fn_start ? "no decorations or no function"
|
|
: vertex_stage && !marks_vs_ok ? "the GPU allows no stores in vertex-pipeline shaders"
|
|
: fragment_stage && !marks_fs_ok ? "the GPU allows no stores in fragment shaders"
|
|
: NULL;
|
|
if (*why) return free(bind), free(set2), *status = -1, NULL;
|
|
early = early && fragment_stage && entries == 1 && em_end && em_end < ann_end;
|
|
// From SPIR-V 1.4 on an entry point lists every global variable its functions use, not only the inputs and outputs
|
|
// (the translator's mesh shaders are 1.4): each entry point gets the marks variable, a word more.
|
|
int iface = w[1] >= 0x00010400;
|
|
// the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site
|
|
uint64_t on = n + 3 + (uint64_t)entries + 20 + 30 + 9 * sites;
|
|
uint32_t *o = malloc(on * 4);
|
|
if (!o) return free(bind), free(set2), *status = -1, *why = "no memory", NULL;
|
|
uint32_t next = bound;
|
|
uint32_t new_uint = uint_t ? 0 : (uint_t = next++), new_c0 = c0 ? 0 : (c0 = next++), new_c1 = c1 ? 0 : (c1 = next++);
|
|
uint32_t rta = next++, st = next++, ptr_st = next++, var = next++, new_ptr_u = ptr_u ? 0 : (ptr_u = next++);
|
|
uint64_t p = 5;
|
|
memcpy(o, w, 5 * 4);
|
|
for (uint64_t i = 5; i < ann_end;) { // the header up to the annotations' end: entry points widened, the mode added
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (early && i == em_end) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = fs_entry, o[p++] = EM_EARLY_FRAGMENT_TESTS;
|
|
memcpy(o + p, w + i, l * 4);
|
|
if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) o[p] = (l + 1) << 16 | op, o[p + l] = var, p++;
|
|
p += l;
|
|
i += l;
|
|
}
|
|
o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = rta, o[p++] = DEC_ARRAY_STRIDE, o[p++] = 4;
|
|
o[p++] = 5u << 16 | OP_MEMBER_DECORATE_, o[p++] = st, o[p++] = 0, o[p++] = DEC_OFFSET_, o[p++] = 0;
|
|
o[p++] = 3u << 16 | SPV_OP_DECORATE, o[p++] = st, o[p++] = DEC_BLOCK;
|
|
o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = var, o[p++] = SPV_DEC_DESCRIPTOR_SET, o[p++] = 2;
|
|
o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = var, o[p++] = SPV_DEC_BINDING, o[p++] = VK_POOL_MARK_BINDING;
|
|
memcpy(o + p, w + ann_end, (fn_start - ann_end) * 4), p += fn_start - ann_end;
|
|
if (new_uint) o[p++] = 4u << 16 | SPV_OP_TYPE_INT, o[p++] = uint_t, o[p++] = 32, o[p++] = 0;
|
|
if (new_c0) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c0, o[p++] = 0;
|
|
if (new_c1) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c1, o[p++] = 1;
|
|
o[p++] = 3u << 16 | OP_TYPE_RUNTIME_ARRAY, o[p++] = rta, o[p++] = uint_t;
|
|
o[p++] = 3u << 16 | OP_TYPE_STRUCT_, o[p++] = st, o[p++] = rta;
|
|
o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_st, o[p++] = SC_STORAGE_BUFFER, o[p++] = st;
|
|
if (new_ptr_u) o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_u, o[p++] = SC_STORAGE_BUFFER, o[p++] = uint_t;
|
|
o[p++] = 4u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_st, o[p++] = var, o[p++] = SC_STORAGE_BUFFER;
|
|
for (uint64_t i = fn_start; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
memcpy(o + p, w + i, l * 4), p += l;
|
|
if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] &&
|
|
bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) {
|
|
uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access
|
|
o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4];
|
|
o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1;
|
|
}
|
|
i += l;
|
|
}
|
|
o[3] = next;
|
|
free(bind), free(set2);
|
|
*out_n = p;
|
|
*status = 1;
|
|
return o;
|
|
}
|
|
|
|
// --selftest with NX2NAVI_SPV_MARK_TEST=<list>: each line "<in.spv> <out.spv>"; the module is moved to the pools
|
|
// (spirv_pool_rebind) and marked, and written to <out> when marked, for spirv-val. Prints the counts; 0 failures unless
|
|
// a file cannot be read or written.
|
|
static int spirv_mark_list_test(const char *list) {
|
|
marks_vs_ok = marks_fs_ok = 1;
|
|
FILE *f = fopen(list, "rb");
|
|
if (!f) return printf("spirv: pool marks test: %s cannot be read\n", list), 1;
|
|
char line[2048];
|
|
int marked = 0, none = 0, missed = 0, bad = 0;
|
|
while (fgets(line, sizeof line, f)) {
|
|
char in[1024], out[1024];
|
|
if (sscanf(line, "%1023s %1023s", in, out) != 2) continue;
|
|
FILE *g = fopen(in, "rb");
|
|
if (!g) { bad++; continue; }
|
|
fseek(g, 0, SEEK_END);
|
|
long sz = ftell(g);
|
|
fseek(g, 0, SEEK_SET);
|
|
uint32_t *w = sz > 20 ? malloc((size_t)sz) : NULL;
|
|
if (!w || fread(w, 1, (size_t)sz, g) != (size_t)sz) { fclose(g), free(w), bad++; continue; }
|
|
fclose(g);
|
|
uint64_t n = (uint64_t)sz / 4, mn = 0;
|
|
uint32_t *pooled = spirv_pool_rebind(w, n);
|
|
int st;
|
|
const char *why;
|
|
uint32_t *m = spirv_pool_mark(pooled ? pooled : w, n, 1, &mn, &st, &why);
|
|
if (st > 0) {
|
|
marked++;
|
|
if (!plat_file_write_atomic(out, m, (size_t)mn * 4)) {
|
|
if (!bad) printf("spirv: pool marks test: %s cannot be written\n", out);
|
|
bad++;
|
|
}
|
|
} else if (st < 0) {
|
|
missed++;
|
|
printf("spirv: pool marks test: %s not markable: %s\n", in, why);
|
|
} else {
|
|
none++;
|
|
}
|
|
free(m), free(pooled), free(w);
|
|
}
|
|
fclose(f);
|
|
printf("spirv: pool marks test: %d marked, %d without pool texture accesses, %d not markable, %d file errors\n", marked,
|
|
none, missed, bad);
|
|
return bad;
|
|
}
|
|
|
|
// early_ok: a fragment shader whose pipeline has no alpha to coverage (spirv_pool_mark)
|
|
static VkShaderModule make_module(const uint32_t *w, uint64_t words, int early_ok) {
|
|
const uint32_t *orig_w = w; // for the log of a module the marks cannot take
|
|
uint64_t orig_words = words, fixed_n = 0;
|
|
uint32_t *fixed = spirv_fixup(w, words, &fixed_n);
|
|
if (fixed) w = fixed, words = fixed_n;
|
|
uint32_t *pooled = spirv_pool_rebind(w, words);
|
|
if (pooled) {
|
|
free(fixed);
|
|
w = fixed = pooled;
|
|
if (vk_pool_marks_enabled()) {
|
|
uint64_t mk_n = 0;
|
|
int st;
|
|
const char *why;
|
|
uint32_t *mk = spirv_pool_mark(w, words, early_ok, &mk_n, &st, &why);
|
|
if (mk) {
|
|
free(fixed);
|
|
w = fixed = mk, words = mk_n;
|
|
} else if (st < 0) {
|
|
int nth = plat_inc32(&vk_pool_marks_missed);
|
|
// the hash is FNV-1a of the words make_module got, as vk_dump hashes a shader's SPIR-V; it differs
|
|
// from the translation's only where the pipeline patched the module first (added vertex outputs)
|
|
if (nth <= 8) {
|
|
uint32_t model = ~0u;
|
|
for (uint64_t i = 5; i < words;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l) break;
|
|
if (op == SPV_OP_ENTRY_POINT && l >= 2) { model = w[i + 1]; break; }
|
|
i += l;
|
|
}
|
|
const char *stage = model == 0 ? "vertex" : model <= 3 ? "tessellation or geometry" : model == 4 ? "fragment"
|
|
: model == 5 ? "compute" : model == 5364 ? "task" : model == 5365 ? "mesh" : "other";
|
|
rt_log("vk_pipeline: a %s shader that indexes the texture pool could not be marked (SPIR-V %u.%u, %llu "
|
|
"words, hash %016llx): %s%s", stage, w[1] >> 16 & 0xff, w[1] >> 8 & 0xff,
|
|
(unsigned long long)words, (unsigned long long)fnv(FNV0, orig_w, (size_t)orig_words * 4), why,
|
|
nth == 1 ? "; textures are not evicted by disuse this run (NX2NAVI_TEX_DISUSE)" : "");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
uint64_t rc_n = 0;
|
|
uint32_t *rc = P.reconverge ? spirv_reconverge(w, words, &rc_n) : NULL;
|
|
if (rc) {
|
|
free(fixed);
|
|
w = fixed = rc, words = rc_n;
|
|
}
|
|
uint64_t np_n = 0;
|
|
uint32_t *np = P.nan_preserve ? spirv_nan_preserve(w, words, P.nan_preserve, &np_n) : NULL;
|
|
if (np) {
|
|
free(fixed);
|
|
w = fixed = np, words = np_n;
|
|
}
|
|
if (P.feedback >= 2) mm_hash = fnv(FNV0, w, (size_t)words * 4);
|
|
VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)words * 4, .pCode = w};
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
VkResult r = vkCreateShaderModule(P.device, &ci, NULL, &m);
|
|
free(fixed);
|
|
return r == VK_SUCCESS ? m : VK_NULL_HANDLE;
|
|
}
|
|
|
|
// ---- descriptor layouts
|
|
|
|
typedef struct {
|
|
uint32_t n;
|
|
VkDescriptorSetLayoutBinding b[MAX_BINDINGS];
|
|
VkPipeBinding pb[MAX_BINDINGS];
|
|
int pool; // a stage indexes the texture or sampler pool (set 2, not in b / pb)
|
|
} Bindings;
|
|
|
|
static VkDescriptorType descriptor_type(int kind, const GfxShaderRes *r) {
|
|
int buffer = (r->type & 0xff) == 2;
|
|
switch (kind) {
|
|
case GFX_RES_UBO: return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
|
|
case GFX_RES_SSBO: return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
|
|
case GFX_RES_TEX:
|
|
if (buffer) return VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER;
|
|
if (r->separate) LOG_ONCE("vk_pipeline: a separate texture handle: bound as SAMPLED_IMAGE");
|
|
return r->separate ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
|
|
default: return buffer ? VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
}
|
|
}
|
|
|
|
static VkShaderStageFlags stage_bit(int stage) {
|
|
return stage == GFX_STAGE_VS ? VK_SHADER_STAGE_VERTEX_BIT : stage == GFX_STAGE_FS ? VK_SHADER_STAGE_FRAGMENT_BIT
|
|
: stage == GFX_STAGE_TS ? VK_SHADER_STAGE_TASK_BIT_EXT : stage == GFX_STAGE_MS ? VK_SHADER_STAGE_MESH_BIT_EXT
|
|
: VK_SHADER_STAGE_COMPUTE_BIT;
|
|
}
|
|
|
|
static int is_mesh(const VkPipeState *s) { return s->vs && gfx_shader_stage(s->vs) == GFX_STAGE_MS; }
|
|
|
|
// The shaders of the pipeline this thread builds, for the crash notes (platform.h plat_crash_context): the first 8
|
|
// bytes of each cache key, as gfx_shader.c names its cache files.
|
|
static void crash_note(const char *what, const GfxShader *a, const GfxShader *b, const GfxShader *c) {
|
|
static __thread char buf[160];
|
|
char *o = buf;
|
|
o += snprintf(o, 40, "building %s", what);
|
|
const GfxShader *sh[3] = {a, b, c};
|
|
for (int i = 0; i < 3; i++)
|
|
if (sh[i]) {
|
|
const uint8_t *k = gfx_shader_key(sh[i]);
|
|
o += snprintf(o, 40, " stage %d %02x%02x%02x%02x%02x%02x%02x%02x", gfx_shader_stage(sh[i]), k[0], k[1], k[2], k[3],
|
|
k[4], k[5], k[6], k[7]);
|
|
}
|
|
plat_crash_context(buf);
|
|
}
|
|
|
|
// One past the highest binding a shader uses (1 with no resources: binding 0 is the support buffer).
|
|
static uint32_t binding_span(const GfxShader *s) {
|
|
uint32_t span = 1;
|
|
for (int k = 0; k < GFX_RES_KINDS; k++) {
|
|
const GfxShaderRes *r;
|
|
int n = gfx_shader_resources(s, k, &r);
|
|
for (int i = 0; i < n; i++)
|
|
if (r[i].binding + (uint32_t)(r[i].count ? r[i].count : 1) > span) span = r[i].binding + (r[i].count ? r[i].count : 1);
|
|
}
|
|
return span;
|
|
}
|
|
|
|
// Adds a stage's resources with bindings b >= 1 moved by off; binding 0 (support buffer) is shared.
|
|
static int add_stage(Bindings *B, const GfxShader *s, uint32_t off) {
|
|
int stage = gfx_shader_stage(s);
|
|
VkShaderStageFlags bit = stage_bit(stage);
|
|
if (!B->n) {
|
|
B->b[0] = (VkDescriptorSetLayoutBinding){0, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1, 0, NULL};
|
|
B->pb[0] = (VkPipeBinding){0, (uint8_t)stage, VK_PIPE_SUPPORT, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1, 0, NULL};
|
|
B->n = 1;
|
|
}
|
|
B->b[0].stageFlags |= bit;
|
|
B->pb[0].stages |= bit;
|
|
for (int k = 0; k < GFX_RES_KINDS; k++) {
|
|
const GfxShaderRes *r;
|
|
int n = gfx_shader_resources(s, k, &r);
|
|
for (int i = 0; i < n; i++) {
|
|
if (r[i].pool) { // set 2 (spirv_pool_rebind), bound by the replay from the pools
|
|
B->pool = 1;
|
|
continue;
|
|
}
|
|
if (B->n == MAX_BINDINGS || !r[i].binding) return 0;
|
|
VkDescriptorType t = descriptor_type(k, &r[i]);
|
|
uint32_t binding = r[i].binding + off, count = r[i].count ? r[i].count : 1;
|
|
B->b[B->n] = (VkDescriptorSetLayoutBinding){binding, t, count, bit, NULL};
|
|
B->pb[B->n] = (VkPipeBinding){(uint16_t)binding, (uint8_t)stage, (uint8_t)k, t, (uint16_t)count, bit, &r[i]};
|
|
B->n++;
|
|
}
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
// The first binding a module's resource variables use that the binding list b[0..n) lacks (module bindings >= 1 moved
|
|
// by off), or -1. A reflection that misses a resource would otherwise hand the driver an invalid layout.
|
|
static int module_binding_missing(const GfxShader *s, const VkDescriptorSetLayoutBinding *b, uint32_t n, uint32_t off) {
|
|
uint64_t words = 0;
|
|
const uint32_t *w = gfx_shader_spirv(s, &words);
|
|
const GfxShaderRes *tr;
|
|
int nt = gfx_shader_resources(s, GFX_RES_TEX, &tr);
|
|
for (uint64_t i = 5; w && i < words;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > words) break;
|
|
int pooled = 0; // a pool array: set 2, not in this layout (spirv_pool_rebind)
|
|
for (int t = 0; t < nt && op == SPV_OP_DECORATE && len >= 4; t++)
|
|
if (tr[t].pool && tr[t].binding == (w[i + 3] & 0xffff)) pooled = 1; // GfxShaderRes.binding is 16 bits
|
|
if (op == SPV_OP_DECORATE && len >= 4 && w[i + 2] == SPV_DEC_BINDING && !pooled) {
|
|
uint32_t want = w[i + 3] + (w[i + 3] >= 1 ? off : 0), j = 0;
|
|
while (j < n && b[j].binding != want) j++;
|
|
if (j == n) return (int)w[i + 3];
|
|
}
|
|
i += len;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
// Checks both stages of a graphics pipeline against its layout; 0 with why filled when one uses a missing binding.
|
|
static int layout_covers(const VkPipeState *s, const Layout *l, uint32_t fs_off, char *why, size_t cap) {
|
|
uint32_t nv = l->set1 ? l->n0 : l->n;
|
|
int m = module_binding_missing(s->vs, l->b, nv, 0);
|
|
const char *stage = "vertex";
|
|
if (m < 0 && s->fs) {
|
|
m = l->set1 ? module_binding_missing(s->fs, l->b + l->n0, l->n - l->n0, 0) : module_binding_missing(s->fs, l->b, l->n, fs_off);
|
|
stage = "fragment";
|
|
}
|
|
if (m >= 0) snprintf(why, cap, "the %s shader uses binding %d, which its reflection and layout lack", stage, m);
|
|
return m < 0;
|
|
}
|
|
|
|
// The update template of a set whose bindings are b[0..n): one entry per binding, in order, over the packed array the
|
|
// replay fills from the set's first descriptor, a VkDescSlot per descriptor. The bind point follows the stages (compute
|
|
// layouts hold the compute stage only). VK_NULL_HANDLE when templates are off, for an empty set, for a push set
|
|
// without vkCmdPushDescriptorSetWithTemplateKHR or when creation fails: the replay then writes the set with write
|
|
// structs.
|
|
static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl,
|
|
int push, VkPipelineLayout pl, uint32_t set) {
|
|
if (!P.tpl || !n || (push && !P.tpl_push)) return VK_NULL_HANDLE;
|
|
VkDescriptorUpdateTemplateEntry e[MAX_BINDINGS];
|
|
size_t off = 0;
|
|
VkShaderStageFlags stages = 0;
|
|
for (uint32_t i = 0; i < n; i++) {
|
|
e[i] = (VkDescriptorUpdateTemplateEntry){b[i].binding, 0, b[i].descriptorCount, b[i].descriptorType, off, sizeof(VkDescSlot)};
|
|
off += sizeof(VkDescSlot) * b[i].descriptorCount;
|
|
stages |= b[i].stageFlags;
|
|
}
|
|
VkDescriptorUpdateTemplateCreateInfo ci = {
|
|
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_UPDATE_TEMPLATE_CREATE_INFO, .descriptorUpdateEntryCount = n,
|
|
.pDescriptorUpdateEntries = e,
|
|
.templateType = push ? VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR : VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET,
|
|
.descriptorSetLayout = sl,
|
|
.pipelineBindPoint = stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS,
|
|
.pipelineLayout = pl, .set = set};
|
|
VkDescriptorUpdateTemplate t;
|
|
if (P.tpl_create(P.device, &ci, NULL, &t) != VK_SUCCESS) {
|
|
LOG_ONCE("vk_pipeline: descriptor update template creation failed: that set is written with write structs");
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
return t;
|
|
}
|
|
|
|
// Finds or creates the layout of a binding list (sorted here); NULL on failure. Caller holds P.lock.
|
|
// Sorts a binding list by binding and hashes it into *h; the descriptor total, or -1 when a binding repeats.
|
|
static int64_t bindings_sort(Bindings *B, uint64_t *h) {
|
|
for (uint32_t i = 1; i < B->n; i++) // insertion sort by binding; the lists are short
|
|
for (uint32_t j = i; j > 0 && B->b[j - 1].binding > B->b[j].binding; j--) {
|
|
VkDescriptorSetLayoutBinding t = B->b[j]; B->b[j] = B->b[j - 1]; B->b[j - 1] = t;
|
|
VkPipeBinding u = B->pb[j]; B->pb[j] = B->pb[j - 1]; B->pb[j - 1] = u;
|
|
}
|
|
int64_t total = 0;
|
|
for (uint32_t i = 0; i < B->n; i++) {
|
|
if (i && B->b[i].binding == B->b[i - 1].binding) return -1;
|
|
uint32_t k[4] = {B->b[i].binding, B->b[i].descriptorType, B->b[i].descriptorCount, B->b[i].stageFlags};
|
|
*h = fnv(*h, k, sizeof k);
|
|
total += B->b[i].descriptorCount;
|
|
}
|
|
return total;
|
|
}
|
|
|
|
static Layout *layout_get(Bindings *B) {
|
|
uint64_t h = FNV0;
|
|
int64_t total = bindings_sort(B, &h);
|
|
if (total < 0) return NULL;
|
|
Layout **head = &layouts[h % LAYOUT_BUCKETS];
|
|
for (Layout *l = *head; l; l = l->next)
|
|
if (l->hash == h && l->n == B->n && !memcmp(l->b, B->b, B->n * sizeof *B->b)) return l;
|
|
Layout *l = calloc(1, sizeof *l + B->n * sizeof *B->b);
|
|
if (!l) return NULL;
|
|
l->hash = h, l->n = B->n;
|
|
memcpy(l->b, B->b, B->n * sizeof *B->b);
|
|
l->push = total <= (int64_t)P.max_push;
|
|
VkDescriptorSetLayoutCreateInfo si = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
.flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0,
|
|
.bindingCount = l->n, .pBindings = l->b};
|
|
if (vkCreateDescriptorSetLayout(P.device, &si, NULL, &l->set) != VK_SUCCESS) {
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
VkDescriptorSetLayout sets3[3] = {l->set, vk_empty_set_layout(), vk_pool_set_layout()}; // set 2: the pools
|
|
VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 3,
|
|
.pSetLayouts = sets3};
|
|
if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) {
|
|
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
l->tpl[0] = make_tpl(l->b, l->n, l->set, l->push, l->layout, 0);
|
|
l->next = *head;
|
|
*head = l;
|
|
P.stats.layouts++;
|
|
P.stats.push_layouts += (uint64_t)l->push;
|
|
return l;
|
|
}
|
|
|
|
// Independent layouts (P.indep): the vertex stage's bindings in set 0, the fragment stage's in set 1 (empty without a
|
|
// fragment shader), each with its own support buffer at binding 0, the pipeline layout created with
|
|
// VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT so stage libraries can use it. Only the P.push_stage set is a
|
|
// push-descriptor set (a layout holds one), when it fits maxPushDescriptors; the other is allocated per draw from the
|
|
// command buffer's pools. On success *B holds both lists, set 0 first, VkPipeBinding.set filled in. Caller holds P.lock.
|
|
static Layout *layout_pair(Bindings *B, Bindings *F) {
|
|
uint64_t h = fnv(FNV0, "indep", 5);
|
|
int64_t tv = bindings_sort(B, &h);
|
|
h = fnv(h, "|", 1);
|
|
int64_t tf = bindings_sort(F, &h);
|
|
if (tv < 0 || tf < 0 || B->n + F->n > MAX_BINDINGS) return NULL;
|
|
uint32_t n0 = B->n, n = B->n + F->n;
|
|
memcpy(B->b + n0, F->b, F->n * sizeof *F->b);
|
|
memcpy(B->pb + n0, F->pb, F->n * sizeof *F->pb);
|
|
for (uint32_t i = n0; i < n; i++) B->pb[i].set = 1;
|
|
B->n = n;
|
|
Layout **head = &layouts[h % LAYOUT_BUCKETS];
|
|
for (Layout *l = *head; l; l = l->next)
|
|
if (l->hash == h && l->set1 && l->n == n && l->n0 == n0 && !memcmp(l->b, B->b, n * sizeof *B->b)) return l;
|
|
Layout *l = calloc(1, sizeof *l + n * sizeof *B->b);
|
|
if (!l) return NULL;
|
|
l->hash = h, l->n = n, l->n0 = n0;
|
|
memcpy(l->b, B->b, n * sizeof *B->b);
|
|
l->push = P.push_stage == GFX_STAGE_VS && tv <= (int64_t)P.max_push;
|
|
l->push1 = P.push_stage == GFX_STAGE_FS && F->n && tf <= (int64_t)P.max_push;
|
|
VkDescriptorSetLayoutCreateInfo s0 = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
.flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0,
|
|
.bindingCount = n0, .pBindings = l->b};
|
|
VkDescriptorSetLayoutCreateInfo s1 = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
.flags = l->push1 ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0,
|
|
.bindingCount = n - n0, .pBindings = l->b + n0};
|
|
if (vkCreateDescriptorSetLayout(P.device, &s0, NULL, &l->set) != VK_SUCCESS) {
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
if (vkCreateDescriptorSetLayout(P.device, &s1, NULL, &l->set1) != VK_SUCCESS) {
|
|
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
VkDescriptorSetLayout sets[3] = {l->set, l->set1, vk_pool_set_layout()};
|
|
VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
|
|
.flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 3,
|
|
.pSetLayouts = sets};
|
|
if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) {
|
|
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
|
|
vkDestroyDescriptorSetLayout(P.device, l->set1, NULL);
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
l->tpl[0] = make_tpl(l->b, n0, l->set, l->push, l->layout, 0);
|
|
l->tpl[1] = make_tpl(l->b + n0, n - n0, l->set1, l->push1, l->layout, 1);
|
|
l->next = *head;
|
|
*head = l;
|
|
P.stats.layouts++;
|
|
P.stats.push_layouts += (uint64_t)(l->push | l->push1);
|
|
return l;
|
|
}
|
|
|
|
// Vertex outputs with independent layouts: each vertex shader declares the union of the inputs of every fragment
|
|
// shader met with it (pipeline-keys.bin at the prewarm, then each pipeline built), so its module does not depend on the
|
|
// fragment shader of one pipeline. Unwritten ones read (0, 0, 0, 1) as before. NX2NAVI_VS_OUTS=all declares all 32.
|
|
#define VS_OUTS_CAP 8192
|
|
static struct {
|
|
PlatMutex lock;
|
|
const GfxShader *vs[VS_OUTS_CAP];
|
|
uint32_t mask[VS_OUTS_CAP];
|
|
GfxShaderIo io[VS_OUTS_CAP][32];
|
|
uint32_t n, full;
|
|
} VO = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
// Adds fs's inputs (packed ones excepted) to vs's union; returns the union, types in io (the first type met wins).
|
|
static uint32_t vs_outs(const GfxShader *vs, const GfxShader *fs, GfxShaderIo io[32]) {
|
|
GfxShaderIo fin[32];
|
|
uint32_t need = fs ? gfx_shader_inputs(fs, fin) : 0, mask;
|
|
for (int i = 0; i < 32; i++)
|
|
if (need >> i & 1 && fin[i].packed) need &= ~(1u << i);
|
|
plat_mutex_lock(&VO.lock);
|
|
uint32_t j = (uint32_t)(((uintptr_t)vs >> 4) * 0x9e3779b1u) & (VS_OUTS_CAP - 1);
|
|
while (VO.vs[j] && VO.vs[j] != vs) j = (j + 1) & (VS_OUTS_CAP - 1);
|
|
if (!VO.vs[j] && VO.n + 1 >= VS_OUTS_CAP * 3 / 4) { // full: this pair alone
|
|
VO.full++;
|
|
plat_mutex_unlock(&VO.lock);
|
|
LOG_ONCE("vk_pipeline: vertex output union table full: outputs from the pair");
|
|
memset(io, 0, 32 * sizeof *io);
|
|
for (int i = 0; i < 32; i++)
|
|
if (need >> i & 1) io[i] = fin[i];
|
|
return need;
|
|
}
|
|
if (!VO.vs[j]) VO.vs[j] = vs, VO.n++;
|
|
for (int i = 0; i < 32; i++)
|
|
if (!(need >> i & 1)) continue;
|
|
else if (!(VO.mask[j] >> i & 1)) VO.io[j][i] = fin[i], VO.mask[j] |= 1u << i;
|
|
else if (VO.io[j][i].type == fin[i].type && VO.io[j][i].comps < fin[i].comps) VO.io[j][i].comps = fin[i].comps;
|
|
mask = VO.mask[j];
|
|
memcpy(io, VO.io[j], 32 * sizeof *io);
|
|
plat_mutex_unlock(&VO.lock);
|
|
return mask;
|
|
}
|
|
|
|
// ---- pipeline cache table
|
|
|
|
static Pipe *pipe_find(const void *key, size_t size, uint64_t h) {
|
|
for (Pipe *p = pipes[h % PIPE_BUCKETS]; p; p = p->next)
|
|
if (p->hash == h && p->key_size == size && !memcmp(p->key, key, size)) return p;
|
|
return NULL;
|
|
}
|
|
|
|
// Stores a result (a failed one too, so it is not retried every draw). Caller holds P.lock.
|
|
static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipeline, const Layout *l, const Bindings *B,
|
|
VkPipelineBindPoint bp, uint32_t vs_inputs) {
|
|
Pipe *p = calloc(1, sizeof *p + size);
|
|
if (!p) return NULL;
|
|
p->hash = h, p->key_size = size;
|
|
memcpy(p->key, key, size);
|
|
if (l && B) {
|
|
p->bindings = malloc(B->n * sizeof *p->bindings);
|
|
if (p->bindings) memcpy(p->bindings, B->pb, B->n * sizeof *p->bindings);
|
|
}
|
|
p->info = (VkPipeInfo){pipeline, l ? l->layout : VK_NULL_HANDLE, l ? l->set : VK_NULL_HANDLE, l ? l->push : 0, bp,
|
|
p->bindings ? B->n : 0, p->bindings, vs_inputs};
|
|
p->info.set_layout1 = l ? l->set1 : VK_NULL_HANDLE;
|
|
p->info.push1 = l ? l->push1 : 0;
|
|
p->info.n_sets = l && l->set1 ? 2 : 1;
|
|
p->info.pool = B ? B->pool : 0;
|
|
for (uint32_t i = 0; i < p->info.n_bindings; i++)
|
|
if (p->bindings[i].kind == GFX_RES_SSBO && p->bindings[i].res->cbuf_slot == VK_GPA_CB_SLOT) p->info.gpa = 1;
|
|
if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1];
|
|
p->lay = l;
|
|
Pipe **head = &pipes[h % PIPE_BUCKETS];
|
|
p->next = *head;
|
|
*head = p;
|
|
return p;
|
|
}
|
|
|
|
// ---- graphics pipelines
|
|
|
|
typedef struct { // normalized: zero where the state is dynamic or unused
|
|
const GfxShader *vs, *fs, *ts; // ts: the task stage of a mesh draw
|
|
uint32_t color[8], depth, stencil; // VkFormat
|
|
uint8_t n_color, samples, topo_class, minus_w;
|
|
uint8_t clamp, poly_mode, a2c, logic_op; // logic_op: VkLogicOp + 1 when enabled
|
|
uint8_t blend_en, dsb, fbl, pad; // dsb: an enabled target blends with a SRC1 factor; fbl: VkPipeState.feedback
|
|
uint32_t wmask;
|
|
uint8_t blend[8][6]; // src, dst, op, src alpha, dst alpha, alpha op (VkBlendFactor / VkBlendOp)
|
|
uint32_t attr_fmt[32], attr_off[32];
|
|
uint8_t attr_bind[32];
|
|
uint8_t rate[17]; // per binding: 0 unused, 1 vertex, 2 instance
|
|
uint8_t indep; // P.indep when built: the probe draws through both layouts
|
|
uint8_t pad2[2];
|
|
} GKey;
|
|
|
|
static int blendable(VkFormat f) {
|
|
return f != VK_FORMAT_UNDEFINED && (format_features(f, 1) & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT);
|
|
}
|
|
|
|
static int target_count(const VkPipeState *s) {
|
|
int n = 0;
|
|
for (int i = 0; i < 8; i++)
|
|
if (s->color_fmt[i]) n = i + 1;
|
|
return n;
|
|
}
|
|
|
|
// SRC1 blend factors read the fragment shader's second output (dual-source blending); the pipeline then needs a
|
|
// fragment module whose output 1 is Location 0, Index 1 (spirv_dual_source).
|
|
static int is_src1(unsigned f) { return f >= NVN_BLEND_FUNC_SRC1_COLOR && f <= NVN_BLEND_FUNC_ONE_MINUS_SRC1_ALPHA; }
|
|
// Runs per draw (gkey_fill, target_wmask): only the blend factors of blend-enabled targets are read until one is SRC1,
|
|
// which NX2navi never sets (0 of 876 pipelines in the field and fight saves).
|
|
// NX2NAVI_VK_NOOPT=1: pipelines built without the driver's optimiser, for a driver whose shader compiler crashes on
|
|
// one (the test switch for such a report); slower shaders, the same picture.
|
|
static VkPipelineCreateFlags noopt_flag(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = cfg_int("NX2NAVI_VK_NOOPT") != 0;
|
|
if (on) rt_log("vk_pipeline: pipelines built without the driver's optimiser (NX2NAVI_VK_NOOPT)");
|
|
}
|
|
return on ? VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT : 0;
|
|
}
|
|
|
|
// ---- pipeline creation feedback (NX2NAVI_PIPE_FEEDBACK, default 1; 0 off)
|
|
// Each pipeline, library and compute build asks the driver whether pipeline-cache.bin (P.cache) held it
|
|
// (VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) and how long the create took. The counts appear
|
|
// with the prewarm, the stage library prebuild and every cache save, so a log tells whether a slow boot compiled or
|
|
// read its pipelines: on NVIDIA the driver's own disk cache is kept per executable, so a renamed or new exe gets nothing
|
|
// from it, and only pipeline-cache.bin can make its first boot quick. =2 also logs every compute build with a hash of
|
|
// its final module, and every graphics pipeline build with the hashes of its final modules and of its fixed state, so
|
|
// two runs can be compared for a module or a state that changes between them.
|
|
enum { FB_GRAPHICS, FB_LIBRARY, FB_COMPUTE, FB_KINDS };
|
|
static struct {
|
|
PlatMutex lock;
|
|
uint64_t n[FB_KINDS], hit[FB_KINDS];
|
|
double hit_ms[FB_KINDS], miss_ms[FB_KINDS];
|
|
} FB = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
// Chains fb into a create info's pNext (next: its current pNext). Returns what the create info's pNext becomes.
|
|
static const void *fb_chain(VkPipelineCreationFeedbackCreateInfo *ci, VkPipelineCreationFeedback *fb, const void *next) {
|
|
if (!P.feedback) return next;
|
|
memset(fb, 0, sizeof *fb);
|
|
*ci = (VkPipelineCreationFeedbackCreateInfo){.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO,
|
|
.pNext = next, .pPipelineCreationFeedback = fb};
|
|
return ci;
|
|
}
|
|
|
|
static void fb_note(int kind, const VkPipelineCreationFeedback *fb) {
|
|
if (!P.feedback || !P.cache || !(fb->flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) return;
|
|
int hit = (fb->flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0;
|
|
double ms = (double)fb->duration / 1e6;
|
|
plat_mutex_lock(&FB.lock);
|
|
FB.n[kind]++, FB.hit[kind] += (uint64_t)hit;
|
|
if (hit) FB.hit_ms[kind] += ms;
|
|
else FB.miss_ms[kind] += ms;
|
|
plat_mutex_unlock(&FB.lock);
|
|
}
|
|
|
|
// One line: per kind, builds that came from pipeline-cache.bin of all builds so far, with the average time of each.
|
|
static void fb_log(const char *when) {
|
|
pb_log(when);
|
|
if (!P.feedback || !P.cache) return;
|
|
static const char *const kn[FB_KINDS] = {"pipelines", "libraries", "compute"};
|
|
char line[512];
|
|
int o = snprintf(line, sizeof line, "vk_pipeline: pipeline-cache.bin hits %s:", when);
|
|
plat_mutex_lock(&FB.lock);
|
|
for (int k = 0; k < FB_KINDS && o < (int)sizeof line - 96; k++) {
|
|
uint64_t miss = FB.n[k] - FB.hit[k];
|
|
o += snprintf(line + o, sizeof line - o, " %s %llu of %llu (%.0f%%; avg %.2f ms from the file, %.1f ms otherwise)%s", kn[k],
|
|
(unsigned long long)FB.hit[k], (unsigned long long)FB.n[k], FB.n[k] ? 100.0 * FB.hit[k] / FB.n[k] : 0.0,
|
|
FB.hit[k] ? FB.hit_ms[k] / FB.hit[k] : 0.0, miss ? FB.miss_ms[k] / miss : 0.0, k + 1 < FB_KINDS ? ";" : "");
|
|
}
|
|
plat_mutex_unlock(&FB.lock);
|
|
rt_log("%s", line);
|
|
}
|
|
|
|
static int uses_src1(const VkPipeState *s) {
|
|
for (uint32_t m = s->color.blend_en & 0xffu; m; m &= m - 1) {
|
|
int i = __builtin_ctz(m);
|
|
const blend_st *b = &s->blend[i];
|
|
if ((is_src1(b->src_rgb) || is_src1(b->dst_rgb) || is_src1(b->src_a) || is_src1(b->dst_a)) &&
|
|
i < target_count(s) && blendable(target_format(s->color_fmt[i], NULL, NULL)))
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Colour write mask of target i: NVN's channel mask, zero for a target outside the fragment shader's output map (SPH
|
|
// omap). Maxwell leaves such a target untouched;
|
|
// Vulkan writes undefined values, and on AMD they lit grass in front of Link as flat lime. Zeros are wrong
|
|
// too: NX2NAVI's half-res transparency pass binds its linear depth (R32F) as target 7, samples it in the same draws and
|
|
// writes only targets 0-2, 4 and 5; zeroed, the depth that later passes rebuild from it was 0 in tiles, and the fog
|
|
// over distant geometry showed rectangles of clear sky.
|
|
// The same holds per component: a shader that writes a target's RGB and not its alpha leaves the alpha alone on
|
|
// Maxwell, while Vulkan writes undefined values, and AMD's driver writes whatever its export registers held (NaN and
|
|
// saturated pixels the bloom then spreads). Components outside the shader's omap are masked out, where it is known.
|
|
static uint32_t target_wmask(const VkPipeState *s, int i) {
|
|
uint32_t m = s->chmask.mask >> (4 * i) & 0xf;
|
|
if (s->fs && !(gfx_shader_outputs(s->fs, NULL) >> i & 1)) m = 0;
|
|
if (s->fs) {
|
|
uint32_t c = gfx_shader_omap(s->fs) >> (4 * i) & 0xf;
|
|
if (c) m &= c;
|
|
}
|
|
if (i == 1 && s->fs && P.f_dual_src && uses_src1(s)) m = 0; // output 1 is the second blend source (spirv_dual_source)
|
|
return m;
|
|
}
|
|
|
|
uint32_t vk_pipeline_target_wmask(const VkPipeState *s, int i) { return target_wmask(s, i); }
|
|
|
|
static void gkey_fill(const VkPipeState *s, GKey *k) {
|
|
memset(k, 0, sizeof *k);
|
|
k->vs = s->vs, k->fs = s->fs, k->ts = s->ts;
|
|
k->indep = (uint8_t)P.indep;
|
|
k->n_color = (uint8_t)target_count(s);
|
|
for (int i = 0; i < k->n_color; i++) k->color[i] = target_format(s->color_fmt[i], NULL, NULL);
|
|
int d, st;
|
|
VkFormat df = target_format(s->depth_fmt, &d, &st);
|
|
k->depth = d ? df : VK_FORMAT_UNDEFINED;
|
|
k->stencil = st ? df : VK_FORMAT_UNDEFINED;
|
|
k->samples = s->samples > 1 ? s->samples : 1;
|
|
k->topo_class = (uint8_t)topology_class(vk_pipeline_topology(s->prim));
|
|
k->minus_w = s->depth_minus_w != 0;
|
|
k->dsb = s->fs && P.f_dual_src && uses_src1(s);
|
|
k->fbl = s->fs ? s->feedback : 0;
|
|
if (!P.dyn_clamp) k->clamp = s->depth_clamp && P.f_depth_clamp;
|
|
if (!P.dyn_poly) k->poly_mode = (uint8_t)polygon_mode(s->poly.mode);
|
|
if (!P.dyn_a2c) k->a2c = s->ms.a2c != 0;
|
|
if (P.f_logic_op && s->color.logic_op != NVN_LOGIC_OP_COPY) k->logic_op = (uint8_t)(s->color.logic_op + 1);
|
|
for (int i = 0; i < k->n_color; i++) {
|
|
int on = (s->color.blend_en >> i & 1) && blendable(k->color[i]);
|
|
if (!P.dyn_blend_en && on) k->blend_en |= (uint8_t)(1u << i);
|
|
if (!P.dyn_wmask) k->wmask |= target_wmask(s, i) << (4 * i);
|
|
if (!P.dyn_blend_eq && on) {
|
|
const blend_st *b = &s->blend[i];
|
|
uint8_t *o = k->blend[i];
|
|
o[0] = (uint8_t)blend_factor(b->src_rgb), o[1] = (uint8_t)blend_factor(b->dst_rgb), o[2] = (uint8_t)blend_op(b->eq_rgb);
|
|
o[3] = (uint8_t)blend_factor(b->src_a), o[4] = (uint8_t)blend_factor(b->dst_a), o[5] = (uint8_t)blend_op(b->eq_a);
|
|
}
|
|
}
|
|
if (is_mesh(s)) k->topo_class = 2; // mesh draws: no vertex input, the mesh stage declares its primitives
|
|
if (!P.dyn_vi && !is_mesh(s)) {
|
|
VtxInput v;
|
|
vertex_input(s, &v);
|
|
for (uint32_t i = 0; i < v.n_attr; i++) {
|
|
uint32_t l = v.a[i].location;
|
|
k->attr_fmt[l] = v.a[i].format, k->attr_off[l] = v.a[i].offset, k->attr_bind[l] = (uint8_t)v.a[i].binding;
|
|
}
|
|
for (uint32_t i = 0; i < v.n_bind; i++)
|
|
k->rate[v.b[i].binding] = v.b[i].inputRate == VK_VERTEX_INPUT_RATE_INSTANCE ? 2 : 1;
|
|
}
|
|
}
|
|
|
|
// Independent layouts: the outputs the vertex shader lacks from its union with s->fs's inputs added (vs_outs), types in
|
|
// fin. The vertex module (and its pre-rasterization library) depends on the vertex shader and this mask only.
|
|
static uint32_t vs_union_missing(const VkPipeState *s, GfxShaderIo fin[32]) {
|
|
GfxShaderIo vout[32];
|
|
uint32_t need = vs_outs(s->vs, s->fs, fin), have = gfx_shader_outputs(s->vs, vout), missing = 0;
|
|
if (s->fs) { // a location another fragment shader reads as another type: this pair's type (rare)
|
|
GfxShaderIo fi[32];
|
|
uint32_t fneed = gfx_shader_inputs(s->fs, fi);
|
|
for (int i = 0; i < 32; i++)
|
|
if (fneed >> i & 1 && !fi[i].packed &&
|
|
(!(need >> i & 1) || fin[i].type != fi[i].type || fin[i].comps < fi[i].comps)) {
|
|
if (need >> i & 1 && fin[i].type != fi[i].type)
|
|
LOG_ONCE("vk_pipeline: varying %d read as type %d and %d by fragment shaders of one vertex shader", i, fin[i].type, fi[i].type);
|
|
fin[i] = fi[i], need |= 1u << i;
|
|
}
|
|
}
|
|
if (P.vs_outs_all)
|
|
for (int i = 0; i < 32; i++)
|
|
if (!(need >> i & 1)) fin[i] = (GfxShaderIo){GFX_TYPE_FLOAT, 4, 0}, need |= 1u << i;
|
|
for (int i = 0; i < 32; i++)
|
|
if (need >> i & 1 && !(have >> i & 1)) missing |= 1u << i;
|
|
return missing;
|
|
}
|
|
|
|
static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Layout *l, uint32_t fs_off) {
|
|
VkShaderModule vm = VK_NULL_HANDLE, fm = VK_NULL_HANDLE;
|
|
uint64_t vw, fw, vm_hash = 0, fm_hash = 0; // the hashes: of the final modules, for the PIPE_FEEDBACK 2 line
|
|
const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw);
|
|
uint32_t *vpatched = NULL, *fpatched = NULL;
|
|
int mesh = is_mesh(s);
|
|
VkShaderModule tm = VK_NULL_HANDLE;
|
|
if (mesh && s->ts) { // the task stage's bindings follow the fragment stage's (pipe_miss)
|
|
uint64_t tw;
|
|
const uint32_t *tcode = gfx_shader_spirv(s->ts, &tw);
|
|
uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0);
|
|
uint32_t *t = tcode && toff ? spirv_rebind(tcode, tw, toff) : NULL;
|
|
tm = make_module(t ? t : tcode, tw, 0);
|
|
free(t);
|
|
}
|
|
if (l->set1 && !mesh) { // independent layouts: outputs from the vertex shader's union, not from this fragment shader
|
|
GfxShaderIo fin[32];
|
|
uint32_t missing = vs_union_missing(s, fin);
|
|
if (missing) {
|
|
uint64_t n2 = 0;
|
|
vpatched = spirv_add_outputs(vcode, vw, missing, fin, &n2);
|
|
if (!vpatched) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", missing);
|
|
else vcode = vpatched, vw = n2;
|
|
}
|
|
}
|
|
if (s->fs) {
|
|
GfxShaderIo fin[32], vout[32];
|
|
uint32_t need = gfx_shader_inputs(s->fs, fin), have = mesh ? ~0u : gfx_shader_outputs(s->vs, vout), missing = 0;
|
|
for (int i = 0; i < 32; i++) {
|
|
if (!(need >> i & 1) || fin[i].packed) continue;
|
|
if (!(have >> i & 1)) missing |= 1u << i;
|
|
else if (!mesh && vout[i].type != fin[i].type)
|
|
LOG_ONCE("vk_pipeline: varying %d is type %d in the vertex stage, %d in the fragment stage", i, vout[i].type, fin[i].type);
|
|
}
|
|
if (missing && !l->set1) {
|
|
uint64_t n2 = 0;
|
|
vpatched = spirv_add_outputs(vcode, vw, missing, fin, &n2);
|
|
if (!vpatched) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", missing);
|
|
else vcode = vpatched, vw = n2;
|
|
}
|
|
const uint32_t *fcode = gfx_shader_spirv(s->fs, &fw);
|
|
if (k->dsb) {
|
|
uint64_t n2 = 0;
|
|
uint32_t *d = spirv_dual_source(fcode, fw, &n2);
|
|
if (d) fcode = fpatched = d, fw = n2;
|
|
else LOG_ONCE("vk_pipeline: SRC1 blend factors but the fragment shader writes no colour output 1");
|
|
}
|
|
if (l->set1 || fs_off) {
|
|
uint32_t *r = !fcode ? NULL : l->set1 ? spirv_set(fcode, fw, 1) : spirv_rebind(fcode, fw, fs_off);
|
|
free(fpatched);
|
|
fcode = fpatched = r;
|
|
}
|
|
// A bound colour target outside the shader's output map gets write mask 0 (target_wmask)
|
|
if (fcode) fm = make_module(fcode, fw, !k->a2c), fm_hash = mm_hash;
|
|
}
|
|
vm = make_module(vcode, vw, 0), vm_hash = mm_hash;
|
|
free(vpatched), free(fpatched);
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (!vm || (s->fs && !fm) || (mesh && s->ts && !tm)) goto out;
|
|
|
|
VkPipelineShaderStageCreateInfo stages[3] = {
|
|
{.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = mesh ? VK_SHADER_STAGE_MESH_BIT_EXT : VK_SHADER_STAGE_VERTEX_BIT,
|
|
.module = vm, .pName = "main"},
|
|
{.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_FRAGMENT_BIT, .module = fm, .pName = "main"},
|
|
{.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_TASK_BIT_EXT, .module = tm, .pName = "main"}};
|
|
uint32_t n_stages = 1;
|
|
if (s->fs) n_stages = 2;
|
|
if (tm) {
|
|
if (!s->fs) stages[1] = stages[2];
|
|
n_stages++;
|
|
}
|
|
VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqv, rqf;
|
|
uint64_t sw;
|
|
const uint32_t *scode = gfx_shader_spirv(s->vs, &sw); // vcode may be a patched copy that is freed by now
|
|
pin_subgroup32(&stages[0], &rqv, scode, sw);
|
|
if (s->fs) pin_subgroup32(&stages[1], &rqf, gfx_shader_spirv(s->fs, &sw), sw);
|
|
VkVertexInputAttributeDescription va[32];
|
|
VkVertexInputBindingDescription vb[17];
|
|
uint32_t na = 0, nb = 0;
|
|
if (!P.dyn_vi) {
|
|
VtxInput v;
|
|
vertex_input(s, &v);
|
|
for (uint32_t i = 0; i < v.n_attr; i++) va[na++] = (VkVertexInputAttributeDescription){v.a[i].location, v.a[i].binding, v.a[i].format, v.a[i].offset};
|
|
for (uint32_t i = 0; i < v.n_bind; i++) vb[nb++] = (VkVertexInputBindingDescription){v.b[i].binding, v.b[i].stride, v.b[i].inputRate};
|
|
}
|
|
VkPipelineVertexInputStateCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
|
|
.vertexBindingDescriptionCount = nb, .pVertexBindingDescriptions = vb,
|
|
.vertexAttributeDescriptionCount = na, .pVertexAttributeDescriptions = va};
|
|
static const VkPrimitiveTopology class_base[4] = {VK_PRIMITIVE_TOPOLOGY_POINT_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST,
|
|
VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_PATCH_LIST};
|
|
VkPipelineInputAssemblyStateCreateInfo ia = {.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
|
|
.topology = class_base[k->topo_class]};
|
|
VkPipelineViewportDepthClipControlCreateInfoEXT clip = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT,
|
|
.negativeOneToOne = k->minus_w};
|
|
VkPipelineViewportStateCreateInfo vp = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pNext = &clip};
|
|
// NVN's default is the last vertex (SetProvokingVertexMode, nvn_ext.c); Vulkan's is the first.
|
|
VkPipelineRasterizationProvokingVertexStateCreateInfoEXT pvs = {
|
|
.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_PROVOKING_VERTEX_STATE_CREATE_INFO_EXT,
|
|
.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT};
|
|
VkPipelineRasterizationStateCreateInfo rs = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
|
|
.pNext = P.pv_last ? &pvs : NULL,
|
|
.depthClampEnable = k->clamp, .polygonMode = (VkPolygonMode)k->poly_mode,
|
|
.lineWidth = 1.0f};
|
|
VkPipelineMultisampleStateCreateInfo ms = {.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
|
|
.rasterizationSamples = (VkSampleCountFlagBits)k->samples,
|
|
.alphaToCoverageEnable = k->a2c};
|
|
VkPipelineDepthStencilStateCreateInfo ds = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
|
|
.maxDepthBounds = 1.0f};
|
|
VkPipelineColorBlendAttachmentState cba[8];
|
|
memset(cba, 0, sizeof cba);
|
|
for (int i = 0; i < k->n_color; i++) {
|
|
const uint8_t *b = k->blend[i];
|
|
cba[i].blendEnable = k->blend_en >> i & 1;
|
|
cba[i].srcColorBlendFactor = (VkBlendFactor)b[0], cba[i].dstColorBlendFactor = (VkBlendFactor)b[1];
|
|
cba[i].colorBlendOp = (VkBlendOp)b[2], cba[i].srcAlphaBlendFactor = (VkBlendFactor)b[3];
|
|
cba[i].dstAlphaBlendFactor = (VkBlendFactor)b[4], cba[i].alphaBlendOp = (VkBlendOp)b[5];
|
|
cba[i].colorWriteMask = k->wmask >> (4 * i) & 0xf;
|
|
}
|
|
VkPipelineColorBlendStateCreateInfo cb = {.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
|
|
.logicOpEnable = k->logic_op != 0,
|
|
.logicOp = k->logic_op ? (VkLogicOp)(k->logic_op - 1) : VK_LOGIC_OP_COPY,
|
|
.attachmentCount = k->n_color, .pAttachments = cba};
|
|
VkPipelineDynamicStateCreateInfo dy = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
|
|
.dynamicStateCount = P.n_dyn, .pDynamicStates = P.dyn};
|
|
VkDynamicState mdyn[sizeof P.dyn / sizeof P.dyn[0] + 1];
|
|
if (mesh) { // a mesh pipeline must not list vertex input, topology or primitive restart as dynamic
|
|
uint32_t n = 0;
|
|
for (uint32_t i = 0; i < P.n_dyn; i++)
|
|
if (P.dyn[i] != VK_DYNAMIC_STATE_VERTEX_INPUT_EXT && P.dyn[i] != VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE &&
|
|
P.dyn[i] != VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY && P.dyn[i] != VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE)
|
|
mdyn[n++] = P.dyn[i];
|
|
if (vk_ctx()->has_vrs) mdyn[n++] = VK_DYNAMIC_STATE_FRAGMENT_SHADING_RATE_KHR; // vk_exec_draw.c mesh_shading_rate
|
|
dy.dynamicStateCount = n, dy.pDynamicStates = mdyn;
|
|
}
|
|
VkPipelineRenderingCreateInfo ri = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, .colorAttachmentCount = k->n_color,
|
|
.pColorAttachmentFormats = (const VkFormat *)k->color,
|
|
.depthAttachmentFormat = (VkFormat)k->depth, .stencilAttachmentFormat = (VkFormat)k->stencil};
|
|
VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &ri,
|
|
.stageCount = n_stages, .pStages = stages, .pVertexInputState = mesh ? NULL : &vi,
|
|
.pInputAssemblyState = mesh ? NULL : &ia, .pViewportState = &vp, .pRasterizationState = &rs,
|
|
.pMultisampleState = &ms, .pDepthStencilState = &ds, .pColorBlendState = &cb,
|
|
.pDynamicState = &dy, .layout = l->layout, .flags = noopt_flag()};
|
|
if (k->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
|
if (k->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
|
VkPipelineCreationFeedbackCreateInfo fci;
|
|
VkPipelineCreationFeedback fb;
|
|
pi.pNext = fb_chain(&fci, &fb, pi.pNext);
|
|
uint64_t p0 = perf_begin();
|
|
crash_note("a graphics pipeline", s->vs, s->fs, s->ts);
|
|
VkResult r = pipe_create(0, &pi, &pipe);
|
|
plat_crash_context(NULL);
|
|
perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it
|
|
if (r != VK_SUCCESS) {
|
|
rt_log("vk_pipeline: vkCreateGraphicsPipelines failed (%d)", (int)r);
|
|
pipe = VK_NULL_HANDLE;
|
|
} else {
|
|
perf_count(PERF_CNT_PIPE_BUILT);
|
|
fb_note(FB_GRAPHICS, &fb);
|
|
if (P.feedback >= 2) { // the state hash: what the create info takes from the key besides the modules
|
|
uint64_t st = fnv(fnv(FNV0, cba, sizeof cba), k->color, sizeof k->color);
|
|
if (!P.dyn_vi) // the vertex layout is pipeline state then (zero in a mesh pipeline's key)
|
|
st = fnv(fnv(fnv(fnv(st, k->attr_fmt, sizeof k->attr_fmt), k->attr_off, sizeof k->attr_off),
|
|
k->attr_bind, sizeof k->attr_bind), k->rate, sizeof k->rate);
|
|
uint32_t bits[] = {(uint32_t)mesh, k->topo_class, k->minus_w, k->clamp, k->poly_mode, k->samples, k->a2c,
|
|
k->logic_op, k->n_color, k->fbl, k->dsb, l->set1 != 0, fs_off, pi.flags, k->depth, k->stencil};
|
|
rt_log("vk_pipeline: graphics: modules %016llx %016llx, state %016llx (mesh %d, topology class %u, samples %u, "
|
|
"colors %u, feedback loop %u, depth format %u, stage-independent layout %d, flags 0x%x), %s, %.1f ms",
|
|
(unsigned long long)vm_hash, (unsigned long long)fm_hash, (unsigned long long)fnv(st, bits, sizeof bits),
|
|
mesh, (unsigned)k->topo_class, (unsigned)k->samples, (unsigned)k->n_color, (unsigned)k->fbl,
|
|
(unsigned)k->depth, l->set1 != 0, (unsigned)pi.flags,
|
|
pb_last ? "from pipeline-binaries.bin" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled",
|
|
(double)fb.duration / 1e6);
|
|
}
|
|
}
|
|
out:
|
|
if (vm) vkDestroyShaderModule(P.device, vm, NULL);
|
|
if (fm) vkDestroyShaderModule(P.device, fm, NULL);
|
|
if (tm) vkDestroyShaderModule(P.device, tm, NULL);
|
|
return pipe;
|
|
}
|
|
|
|
// ---- graphics pipeline library (vk_caps.gpl)
|
|
// A pipeline is linked from four libraries, each cached by the part of the pipeline key it depends on: vertex input
|
|
// (topology class, vertex attributes without dynamic vertex input), pre-rasterization (vertex shader, the outputs its
|
|
// union adds, static rasterization state), fragment shader (fragment shader, dual-source rewrite, samples) and fragment
|
|
// output (targets, blend, samples). With independent layouts each stage's set depends on its shader alone, so the
|
|
// pre-rasterization library uses {set 0, null} and the fragment shader library {null, set 1}. The fast link runs on the
|
|
// thread of the draw that needs it (no link-time optimisation); the optimised pipeline is the job pipe_miss queued
|
|
// (a monolithic build with the same modules), which replaces the fast one in the Pipe when done. Libraries and fast
|
|
// pipelines live until vk_pipeline_shutdown.
|
|
// Background build signalling, both with P.lock: work_cv wakes the pipeline workers, done_cv wakes threads waiting for a
|
|
// pipeline or a stage library to be published.
|
|
static PlatCond work_cv = PLAT_COND_INIT, done_cv = PLAT_COND_INIT;
|
|
static PlatCond lib_cv = PLAT_COND_INIT; // wakes the library workers (lib_worker): work_cv's signal must reach a pool worker
|
|
#define LIB_BUCKETS 1024
|
|
enum { LIB_VI, LIB_PR, LIB_FS, LIB_FO };
|
|
typedef struct Lib { struct Lib *next; uint64_t hash; VkPipeline pipe; uint32_t size; uint8_t pre, used, building, pad;
|
|
struct LibJob *job; uint8_t key[]; } Lib;
|
|
// pre: built by the boot prebuild ("stage library prebuild" below); used: a draw's link found it; building: a thread
|
|
// builds it or job holds it queued (pipe not set yet); entries live until libs_destroy
|
|
static Lib *libs[LIB_BUCKETS];
|
|
|
|
typedef struct { uint32_t kind, topo_class, attr_fmt[32], attr_off[32]; uint8_t attr_bind[32], rate[17], pad[3]; } ViKey;
|
|
typedef struct { uint32_t kind, missing; const GfxShader *vs; uint8_t clamp, poly_mode, minus_w, pad; GfxShaderIo fin[32]; } PrKey;
|
|
typedef struct { uint32_t kind; uint8_t dsb, samples, a2c, pad; const GfxShader *fs; } FsKey;
|
|
typedef struct { uint32_t kind; uint8_t n_color, samples, a2c, logic_op, blend_en, pad[3]; uint32_t color[8], depth, stencil, wmask;
|
|
uint8_t blend[8][6]; } FoKey;
|
|
|
|
static uint64_t qpc(void) { return plat_now_ns(); }
|
|
static double ms_since(uint64_t t0) { return (double)(qpc() - t0) / 1e6; }
|
|
|
|
static VkPipelineLayout half_layout(Layout *l, int h) {
|
|
if (l->half[h]) return l->half[h];
|
|
VkDescriptorSetLayout sets[3] = {h ? VK_NULL_HANDLE : l->set, h ? l->set1 : VK_NULL_HANDLE, vk_pool_set_layout()};
|
|
VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
|
|
.flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 3,
|
|
.pSetLayouts = sets};
|
|
if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->half[h]) != VK_SUCCESS) l->half[h] = VK_NULL_HANDLE;
|
|
return l->half[h];
|
|
}
|
|
|
|
// Builds one library from the pipeline's state; key is the library's key (PrKey: fin holds the added outputs).
|
|
static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPipelineLayout lay, const void *key) {
|
|
static const VkGraphicsPipelineLibraryFlagsEXT part[4] = {
|
|
VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT, VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT,
|
|
VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT, VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT};
|
|
VkGraphicsPipelineLibraryCreateInfoEXT gl = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, .flags = part[kind]};
|
|
VkPipelineRenderingCreateInfo ri = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, .pNext = &gl};
|
|
VkPipelineDynamicStateCreateInfo dy = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
|
|
.dynamicStateCount = P.n_dyn, .pDynamicStates = P.dyn};
|
|
VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &ri,
|
|
.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | noopt_flag(), .pDynamicState = &dy};
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
VkPipelineShaderStageCreateInfo st = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pName = "main"};
|
|
VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rq;
|
|
VkVertexInputAttributeDescription va[32];
|
|
VkVertexInputBindingDescription vb[17];
|
|
VkPipelineVertexInputStateCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO};
|
|
static const VkPrimitiveTopology class_base[4] = {VK_PRIMITIVE_TOPOLOGY_POINT_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST,
|
|
VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_PATCH_LIST};
|
|
VkPipelineInputAssemblyStateCreateInfo ia = {.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
|
|
.topology = class_base[k->topo_class]};
|
|
VkPipelineViewportDepthClipControlCreateInfoEXT clip = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT,
|
|
.negativeOneToOne = k->minus_w};
|
|
VkPipelineViewportStateCreateInfo vp = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pNext = &clip};
|
|
VkPipelineRasterizationProvokingVertexStateCreateInfoEXT pvs = {
|
|
.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_PROVOKING_VERTEX_STATE_CREATE_INFO_EXT,
|
|
.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT};
|
|
VkPipelineRasterizationStateCreateInfo rs = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
|
|
.pNext = P.pv_last ? &pvs : NULL, .depthClampEnable = k->clamp,
|
|
.polygonMode = (VkPolygonMode)k->poly_mode, .lineWidth = 1.0f};
|
|
VkPipelineMultisampleStateCreateInfo ms = {.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
|
|
.rasterizationSamples = (VkSampleCountFlagBits)k->samples,
|
|
.alphaToCoverageEnable = k->a2c};
|
|
VkPipelineDepthStencilStateCreateInfo ds = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
|
|
.maxDepthBounds = 1.0f};
|
|
VkPipelineColorBlendAttachmentState cba[8];
|
|
VkPipelineColorBlendStateCreateInfo cb = {.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO};
|
|
uint32_t *patched = NULL;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (kind == LIB_VI) {
|
|
if (!P.dyn_vi) {
|
|
VtxInput v;
|
|
vertex_input(s, &v);
|
|
for (uint32_t i = 0; i < v.n_attr; i++)
|
|
va[i] = (VkVertexInputAttributeDescription){v.a[i].location, v.a[i].binding, v.a[i].format, v.a[i].offset};
|
|
for (uint32_t i = 0; i < v.n_bind; i++)
|
|
vb[i] = (VkVertexInputBindingDescription){v.b[i].binding, v.b[i].stride, v.b[i].inputRate};
|
|
vi.vertexAttributeDescriptionCount = v.n_attr, vi.pVertexAttributeDescriptions = va;
|
|
vi.vertexBindingDescriptionCount = v.n_bind, vi.pVertexBindingDescriptions = vb;
|
|
}
|
|
pi.pVertexInputState = &vi, pi.pInputAssemblyState = &ia;
|
|
} else if (kind == LIB_PR) {
|
|
const PrKey *pk = key;
|
|
uint64_t vw, sw, n2 = 0;
|
|
const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw), *scode = vcode;
|
|
sw = vw;
|
|
if (pk->missing && (patched = spirv_add_outputs(vcode, vw, pk->missing, pk->fin, &n2))) vcode = patched, vw = n2;
|
|
else if (pk->missing) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", pk->missing);
|
|
if (!(m = make_module(vcode, vw, 0))) goto out;
|
|
st.stage = VK_SHADER_STAGE_VERTEX_BIT, st.module = m;
|
|
pin_subgroup32(&st, &rq, scode, sw);
|
|
pi.stageCount = 1, pi.pStages = &st, pi.pViewportState = &vp, pi.pRasterizationState = &rs;
|
|
pi.layout = lay;
|
|
} else if (kind == LIB_FS) {
|
|
if (s->fs) {
|
|
uint64_t fw, sw, n2 = 0;
|
|
const uint32_t *fcode = gfx_shader_spirv(s->fs, &fw), *scode = fcode;
|
|
sw = fw;
|
|
uint32_t *d = NULL;
|
|
if (k->dsb && fcode) {
|
|
if ((d = spirv_dual_source(fcode, fw, &n2))) fcode = d, fw = n2;
|
|
else LOG_ONCE("vk_pipeline: SRC1 blend factors but the fragment shader writes no colour output 1");
|
|
}
|
|
patched = fcode ? spirv_set(fcode, fw, 1) : NULL;
|
|
free(d);
|
|
if (!patched || !(m = make_module(patched, fw, !k->a2c))) goto out;
|
|
st.stage = VK_SHADER_STAGE_FRAGMENT_BIT, st.module = m;
|
|
pin_subgroup32(&st, &rq, scode, sw);
|
|
pi.stageCount = 1, pi.pStages = &st;
|
|
}
|
|
pi.pMultisampleState = &ms, pi.pDepthStencilState = &ds;
|
|
pi.layout = lay;
|
|
} else {
|
|
memset(cba, 0, sizeof cba);
|
|
for (int i = 0; i < k->n_color; i++) {
|
|
const uint8_t *b = k->blend[i];
|
|
cba[i].blendEnable = k->blend_en >> i & 1;
|
|
cba[i].srcColorBlendFactor = (VkBlendFactor)b[0], cba[i].dstColorBlendFactor = (VkBlendFactor)b[1];
|
|
cba[i].colorBlendOp = (VkBlendOp)b[2], cba[i].srcAlphaBlendFactor = (VkBlendFactor)b[3];
|
|
cba[i].dstAlphaBlendFactor = (VkBlendFactor)b[4], cba[i].alphaBlendOp = (VkBlendOp)b[5];
|
|
cba[i].colorWriteMask = k->wmask >> (4 * i) & 0xf;
|
|
}
|
|
cb.logicOpEnable = k->logic_op != 0, cb.logicOp = k->logic_op ? (VkLogicOp)(k->logic_op - 1) : VK_LOGIC_OP_COPY;
|
|
cb.attachmentCount = k->n_color, cb.pAttachments = cba;
|
|
ri.colorAttachmentCount = k->n_color, ri.pColorAttachmentFormats = (const VkFormat *)k->color;
|
|
ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil;
|
|
pi.pColorBlendState = &cb, pi.pMultisampleState = &ms;
|
|
}
|
|
VkPipelineCreationFeedbackCreateInfo fci;
|
|
VkPipelineCreationFeedback fb;
|
|
pi.pNext = fb_chain(&fci, &fb, pi.pNext);
|
|
crash_note("a pipeline library", s->vs, s->fs, NULL);
|
|
if (pipe_create(1, &pi, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
|
plat_crash_context(NULL);
|
|
if (pipe && (kind == LIB_PR || kind == LIB_FS)) fb_note(FB_LIBRARY, &fb); // the shader libraries: the others compile nothing
|
|
out:
|
|
if (m) vkDestroyShaderModule(P.device, m, NULL);
|
|
free(patched);
|
|
return pipe;
|
|
}
|
|
|
|
// The cached library for key. Caller holds P.lock. build 0: lookup only (VK_NULL_HANDLE and *miss when absent);
|
|
// build 1: a missing one is built with P.lock released (a failed build is cached too).
|
|
// The library entry for key, NULL when absent. Caller holds P.lock.
|
|
static Lib *lib_find(uint64_t h, const void *key, uint32_t size) {
|
|
for (Lib *b = libs[h % LIB_BUCKETS]; b; b = b->next)
|
|
if (b->hash == h && b->size == size && !memcmp(b->key, key, size)) return b;
|
|
return NULL;
|
|
}
|
|
|
|
// Enters a library built with P.lock released; another thread's entry made meanwhile wins and pipe is destroyed.
|
|
// Returns the entered handle. Caller holds P.lock.
|
|
static VkPipeline lib_put(uint64_t h, const void *key, uint32_t size, VkPipeline pipe) {
|
|
Lib *b = lib_find(h, key, size);
|
|
if (b && b->building) { // another thread's build publishes the entry
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
if (b) {
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
return b->pipe;
|
|
}
|
|
if (!(b = calloc(1, sizeof *b + size))) return pipe;
|
|
b->hash = h, b->size = size, b->pipe = pipe;
|
|
memcpy(b->key, key, size);
|
|
b->next = libs[h % LIB_BUCKETS];
|
|
libs[h % LIB_BUCKETS] = b;
|
|
return pipe;
|
|
}
|
|
|
|
// Shader libraries a waiting draw needs (lib_wait, link_queue): queued here ahead of every pipeline job, so an idle
|
|
// worker builds each at once and the vertex and fragment libraries of one draw build in parallel. A job no worker took
|
|
// is built by the first thread that needs its library (lib_get; lib_wait only without library workers). A job with p
|
|
// is a waiting draw's fast link of p (after its library jobs), run on a worker so the drawing thread only waits.
|
|
// Under P.lock.
|
|
typedef struct LibJob { struct LibJob *next; Lib *b; int kind; Layout *l; VkPipeState s; GKey k; Pipe *p; } LibJob;
|
|
static struct {
|
|
LibJob *head, *tail;
|
|
int idle; // workers asleep on work_cv
|
|
int running; // jobs workers are building
|
|
int on; // NX2NAVI_PIPE_WAIT: 1 workers (default), 0 the drawing thread builds them (draw);
|
|
// set by vk_pipeline_async_enable, before the replay runs
|
|
int lib_workers; // workers that take only these jobs (lib_worker), started with the pool
|
|
uint64_t waits, queued, by_worker, stolen, timeouts, links;
|
|
} U;
|
|
|
|
// A building entry for key, NULL when out of memory. Caller holds P.lock and found none.
|
|
static Lib *lib_new(uint64_t h, const void *key, uint32_t size) {
|
|
Lib *b = calloc(1, sizeof *b + size);
|
|
if (!b) return NULL;
|
|
b->hash = h, b->size = size, b->building = 1;
|
|
memcpy(b->key, key, size);
|
|
b->next = libs[h % LIB_BUCKETS];
|
|
libs[h % LIB_BUCKETS] = b;
|
|
return b;
|
|
}
|
|
|
|
// Builds b (building, not queued) with P.lock released and publishes it; a failed build is entered as VK_NULL_HANDLE.
|
|
static VkPipeline lib_build(Lib *b, int kind, const VkPipeState *s, const GKey *k, Layout *l) {
|
|
VkPipelineLayout lay = kind == LIB_PR ? half_layout(l, 0) : kind == LIB_FS ? half_layout(l, 1) : VK_NULL_HANDLE;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (lay || kind == LIB_VI || kind == LIB_FO) {
|
|
plat_mutex_unlock(&P.lock);
|
|
uint64_t t0 = qpc();
|
|
pipe = lib_create(kind, s, k, lay, b->key);
|
|
double ms = ms_since(t0);
|
|
plat_mutex_lock(&P.lock);
|
|
P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms;
|
|
if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms;
|
|
}
|
|
if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind);
|
|
b->pipe = pipe, b->building = 0;
|
|
plat_cond_broadcast(&done_cv);
|
|
return pipe;
|
|
}
|
|
|
|
// Takes b's queued job out of the queue and builds it on this thread. Caller holds P.lock.
|
|
static void lib_steal(Lib *b) {
|
|
LibJob *j = b->job, **pp = &U.head, *prev = NULL;
|
|
while (*pp && *pp != j) prev = *pp, pp = &(*pp)->next;
|
|
if (*pp) *pp = j->next;
|
|
if (U.tail == j) U.tail = prev;
|
|
b->job = NULL;
|
|
lib_build(b, j->kind, &j->s, &j->k, j->l);
|
|
free(j);
|
|
}
|
|
|
|
static VkPipeline lib_get(int kind, const VkPipeState *s, const GKey *k, Layout *l, const void *key, uint32_t size,
|
|
int build, int *miss) {
|
|
uint64_t h = key_hash(key, size);
|
|
Lib *have = lib_find(h, key, size);
|
|
while (have && have->building) { // being built (wait) or queued (build it here)
|
|
if (!build) {
|
|
*miss = 1;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
if (have->job) U.stolen++, lib_steal(have);
|
|
else plat_cond_wait(&done_cv, &P.lock);
|
|
}
|
|
if (have) {
|
|
if (have->pre && !have->used) have->used = 1, P.g.pre_used[kind]++;
|
|
return have->pipe;
|
|
}
|
|
VkPipelineLayout lay = kind == LIB_PR ? half_layout(l, 0) : kind == LIB_FS ? half_layout(l, 1) : VK_NULL_HANDLE;
|
|
if (!build || ((kind == LIB_PR || kind == LIB_FS) && !lay)) {
|
|
*miss = 1;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
Lib *b = lib_new(h, key, size); // entered first, so other threads wait for this build instead of repeating it
|
|
if (b) return lib_build(b, kind, s, k, l);
|
|
plat_mutex_unlock(&P.lock);
|
|
uint64_t t0 = qpc();
|
|
VkPipeline pipe = lib_create(kind, s, k, lay, key);
|
|
double ms = ms_since(t0);
|
|
plat_mutex_lock(&P.lock);
|
|
if ((have = lib_find(h, key, size)) && !have->building) { // another thread built it meanwhile
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
return have->pipe;
|
|
}
|
|
P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms;
|
|
if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms;
|
|
if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind);
|
|
return lib_put(h, key, size, pipe);
|
|
}
|
|
|
|
static void libs_destroy(void) {
|
|
for (int i = 0; i < LIB_BUCKETS; i++) {
|
|
for (Lib *b = libs[i], *n; b; b = n) {
|
|
n = b->next;
|
|
if (b->pipe) vkDestroyPipeline(P.device, b->pipe, NULL);
|
|
free(b);
|
|
}
|
|
libs[i] = NULL;
|
|
}
|
|
}
|
|
|
|
// The four library keys of pipeline s (key k).
|
|
static void lib_keys(const VkPipeState *s, const GKey *k, ViKey *v, PrKey *pk, FsKey *fk, FoKey *ok) {
|
|
memset(v, 0, sizeof *v);
|
|
v->kind = LIB_VI, v->topo_class = k->topo_class;
|
|
memcpy(v->attr_fmt, k->attr_fmt, sizeof v->attr_fmt), memcpy(v->attr_off, k->attr_off, sizeof v->attr_off);
|
|
memcpy(v->attr_bind, k->attr_bind, sizeof v->attr_bind), memcpy(v->rate, k->rate, sizeof v->rate);
|
|
memset(pk, 0, sizeof *pk);
|
|
GfxShaderIo fin[32];
|
|
pk->kind = LIB_PR, pk->vs = s->vs, pk->clamp = k->clamp, pk->poly_mode = k->poly_mode, pk->minus_w = k->minus_w;
|
|
pk->missing = vs_union_missing(s, fin);
|
|
for (int i = 0; i < 32; i++)
|
|
if (pk->missing >> i & 1) pk->fin[i] = fin[i];
|
|
memset(fk, 0, sizeof *fk);
|
|
fk->kind = LIB_FS, fk->fs = s->fs, fk->dsb = k->dsb, fk->samples = k->samples, fk->a2c = k->a2c;
|
|
memset(ok, 0, sizeof *ok);
|
|
ok->kind = LIB_FO, ok->n_color = k->n_color, ok->samples = k->samples, ok->a2c = k->a2c, ok->logic_op = k->logic_op;
|
|
ok->blend_en = k->blend_en, ok->depth = k->depth, ok->stencil = k->stencil, ok->wmask = k->wmask;
|
|
memcpy(ok->color, k->color, sizeof ok->color), memcpy(ok->blend, k->blend, sizeof ok->blend);
|
|
}
|
|
|
|
// A fast-linked pipeline for s from its stage libraries. Caller holds P.lock.
|
|
// build 0 (drawing thread): links only when the shader libraries exist (vertex input and fragment output libraries,
|
|
// which hold no shader, are built here); *final 0 = not yet, a worker builds them. build 1 (worker): builds the
|
|
// missing libraries and links with P.lock released. VK_NULL_HANDLE with *final 1 when GPL cannot express the pipeline
|
|
// (shared layout, feedback loop) or a build failed.
|
|
static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc, int build, int *final) {
|
|
Layout *l = (Layout *)lc;
|
|
*final = 1;
|
|
if (!l->set1 || k->fbl) return VK_NULL_HANDLE;
|
|
uint64_t t0 = qpc();
|
|
ViKey vk_;
|
|
PrKey pk;
|
|
FsKey fk;
|
|
FoKey ok;
|
|
lib_keys(s, k, &vk_, &pk, &fk, &ok);
|
|
int miss = 0;
|
|
VkPipeline lib[4];
|
|
lib[1] = lib_get(LIB_PR, s, k, l, &pk, sizeof pk, build, &miss); // shader libraries first: no work when absent
|
|
lib[2] = miss ? VK_NULL_HANDLE : lib_get(LIB_FS, s, k, l, &fk, sizeof fk, build, &miss);
|
|
if (miss) {
|
|
if (!build) *final = 0, P.g.defer++;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
lib[0] = lib_get(LIB_VI, s, k, l, &vk_, sizeof vk_, 1, &miss);
|
|
lib[3] = lib_get(LIB_FO, s, k, l, &ok, sizeof ok, 1, &miss);
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (lib[0] && lib[1] && lib[2] && lib[3]) {
|
|
VkPipelineLibraryCreateInfoKHR li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR, .libraryCount = 4,
|
|
.pLibraries = lib};
|
|
VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &li,
|
|
.layout = l->layout};
|
|
if (build) plat_mutex_unlock(&P.lock);
|
|
uint64_t t1 = qpc();
|
|
crash_note("a library link", s->vs, s->fs, NULL);
|
|
if (vkCreateGraphicsPipelines(P.device, VK_NULL_HANDLE, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
|
plat_crash_context(NULL);
|
|
double ms = ms_since(t1); // the link alone: library builds are counted apart
|
|
if (build) plat_mutex_lock(&P.lock);
|
|
P.g.link_ms += ms;
|
|
if (ms > P.g.link_max) P.g.link_max = ms;
|
|
}
|
|
if (pipe) P.g.links++;
|
|
else if (!P.g.link_fail++) rt_log("vk_pipeline: gpl: fast link failed; such draws wait or skip as without GPL");
|
|
double total = ms_since(t0);
|
|
if (!build && total > 5.0 && P.g.slow++ < 16)
|
|
rt_log("vk_pipeline: gpl: fast link on the drawing thread took %.1f ms", total);
|
|
return pipe;
|
|
}
|
|
|
|
// ---- background graphics pipeline builds (vk_pipeline_async_enable)
|
|
// A miss in vk_pipeline_graphics_async enters the Pipe as building and queues a job; the replay skips draws with it
|
|
// until a worker publishes the handle. The job holds copies of the state and the key. What they point to outlives it:
|
|
// shaders (gfx_shader_get results) live for the process (gfx_shader_drop_memory is the shader probe's, which builds no
|
|
// pipelines), layouts until vk_pipeline_shutdown, which drains the queue first. Workers call vkCreateGraphicsPipelines
|
|
// without P.lock: P.cache is created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT, so the driver
|
|
// synchronises builds into it; only the copy of its data in cache_save is kept apart from them (cache_rw). Compute
|
|
// pipelines stay synchronous: a skipped
|
|
// dispatch can change what the game computes.
|
|
//
|
|
// Two queues: draws' misses (A.head) and the boot prebuild (A.low, see "pipeline keys" below). Workers take the low
|
|
// queue only when the other is empty and run its jobs at PLAT_PRIO_BELOW_NORMAL, so guest threads win. A draw
|
|
// that finds a prebuild job still queued moves it to the draw queue; one that finds it running raises its worker.
|
|
typedef struct Job {
|
|
struct Job *next, *prev; // prev: low queue only
|
|
Pipe *p;
|
|
const Layout *l;
|
|
uint32_t fs_off;
|
|
uint8_t low; // in the low queue, or running at below-normal priority
|
|
uint8_t prebuild, demanded; // from the key file; a draw asked for it before it was ready
|
|
int worker; // running on A.thread[worker]; -1 queued
|
|
VkPipeState s;
|
|
GKey k;
|
|
} Job;
|
|
|
|
#define MAX_WORKERS 4
|
|
|
|
static struct {
|
|
int on; // set once by vk_pipeline_async_enable, before the replay runs
|
|
int threads;
|
|
PlatThread *thread[MAX_WORKERS];
|
|
uint8_t prio_low[MAX_WORKERS]; // worker runs below normal (under P.lock)
|
|
Job *head, *tail; // draws' misses
|
|
Job *low, *low_tail; // prebuild
|
|
uint32_t inflight; // queued or running, both queues
|
|
int low_running; // pool workers running a job taken from the low queue
|
|
PlatCond idle; // with P.lock: no job queued or running
|
|
} A = {.idle = PLAT_COND_INIT};
|
|
|
|
// ---- pipeline keys (pipeline-keys.bin next to the pipeline cache)
|
|
// Every graphics pipeline built in a game run appends a KeyRec: the VkPipeState it was built from with the shader
|
|
// pointers zeroed and the shaders named by their gfx_shader input ids instead, which do not depend on the translator,
|
|
// so the record rebuilds the pipeline without guest memory. At the next boot vk_pipeline_prewarm (rt_boot, before the
|
|
// guest runs; NX2NAVI_PIPE_PREBUILD=0 skips it) loads each record's shaders, translating the ones not cached under the
|
|
// current translator from their stored inputs (gfx_shader_prewarm; a record whose shader has neither is dropped),
|
|
// enters each Pipe as building under the key a draw would compute, queues it to the low queue and builds the queue on
|
|
// every core. Records are deduplicated by a hash of their bytes; they are collected under P.lock by the worker that
|
|
// built the pipeline and appended by the present thread (vk_pipeline_autosave) or at exit (vk_pipeline_save). A file
|
|
// with another magic, version or record size is ignored and rewritten; a version 3 file (no task stage) is converted
|
|
// and rewritten. Bump KEYS_VERSION when VkPipeState or the record changes meaning without changing size.
|
|
#define KEYS_VERSION 4 // 3: shaders named by input id (gfx_shader.c); 4: mesh pipelines, with their task stage
|
|
typedef struct { char magic[8]; uint32_t version, rec_size; } KeysHeader;
|
|
typedef struct {
|
|
uint8_t vs[32], fs[32];
|
|
uint32_t has_fs, pad;
|
|
VkPipeState s;
|
|
} KeyRecV3;
|
|
typedef struct {
|
|
uint8_t vs[32], fs[32], ts[32]; // gfx_shader_input_id of each stage (vs: the mesh stage of a mesh pipeline); fs all
|
|
// zero with has_fs 0, ts with has_ts 0
|
|
uint32_t has_fs, has_ts;
|
|
VkPipeState s; // vs, fs, ts zero
|
|
} KeyRec;
|
|
static struct {
|
|
char path[PLAT_PATH_MAX * 2];
|
|
int header_ok; // the file exists with our header: append
|
|
uint64_t *seen; // open addressing set of record hashes (0 empty)
|
|
uint32_t seen_cap, seen_n;
|
|
KeyRec *pend; // records not written yet
|
|
uint32_t pend_n, pend_cap;
|
|
uint8_t *file; // the file as read at boot, until the prewarm is done with it
|
|
size_t file_n; // records in it
|
|
PlatMutex io; // one writer
|
|
// prebuild progress (under P.lock)
|
|
int loading; // the prewarm still queues records
|
|
int warm; // vk_pipeline_prewarm owns the file (prebuild_start leaves it)
|
|
uint32_t left; // prebuild jobs queued or running
|
|
uint32_t built, failed, promoted;
|
|
double t0; // now_ms() when the prebuild started
|
|
double until; // now_ms() after which the boot builders stop (comment at prewarm_run); 0 none
|
|
} K = {.io = PLAT_MUTEX_INIT};
|
|
|
|
static uint64_t rec_hash(const KeyRec *r) {
|
|
uint64_t h = fnv(FNV0, r, sizeof *r);
|
|
return h ? h : 1;
|
|
}
|
|
|
|
// 1 if h was new. Caller holds P.lock.
|
|
static int seen_add(uint64_t h) {
|
|
if ((K.seen_n + 1) * 2 > K.seen_cap) {
|
|
uint32_t cap = K.seen_cap ? K.seen_cap * 2 : 4096;
|
|
uint64_t *t = calloc(cap, sizeof *t);
|
|
if (!t) return 1;
|
|
for (uint32_t i = 0; i < K.seen_cap; i++) {
|
|
if (!K.seen[i]) continue;
|
|
uint32_t j = (uint32_t)K.seen[i] & (cap - 1);
|
|
while (t[j]) j = (j + 1) & (cap - 1);
|
|
t[j] = K.seen[i];
|
|
}
|
|
free(K.seen);
|
|
K.seen = t, K.seen_cap = cap;
|
|
}
|
|
uint32_t j = (uint32_t)h & (K.seen_cap - 1);
|
|
while (K.seen[j]) {
|
|
if (K.seen[j] == h) return 0;
|
|
j = (j + 1) & (K.seen_cap - 1);
|
|
}
|
|
K.seen[j] = h;
|
|
K.seen_n++;
|
|
return 1;
|
|
}
|
|
|
|
static void rec_fill(KeyRec *r, const VkPipeState *s) {
|
|
memset(r, 0, sizeof *r);
|
|
memcpy(r->vs, gfx_shader_input_id(s->vs), 32);
|
|
if (s->fs) memcpy(r->fs, gfx_shader_input_id(s->fs), 32), r->has_fs = 1;
|
|
if (s->ts) memcpy(r->ts, gfx_shader_input_id(s->ts), 32), r->has_ts = 1;
|
|
r->s = *s;
|
|
r->s.vs = r->s.fs = r->s.ts = NULL;
|
|
r->s.prim_restart = 0; // dynamic state: the same pipeline either way
|
|
}
|
|
|
|
// A pipeline was built for a draw: queue its record for the file. Caller holds P.lock.
|
|
static void keys_note(const VkPipeState *s) {
|
|
if (!A.on || !K.path[0]) return;
|
|
static const uint8_t none[32];
|
|
if (s->fs && !memcmp(gfx_shader_input_id(s->fs), none, 32)) return; // the runtime's own shader (gfx_shader_own)
|
|
if (s->ts && !memcmp(gfx_shader_input_id(s->ts), none, 32)) return;
|
|
KeyRec r;
|
|
rec_fill(&r, s);
|
|
if (!seen_add(rec_hash(&r))) return;
|
|
if (K.pend_n == K.pend_cap) {
|
|
uint32_t cap = K.pend_cap ? K.pend_cap * 2 : 64;
|
|
KeyRec *t = realloc(K.pend, cap * sizeof *t);
|
|
if (!t) return;
|
|
K.pend = t, K.pend_cap = cap;
|
|
}
|
|
K.pend[K.pend_n++] = r;
|
|
}
|
|
|
|
// Appends the pending records (present thread or exit; never the render thread).
|
|
static void keys_flush(void) {
|
|
if (!K.path[0]) return;
|
|
plat_mutex_lock(&K.io);
|
|
plat_mutex_lock(&P.lock);
|
|
KeyRec *r = K.pend;
|
|
uint32_t n = K.pend_n;
|
|
K.pend = NULL, K.pend_n = K.pend_cap = 0;
|
|
plat_mutex_unlock(&P.lock);
|
|
if (n) {
|
|
FILE *f = fopen(K.path, K.header_ok ? "ab" : "wb");
|
|
int ok = f != NULL;
|
|
if (ok && !K.header_ok) {
|
|
KeysHeader h = {{'N', 'X', '2', 'N', 'P', 'K', 'E', 'Y'}, KEYS_VERSION, sizeof(KeyRec)};
|
|
ok = fwrite(&h, sizeof h, 1, f) == 1;
|
|
}
|
|
if (ok) ok = fwrite(r, sizeof *r, n, f) == n;
|
|
if (ok) ok = plat_file_sync(f); // appended records on the disk before the next append
|
|
if (f) ok = !fclose(f) && ok;
|
|
if (ok) K.header_ok = 1;
|
|
else rt_log("vk_pipeline: cannot write %s", K.path);
|
|
}
|
|
free(r);
|
|
plat_mutex_unlock(&K.io);
|
|
}
|
|
|
|
// Reads the key file into K.file and its record hashes into the set. Caller holds P.lock (ensure_init).
|
|
static void keys_open(void) {
|
|
if (!A.on || !P.cache_path[0]) return;
|
|
snprintf(K.path, sizeof K.path, "%s", P.cache_path);
|
|
char *slash = strrchr(K.path, '\\'), *fslash = strrchr(K.path, '/');
|
|
if (fslash > slash) slash = fslash;
|
|
snprintf(slash ? slash + 1 : K.path, sizeof K.path - (size_t)(slash ? slash + 1 - K.path : 0), "pipeline-keys.bin");
|
|
gfx_shader_record_at(K.path); // shader-answers.bin next to it (gfx_shader.c)
|
|
FILE *f = fopen(K.path, "rb");
|
|
if (!f) return;
|
|
KeysHeader h;
|
|
fseek(f, 0, SEEK_END);
|
|
long len = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
int head = len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8);
|
|
if (head && h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec)) {
|
|
K.header_ok = 1;
|
|
size_t n = ((size_t)len - sizeof h) / sizeof(KeyRec);
|
|
if (n && (K.file = malloc(n * sizeof(KeyRec))) && fread(K.file, sizeof(KeyRec), n, f) == n) K.file_n = n;
|
|
else free(K.file), K.file = NULL;
|
|
for (size_t i = 0; i < K.file_n; i++) seen_add(rec_hash((const KeyRec *)K.file + i));
|
|
if (len != (long)(sizeof h + n * sizeof(KeyRec))) // a torn append: rewrite the file from what was read
|
|
K.header_ok = 0;
|
|
} else if (head && h.version == 3 && h.rec_size == sizeof(KeyRecV3)) {
|
|
// the vertex pipelines of a version 3 file keep their prewarm: each record without a task stage, and the
|
|
// file rewritten in this version below (header_ok 0)
|
|
size_t n = ((size_t)len - sizeof h) / sizeof(KeyRecV3);
|
|
KeyRecV3 *old = n ? malloc(n * sizeof *old) : NULL;
|
|
if (old && fread(old, sizeof *old, n, f) == n && (K.file = calloc(n, sizeof(KeyRec)))) {
|
|
KeyRec *r = (KeyRec *)K.file;
|
|
for (size_t i = 0; i < n; i++) {
|
|
memcpy(r[i].vs, old[i].vs, 32), memcpy(r[i].fs, old[i].fs, 32);
|
|
r[i].has_fs = old[i].has_fs, r[i].s = old[i].s;
|
|
seen_add(rec_hash(&r[i]));
|
|
}
|
|
K.file_n = n;
|
|
rt_log("vk_pipeline: %s: %zu version 3 records converted; the file is rewritten", K.path, n);
|
|
}
|
|
free(old);
|
|
} else {
|
|
rt_log("vk_pipeline: %s has another format or version; it is rewritten", K.path);
|
|
}
|
|
fclose(f);
|
|
if (!K.header_ok && K.file_n) { // keep the whole records of a torn file
|
|
KeyRec *t = malloc(K.file_n * sizeof *t);
|
|
if (t) memcpy(t, K.file, K.file_n * sizeof *t), K.pend = t, K.pend_n = K.pend_cap = (uint32_t)K.file_n;
|
|
}
|
|
}
|
|
|
|
static double now_ms(void) { return (double)plat_now_ns() / 1e6; }
|
|
|
|
// Caller holds P.lock.
|
|
static void prebuild_report(void) {
|
|
rt_log("vk_pipeline: prebuild done: %u built, %u failed, %u asked for by a draw first; %.0f ms after the device was "
|
|
"ready", K.built, K.failed, K.promoted, now_ms() - K.t0);
|
|
}
|
|
|
|
static void job_lower(Job *j);
|
|
|
|
// Runs a waiting draw's job taken from U on a worker: a library build or, for a job with p, p's fast link (gpl_link
|
|
// build 1 builds a library still missing and waits for one another worker builds). Releases P.lock while building.
|
|
static void u_run(LibJob *lj) {
|
|
U.by_worker++, U.running++;
|
|
if (lj->p) {
|
|
Pipe *p = lj->p;
|
|
if (!p->info.pipeline) {
|
|
int final = 0;
|
|
VkPipeline f = gpl_link(&lj->s, &lj->k, lj->l, 1, &final); // releases P.lock while linking
|
|
if (f && !p->info.pipeline) {
|
|
p->fast = p->info.pipeline = f;
|
|
job_lower(p->job); // its job now only does the optimised build
|
|
} else if (f) {
|
|
vkDestroyPipeline(P.device, f, NULL); // the job published one meanwhile
|
|
}
|
|
}
|
|
p->link_queued = 0;
|
|
U.links++;
|
|
plat_cond_broadcast(&done_cv);
|
|
} else {
|
|
lib_build(lj->b, lj->kind, &lj->s, &lj->k, lj->l);
|
|
}
|
|
U.running--;
|
|
free(lj);
|
|
}
|
|
|
|
// A worker for waiting draws' jobs only (U): a draw that waits finds one free even while the pool builds pipelines,
|
|
// so the drawing thread never builds a library itself.
|
|
static void lib_worker(void *arg) {
|
|
(void)arg;
|
|
plat_mutex_lock(&P.lock);
|
|
for (;;) {
|
|
while (!U.head) plat_cond_wait(&lib_cv, &P.lock);
|
|
LibJob *lj = U.head;
|
|
if (!(U.head = lj->next)) U.tail = NULL;
|
|
if (lj->b) lj->b->job = NULL;
|
|
u_run(lj);
|
|
}
|
|
}
|
|
|
|
static void pipe_worker(void *arg) {
|
|
int me = (int)(intptr_t)arg;
|
|
plat_mutex_lock(&P.lock);
|
|
for (;;) {
|
|
// With 3 or more workers one never takes low-queue work, so a waiting draw's library starts at once
|
|
// (not with NX2NAVI_PIPE_WAIT=draw).
|
|
while (!U.head && !A.head && !(A.low && (A.threads < 3 || !U.on || A.low_running < A.threads - 1))) {
|
|
U.idle++;
|
|
plat_cond_wait(&work_cv, &P.lock);
|
|
U.idle--;
|
|
}
|
|
if (U.head) { // a library a draw waits for
|
|
LibJob *lj = U.head;
|
|
if (!(U.head = lj->next)) U.tail = NULL;
|
|
if (lj->b) lj->b->job = NULL;
|
|
if (A.prio_low[me]) A.prio_low[me] = 0, plat_thread_priority(PLAT_PRIO_NORMAL);
|
|
u_run(lj); // releases P.lock while building
|
|
continue;
|
|
}
|
|
Job *j = A.head;
|
|
int lowrun = !j;
|
|
if (j) {
|
|
if (!(A.head = j->next)) A.tail = NULL;
|
|
} else {
|
|
j = A.low;
|
|
if ((A.low = j->next)) A.low->prev = NULL;
|
|
else A.low_tail = NULL;
|
|
A.low_running++;
|
|
}
|
|
j->next = j->prev = NULL;
|
|
j->worker = me;
|
|
if (j->low != A.prio_low[me]) { // under P.lock, so a draw raising this thread (promote) is not undone
|
|
A.prio_low[me] = j->low;
|
|
plat_thread_priority(j->low ? PLAT_PRIO_BELOW_NORMAL : PLAT_PRIO_NORMAL);
|
|
}
|
|
// Step 1 for a job a draw waits for: its stage libraries and the fast link. Step 2, the optimised build, goes to
|
|
// the head of the low queue, so a burst of new pipelines gets every fast link before any optimised build.
|
|
if (P.gpl && (!j->prebuild || j->demanded) && !j->p->info.pipeline && !j->p->fast_tried && j->p->lay &&
|
|
!is_mesh(&j->s)) {
|
|
int final;
|
|
VkPipeline f = gpl_link(&j->s, &j->k, j->p->lay, 1, &final); // releases P.lock while building
|
|
j->p->fast_tried = 1;
|
|
if (f && !j->p->info.pipeline) {
|
|
j->p->fast = j->p->info.pipeline = f, P.g.wlinks++;
|
|
plat_cond_broadcast(&done_cv);
|
|
j->low = 1, j->worker = -1, P.g.requeued++;
|
|
j->prev = NULL, j->next = A.low;
|
|
if (A.low) A.low->prev = j;
|
|
else A.low_tail = j;
|
|
A.low = j;
|
|
A.low_running -= lowrun;
|
|
continue;
|
|
} else if (f) {
|
|
vkDestroyPipeline(P.device, f, NULL);
|
|
}
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off);
|
|
plat_mutex_lock(&P.lock);
|
|
if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one
|
|
if (pipe && j->p->fast) P.g.swaps++;
|
|
j->p->building = 0;
|
|
j->p->job = NULL;
|
|
if (pipe) {
|
|
P.stats.graphics++, P.stats.async_built++;
|
|
if (!j->prebuild) keys_note(&j->s);
|
|
} else {
|
|
P.stats.failures++;
|
|
rt_log("vk_pipeline: graphics pipeline not built: pipeline creation failed");
|
|
}
|
|
if (j->prebuild) {
|
|
if (pipe) K.built++;
|
|
else K.failed++;
|
|
K.promoted += j->demanded;
|
|
if (!--K.left && !K.loading) prebuild_report();
|
|
}
|
|
A.low_running -= lowrun;
|
|
if (!--A.inflight) plat_cond_broadcast(&A.idle);
|
|
plat_cond_broadcast(&done_cv);
|
|
if (pipe && (!j->prebuild || j->demanded)) perf_count(PERF_CNT_PIPE_ASYNC); // a draw waited for it
|
|
free(j);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Starts the pool on first use: a quarter of the logical processors, 2 to 4. Caller holds P.lock.
|
|
static int workers_start(void) {
|
|
if (A.threads) return 1;
|
|
uint32_t cpus = plat_cpu_usable();
|
|
int n = cpus / 4 < 2 ? 2 : cpus / 4 > MAX_WORKERS ? MAX_WORKERS : (int)(cpus / 4);
|
|
for (int i = 0; i < n; i++) {
|
|
PlatThread *t = plat_thread_new(pipe_worker, (void *)(intptr_t)A.threads, "pipe worker");
|
|
if (!t) break;
|
|
A.thread[A.threads++] = t; // kept: promote() raises a worker running a prebuild job
|
|
}
|
|
// two for waiting draws' libraries and links (the vertex and fragment libraries of one draw in parallel)
|
|
for (int i = 0; A.threads && U.on && P.gpl && i < 2; i++)
|
|
if (plat_thread_new(lib_worker, NULL, "pipe lib worker")) U.lib_workers++;
|
|
if (A.threads)
|
|
rt_log("vk_pipeline: %d background graphics pipeline workers, %d for waiting draws' libraries", A.threads,
|
|
U.lib_workers);
|
|
else rt_log("vk_pipeline: cannot start pipeline workers; building synchronously");
|
|
return A.threads != 0;
|
|
}
|
|
|
|
static void workers_drain(void) {
|
|
plat_mutex_lock(&P.lock);
|
|
K.loading = 0;
|
|
for (Job *j = A.low, *n; j; j = n) { // queued prebuild jobs are dropped, not built
|
|
n = j->next;
|
|
j->p->building = 0, j->p->job = NULL;
|
|
A.inflight--, K.left--;
|
|
free(j);
|
|
}
|
|
A.low = A.low_tail = NULL;
|
|
for (LibJob *j = U.head, *n; j; j = n) { // queued library jobs too: their entries read as failed builds
|
|
n = j->next;
|
|
j->b->job = NULL, j->b->building = 0;
|
|
free(j);
|
|
}
|
|
U.head = U.tail = NULL;
|
|
while (A.inflight) plat_cond_wait(&A.idle, &P.lock);
|
|
while (U.running) plat_cond_wait(&done_cv, &P.lock);
|
|
plat_mutex_unlock(&P.lock);
|
|
}
|
|
|
|
void vk_pipeline_async_enable(void) {
|
|
U.on = strcmp(cfg_value("NX2NAVI_PIPE_WAIT"), "draw") != 0;
|
|
const char *e = cfg_get("NX2NAVI_PIPE_SYNC");
|
|
if (e && *e && strcmp(e, "0")) return;
|
|
A.on = 1;
|
|
}
|
|
|
|
int vk_pipeline_async_on(void) { return A.on; }
|
|
|
|
// A draw needs a pipeline a prebuild job owns: the job moves to the draw queue, or its worker to normal priority.
|
|
// Caller holds P.lock.
|
|
static void promote(Pipe *p) {
|
|
Job *j = p->job;
|
|
if (!j || !j->low) return;
|
|
j->low = 0, j->demanded = 1;
|
|
if (j->worker < 0) {
|
|
if (j->prev) j->prev->next = j->next;
|
|
else A.low = j->next;
|
|
if (j->next) j->next->prev = j->prev;
|
|
else A.low_tail = j->prev;
|
|
j->next = j->prev = NULL;
|
|
if (A.tail) A.tail->next = j;
|
|
else A.head = j;
|
|
A.tail = j;
|
|
plat_cond_signal(&work_cv);
|
|
} else if (A.prio_low[j->worker]) {
|
|
A.prio_low[j->worker] = 0;
|
|
plat_thread_set_priority(A.thread[j->worker], PLAT_PRIO_NORMAL);
|
|
}
|
|
}
|
|
|
|
// Enters a missing pipeline: built here (mode 0), or queued for a draw (1) or for the prebuild (2). Failures are
|
|
// entered too. *queued: 1 when a job owns it. Caller holds P.lock.
|
|
static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode, int *queued) {
|
|
*queued = 0;
|
|
const char *why = NULL;
|
|
Bindings B = {0};
|
|
Layout *l = NULL;
|
|
uint32_t span = binding_span(s->vs);
|
|
int mesh = is_mesh(s);
|
|
if (mesh) {
|
|
// Mesh draws (VK_EXT_mesh_shader): one set holding the mesh stage's bindings, the fragment stage's after them and
|
|
// the task stage's after those (create_graphics renumbers both modules); no libraries (the GPL fast link is for
|
|
// vertex pipelines). They go to the workers like the others: built here, each new one stopped the frame for its
|
|
// whole compile (0.26-0.51 s in Kokiri Forest), and they are recorded for the boot prewarm (keys_note).
|
|
uint32_t fspan = s->fs ? binding_span(s->fs) : 1;
|
|
if (!vk_ctx()->has_mesh_shader) why = "the GPU has no mesh shaders";
|
|
else if ((s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS) || (s->ts && gfx_shader_stage(s->ts) != GFX_STAGE_TS)) why = "stage mismatch";
|
|
else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1)) ||
|
|
(s->ts && !add_stage(&B, s->ts, span - 1 + fspan - 1))) why = "too many bindings";
|
|
else if (!(l = layout_get(&B))) why = "descriptor layout creation failed";
|
|
} else if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) why = "stage mismatch";
|
|
else if (k->topo_class == 3) why = "patches need tessellation stages, which are not translated";
|
|
else if (P.indep) {
|
|
Bindings F = {0};
|
|
if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&F, s->fs, 0))) why = "too many bindings";
|
|
else if (!(l = layout_pair(&B, &F))) why = "descriptor layout creation failed";
|
|
B.pool |= F.pool;
|
|
span = 1;
|
|
} else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1))) why = "too many bindings";
|
|
else if (!(l = layout_get(&B))) why = "descriptor layout creation failed";
|
|
char lack[128];
|
|
if (!why && !layout_covers(s, l, span - 1, lack, sizeof lack)) why = lack, l = NULL;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
Job *j = NULL;
|
|
if (!why && mode && workers_start() && (j = calloc(1, sizeof *j))) {
|
|
*j = (Job){.l = l, .fs_off = span - 1, .low = mode == 2, .prebuild = mode == 2, .worker = -1, .s = *s, .k = *k};
|
|
} else if (!why && !(pipe = create_graphics(s, k, l, span - 1))) {
|
|
why = "pipeline creation failed";
|
|
}
|
|
if (why) {
|
|
P.stats.failures++;
|
|
rt_log("vk_pipeline: graphics pipeline not built: %s", why);
|
|
} else if (!j) {
|
|
P.stats.graphics++;
|
|
if (mode != 2) keys_note(s);
|
|
}
|
|
Pipe *p = pipe_add(k, sizeof *k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_GRAPHICS, gfx_shader_inputs(s->vs, NULL));
|
|
if (!j || !p) {
|
|
free(j);
|
|
return p;
|
|
}
|
|
j->p = p;
|
|
p->building = 1, p->job = j;
|
|
if (mode == 2) {
|
|
j->prev = A.low_tail;
|
|
if (A.low_tail) A.low_tail->next = j;
|
|
else A.low = j;
|
|
A.low_tail = j;
|
|
K.left++;
|
|
} else {
|
|
if (A.tail) A.tail->next = j;
|
|
else A.head = j;
|
|
A.tail = j;
|
|
}
|
|
A.inflight++;
|
|
plat_cond_signal(&work_cv);
|
|
*queued = 1;
|
|
return p;
|
|
}
|
|
|
|
// A job still in the draw queue goes to the head of the low queue: its pipeline has (or is about to have) a fast-linked
|
|
// one, so the optimised build waits behind other draws' misses. Caller holds P.lock.
|
|
static void job_lower(Job *j) {
|
|
if (!j || j->worker >= 0 || j->low) return;
|
|
if (A.head == j) {
|
|
if (!(A.head = j->next)) A.tail = NULL;
|
|
} else {
|
|
Job *q = A.head;
|
|
while (q && q->next != j) q = q->next;
|
|
if (!q) return;
|
|
if (!(q->next = j->next)) A.tail = q;
|
|
}
|
|
j->low = 1, P.g.requeued++;
|
|
j->prev = NULL, j->next = A.low;
|
|
if (A.low) A.low->prev = j;
|
|
else A.low_tail = j;
|
|
A.low = j;
|
|
}
|
|
|
|
// A draw waits for pipeline s: its missing pre-rasterization and fragment shader libraries go to the head of the
|
|
// worker queues (U), so they build in parallel on idle workers, and this thread sleeps until both are published or
|
|
// end (now_ms) passes. A job no worker took within 1 ms (all busy) is built here. Caller holds P.lock.
|
|
static void u_push(LibJob *j) {
|
|
if (U.tail) U.tail->next = j;
|
|
else U.head = j;
|
|
U.tail = j;
|
|
}
|
|
|
|
// Queues s's missing pre-rasterization and fragment shader libraries (b: both entries). 0 when one cannot be entered
|
|
// (no layout half, out of memory). Caller holds P.lock.
|
|
static int lib_queue(const VkPipeState *s, const GKey *k, Layout *l, Lib *b[2]) {
|
|
ViKey vk_;
|
|
PrKey pk;
|
|
FsKey fk;
|
|
FoKey ok;
|
|
lib_keys(s, k, &vk_, &pk, &fk, &ok);
|
|
U.waits++;
|
|
const void *key[2] = {&pk, &fk};
|
|
uint32_t size[2] = {sizeof pk, sizeof fk};
|
|
int queued = 0;
|
|
for (int i = 0; i < 2; i++) {
|
|
uint64_t h = key_hash(key[i], size[i]);
|
|
if ((b[i] = lib_find(h, key[i], size[i]))) continue;
|
|
if (!half_layout(l, i)) return 0; // gpl_link reports it
|
|
LibJob *j = calloc(1, sizeof *j);
|
|
if (!j || !(b[i] = lib_new(h, key[i], size[i]))) {
|
|
free(j);
|
|
return 0;
|
|
}
|
|
*j = (LibJob){.b = b[i], .kind = i ? LIB_FS : LIB_PR, .l = l, .s = *s, .k = *k};
|
|
b[i]->job = j;
|
|
u_push(j);
|
|
U.queued++, queued++;
|
|
}
|
|
if (queued) plat_cond_broadcast(&work_cv), plat_cond_broadcast(&lib_cv);
|
|
return 1;
|
|
}
|
|
|
|
// A waiting draw's whole fast link of p on the library workers: its library jobs, then the link job behind them.
|
|
// The caller then waits for p->info.pipeline. 0 when nothing was queued (the caller links as before). Under P.lock.
|
|
static int link_queue(Pipe *p, const VkPipeState *s, const GKey *k, Layout *l) {
|
|
Lib *b[2];
|
|
if (!U.lib_workers || !lib_queue(s, k, l, b)) return 0;
|
|
LibJob *j = calloc(1, sizeof *j);
|
|
if (!j) return 0;
|
|
*j = (LibJob){.l = l, .s = *s, .k = *k, .p = p};
|
|
p->link_queued = 1;
|
|
u_push(j);
|
|
plat_cond_broadcast(&work_cv), plat_cond_broadcast(&lib_cv);
|
|
return 1;
|
|
}
|
|
|
|
static void lib_wait(const VkPipeState *s, const GKey *k, Layout *l, double end) {
|
|
Lib *b[2];
|
|
if (!lib_queue(s, k, l, b)) return;
|
|
double steal_at = now_ms() + 1.0;
|
|
for (;;) {
|
|
if (!b[0]->building && !b[1]->building) return;
|
|
double now = now_ms();
|
|
if (now >= end) {
|
|
U.timeouts++;
|
|
return;
|
|
}
|
|
Lib *q = b[0]->job ? b[0] : b[1]->job ? b[1] : NULL;
|
|
if (q && !U.lib_workers && (now >= steal_at || !U.idle)) {
|
|
U.stolen++;
|
|
lib_steal(q); // releases P.lock while building
|
|
continue;
|
|
}
|
|
double until = q && steal_at < end ? steal_at : end;
|
|
plat_cond_wait_ms(&done_cv, &P.lock, (uint32_t)(until - now) + 1);
|
|
}
|
|
}
|
|
|
|
static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int async, int *pending, uint32_t wait_ms) {
|
|
if (info) *info = NULL;
|
|
if (pending) *pending = 0;
|
|
if (!s->vs || !ensure_init()) return VK_NULL_HANDLE;
|
|
GKey k;
|
|
gkey_fill(s, &k);
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
plat_mutex_lock(&P.lock);
|
|
Pipe *p = pipe_find(&k, sizeof k, h);
|
|
int queued;
|
|
if (p) {
|
|
if (!p->building) P.stats.hits++;
|
|
} else {
|
|
p = pipe_miss(s, &k, h, async, &queued);
|
|
}
|
|
// A draw that would wait (wait_ms) gets its missing shader libraries built at once, the vertex and fragment ones in
|
|
// parallel on idle workers (lib_wait; about 0.12 ms per library from the driver's disk cache, 5-180 ms cold), and
|
|
// links them here: its own job may queue behind optimised builds of earlier pipelines. The job then only does the
|
|
// optimised build, from the low queue. Other draws leave the libraries to the job and are skipped meanwhile.
|
|
// NX2NAVI_PIPE_WAIT=draw builds them on this thread one after the other instead, as before; NX2NAVI_GPL_DRAW_BUILD=0
|
|
// leaves them to the job.
|
|
static int draw_build = -1;
|
|
if (draw_build < 0) {
|
|
const char *e = cfg_get("NX2NAVI_GPL_DRAW_BUILD");
|
|
draw_build = !(e && !strcmp(e, "0"));
|
|
rt_log("vk_pipeline: a draw that waits for its pipeline %s", !draw_build ? "leaves its libraries to its job"
|
|
: U.on ? "has its libraries built in parallel on the workers, ahead of other jobs"
|
|
: "builds its libraries itself (NX2NAVI_PIPE_WAIT=draw)");
|
|
}
|
|
double end = wait_ms ? now_ms() + wait_ms : 0;
|
|
if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && !p->link_queued && p->lay && !is_mesh(s) &&
|
|
wait_ms && draw_build && U.on && link_queue(p, s, &k, (Layout *)p->lay)) {
|
|
// the libraries and the link on the library workers; this thread waits below without building
|
|
p->fast_tried = 1;
|
|
job_lower(p->job);
|
|
} else if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && !p->link_queued && p->lay &&
|
|
!is_mesh(s)) {
|
|
int final = 0, build = wait_ms != 0 && draw_build;
|
|
uint64_t p0 = perf_begin(), t0 = qpc();
|
|
if (build && U.on) {
|
|
job_lower(p->job); // its worker would only wait for the same libraries; it links them if this draw cannot
|
|
lib_wait(s, &k, (Layout *)p->lay, end); // releases P.lock while waiting
|
|
build = 0; // links here when both libraries exist (gpl_link build 0)
|
|
} else if (build) {
|
|
p->fast_tried = 1; // the job, if a worker takes it meanwhile, goes straight to the optimised build
|
|
}
|
|
VkPipeline f = p->info.pipeline ? VK_NULL_HANDLE : gpl_link(s, &k, p->lay, build, &final); // build 1 releases P.lock
|
|
perf_end(PERF_PIPE, p0);
|
|
if (wait_ms && draw_build) {
|
|
double ms = ms_since(t0);
|
|
P.g.dlinks++, P.g.dlink_ms += ms;
|
|
if (ms > P.g.dlink_max) P.g.dlink_max = ms;
|
|
if (ms > 5.0 && P.g.slow++ < 16)
|
|
rt_log("vk_pipeline: gpl: a waiting draw %s its libraries in %.1f ms", U.on ? "got" : "built", ms);
|
|
}
|
|
if (final) p->fast_tried = 1;
|
|
if (f && !p->info.pipeline) {
|
|
p->fast = p->info.pipeline = f;
|
|
job_lower(p->job);
|
|
plat_cond_broadcast(&done_cv);
|
|
} else if (f) {
|
|
vkDestroyPipeline(P.device, f, NULL); // a worker published one while the lock was free
|
|
}
|
|
}
|
|
if (p && p->building && !p->info.pipeline) {
|
|
promote(p);
|
|
uint64_t p0 = wait_ms ? perf_begin() : 0; // the wait counts as the render thread's pipeline time
|
|
double now;
|
|
while (p->building && !p->info.pipeline && wait_ms && (now = now_ms()) < end)
|
|
plat_cond_wait_ms(&done_cv, &P.lock, (uint32_t)(end - now) + 1);
|
|
if (wait_ms) perf_end(PERF_PIPE, p0);
|
|
if (wait_ms && !p->building) perf_count(PERF_CNT_PIPE_WAIT);
|
|
}
|
|
int wait = p && p->building && !p->info.pipeline;
|
|
if (wait) P.stats.pending_skips++;
|
|
VkPipeline pipe = p ? p->info.pipeline : VK_NULL_HANDLE;
|
|
plat_mutex_unlock(&P.lock);
|
|
if (wait) {
|
|
perf_count(PERF_CNT_PIPE_SKIP);
|
|
if (pending) *pending = 1;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
if (!p) return VK_NULL_HANDLE;
|
|
if (info) *info = &p->info;
|
|
return pipe;
|
|
}
|
|
|
|
VkPipeline vk_pipeline_graphics(const VkPipeState *s, const VkPipeInfo **info) {
|
|
return graphics(s, info, 0, NULL, 0);
|
|
}
|
|
|
|
VkPipeline vk_pipeline_graphics_async(const VkPipeState *s, const VkPipeInfo **info, int *pending, uint32_t wait_ms) {
|
|
return graphics(s, info, A.on, pending, wait_ms);
|
|
}
|
|
|
|
// ---- boot prewarm from the key file (comment at "pipeline keys")
|
|
|
|
static int id_cmp(const void *x, const void *y) { return memcmp(x, y, 32); }
|
|
|
|
// Builds queued prebuild jobs until the low queue is empty or K.until passes, beside the pipe_worker pool (same
|
|
// bookkeeping). Jobs left queued at K.until stay for the pool.
|
|
static void warm_builder(void *arg) {
|
|
(void)arg;
|
|
plat_mutex_lock(&P.lock);
|
|
while (A.low && !(K.until && now_ms() >= K.until)) {
|
|
Job *j = A.low;
|
|
if ((A.low = j->next)) A.low->prev = NULL;
|
|
else A.low_tail = NULL;
|
|
j->next = j->prev = NULL;
|
|
j->low = 0; // not on a pool thread: promote() has nothing to raise
|
|
plat_mutex_unlock(&P.lock);
|
|
VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off);
|
|
plat_mutex_lock(&P.lock);
|
|
if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one
|
|
if (pipe && j->p->fast) P.g.swaps++;
|
|
j->p->building = 0;
|
|
j->p->job = NULL;
|
|
if (pipe) {
|
|
P.stats.graphics++, P.stats.async_built++, K.built++;
|
|
} else {
|
|
P.stats.failures++, K.failed++;
|
|
rt_log("vk_pipeline: graphics pipeline not built: pipeline creation failed");
|
|
}
|
|
K.promoted += j->demanded;
|
|
K.left--;
|
|
if (!--A.inflight) plat_cond_broadcast(&A.idle);
|
|
plat_cond_broadcast(&done_cv);
|
|
free(j);
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
return;
|
|
}
|
|
|
|
static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]);
|
|
static int pm_skipped(void);
|
|
static void pm_set(void);
|
|
|
|
// vk_pipeline_prewarm_progress's numbers: the phase, and the pipelines finished (built or failed) of those queued, copied
|
|
// while prewarm_run waits for its builders.
|
|
static int32_t pw_phase;
|
|
static int64_t pw_pipe_done, pw_pipe_total;
|
|
|
|
void vk_pipeline_prewarm_progress(VkPrewarmProgress *p) {
|
|
p->phase = __atomic_load_n(&pw_phase, __ATOMIC_ACQUIRE);
|
|
gfx_shader_prewarm_progress(&p->sh_done, &p->sh_total, &p->sh_translated);
|
|
p->pipe_done = __atomic_load_n(&pw_pipe_done, __ATOMIC_RELAXED);
|
|
p->pipe_total = __atomic_load_n(&pw_pipe_total, __ATOMIC_RELAXED);
|
|
}
|
|
|
|
static void prewarm_run(void);
|
|
|
|
// The boot's share of the pipeline prebuild with the graphics pipeline library. The boot builds the recorded
|
|
// pipelines in record order, so those met first in play (the title screen, the first places) come first, for at most
|
|
// this long after the shaders are ready; the rest stay queued for the pool workers, which build them below normal
|
|
// priority while the game runs. A draw that meets one of them before it is built gets a pipeline linked from stage
|
|
// libraries at once (gpl_link), as for any new pipeline in play. After a driver update, when every pipeline is built
|
|
// from nothing, the boot thus waits this long instead of for all of them (15 minutes and more with a long play
|
|
// history). Without the library a draw would have to wait for a whole build, so the boot waits for all; so it does
|
|
// with NX2NAVI_PREWARM_FULL=1.
|
|
#define PREWARM_PIPE_MS 20000.0
|
|
|
|
void vk_pipeline_prewarm(void) {
|
|
prewarm_run();
|
|
__atomic_store_n(&pw_phase, 3, __ATOMIC_RELEASE);
|
|
}
|
|
|
|
static void prewarm_run(void) {
|
|
const char *gpu = rt_config()->gpu;
|
|
uint32_t pr_hist[16] = {0}, fs_hist[4] = {0}; // library state of the recorded pipelines (lb_start)
|
|
if (!A.on || (gpu && !strcmp(gpu, "null"))) return;
|
|
if (!cfg_bool("NX2NAVI_PIPE_PREBUILD")) { // prebuild_start logs the off case
|
|
if (ensure_init()) lb_start(pr_hist, fs_hist);
|
|
return;
|
|
}
|
|
double t0 = now_ms();
|
|
K.warm = 1;
|
|
if (!ensure_init() || !K.file_n) {
|
|
rt_log("vk_pipeline: prewarm: no pipeline keys (%s)", K.path[0] ? K.path : "<none>");
|
|
if (P.ready) lb_start(pr_hist, fs_hist);
|
|
return;
|
|
}
|
|
const KeyRec *r = (const KeyRec *)K.file;
|
|
size_t n = K.file_n, m = 0;
|
|
uint8_t (*ids)[32] = malloc(3 * n * 32);
|
|
const GfxShader **sh = calloc(3 * n, sizeof *sh);
|
|
if (!ids || !sh) {
|
|
free(ids), free(sh);
|
|
return;
|
|
}
|
|
for (size_t i = 0; i < n; i++) {
|
|
memcpy(ids[m++], r[i].vs, 32);
|
|
if (r[i].has_fs) memcpy(ids[m++], r[i].fs, 32);
|
|
if (r[i].has_ts) memcpy(ids[m++], r[i].ts, 32);
|
|
}
|
|
qsort(ids, m, 32, id_cmp);
|
|
size_t u = 0;
|
|
for (size_t i = 0; i < m; i++)
|
|
if (!u || memcmp(ids[u - 1], ids[i], 32)) memcpy(ids[u++], ids[i], 32);
|
|
int cpus = (int)plat_cpu_usable();
|
|
const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset
|
|
int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2;
|
|
__atomic_store_n(&pw_phase, 1, __ATOMIC_RELEASE);
|
|
gfx_shader_prewarm((const uint8_t(*)[32])ids, u, sh, cpus, procs < 1 ? 1 : procs);
|
|
double t1 = now_ms();
|
|
vk_mem_stage_log("the shader prewarm");
|
|
|
|
if (P.indep) { // vertex output unions first: workers may take a job before the queue is complete
|
|
uint64_t pair = 0, uni = 0, pairs = 0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp);
|
|
const uint8_t (*f)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL;
|
|
if (!v || !f || !sh[v - ids] || !sh[f - ids] || gfx_shader_stage(sh[v - ids]) != GFX_STAGE_VS) continue;
|
|
GfxShaderIo io[32];
|
|
vs_outs(sh[v - ids], sh[f - ids], io);
|
|
pair += (uint64_t)__builtin_popcount(gfx_shader_inputs(sh[f - ids], NULL)), pairs++;
|
|
}
|
|
for (uint32_t j = 0; j < VS_OUTS_CAP; j++) uni += VO.vs[j] ? (uint64_t)__builtin_popcount(VO.mask[j]) : 0;
|
|
rt_log("vk_pipeline: vertex output unions: %u vertex shaders, %.2f locations each; %llu pairs read %.2f each",
|
|
VO.n, VO.n ? (double)uni / VO.n : 0.0, (unsigned long long)pairs, pairs ? (double)pair / pairs : 0.0);
|
|
}
|
|
if (pm_skipped()) { // the boot's pipeline builds killed an earlier start on this GPU and driver
|
|
plat_mutex_lock(&P.lock);
|
|
free(K.file);
|
|
K.file = NULL;
|
|
plat_mutex_unlock(&P.lock);
|
|
free(ids), free(sh);
|
|
lb_start(pr_hist, fs_hist);
|
|
return;
|
|
}
|
|
pm_set();
|
|
uint32_t queued = 0, dropped = 0, known = 0;
|
|
plat_mutex_lock(&P.lock);
|
|
K.t0 = t1;
|
|
K.loading = 1;
|
|
for (size_t i = 0; i < n; i++) {
|
|
const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp);
|
|
const uint8_t (*f)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL;
|
|
const uint8_t (*t)[32] = r[i].has_ts ? bsearch(r[i].ts, ids, u, 32, id_cmp) : NULL;
|
|
const GfxShader *vs = v ? sh[v - ids] : NULL, *fs = f ? sh[f - ids] : NULL, *ts = t ? sh[t - ids] : NULL;
|
|
if (!vs || (r[i].has_fs && !fs) || (r[i].has_ts && !ts)) {
|
|
dropped++;
|
|
continue;
|
|
}
|
|
VkPipeState s = r[i].s;
|
|
s.vs = vs, s.fs = fs, s.ts = ts;
|
|
GKey k;
|
|
gkey_fill(&s, &k);
|
|
if (!is_mesh(&s)) { // the stage libraries lb_start prebuilds are for vertex pipelines
|
|
pr_hist[(k.clamp & 1) | (k.poly_mode & 3) << 1 | (k.minus_w & 1) << 3]++;
|
|
fs_hist[k.samples >= 8 ? 3 : k.samples >= 4 ? 2 : k.samples >= 2 ? 1 : 0]++;
|
|
}
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
int q = 0;
|
|
if (pipe_find(&k, sizeof k, h)) known++;
|
|
else pipe_miss(&s, &k, h, 2, &q), queued += (uint32_t)q;
|
|
}
|
|
K.loading = 0;
|
|
plat_mutex_unlock(&P.lock);
|
|
free(ids), free(sh);
|
|
|
|
__atomic_store_n(&pw_pipe_done, 0, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&pw_pipe_total, (int64_t)queued, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&pw_phase, 2, __ATOMIC_RELEASE);
|
|
const char *full = cfg_get("NX2NAVI_PREWARM_FULL");
|
|
K.until = P.gpl && !(full && !strcmp(full, "1")) ? now_ms() + PREWARM_PIPE_MS : 0; // before the builders start
|
|
PlatThread *t[64];
|
|
int nt = 0;
|
|
for (int i = 0; i < cpus && i < 64 && queued; i++)
|
|
if ((t[nt] = plat_thread_new(warm_builder, NULL, "pipe prewarm"))) nt++;
|
|
if (!nt) warm_builder(NULL);
|
|
double logged = now_ms();
|
|
for (int i = 0; i < nt; i++)
|
|
while (!plat_thread_join_ns(t[i], 100000000u)) { // every 100 ms the numbers, every 2 s a log line
|
|
uint32_t fin = __atomic_load_n(&K.built, __ATOMIC_RELAXED) + __atomic_load_n(&K.failed, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&pw_pipe_done, (int64_t)fin, __ATOMIC_RELAXED);
|
|
if (now_ms() - logged < 2000) continue;
|
|
logged = now_ms();
|
|
rt_log("vk_pipeline: prewarm: %u of %u pipelines, %.0f ms", fin, queued, now_ms() - t1);
|
|
}
|
|
__atomic_store_n(&pw_pipe_done, (int64_t)queued, __ATOMIC_RELAXED);
|
|
plat_mutex_lock(&P.lock);
|
|
uint32_t later = 0;
|
|
for (Job *j = A.low; j; j = j->next) later += j->prebuild;
|
|
if (!later) // all built: wait for those a pool worker took
|
|
while (K.left) plat_cond_wait(&done_cv, &P.lock);
|
|
else // the boot's time is up: the pool builds the rest while the game runs
|
|
plat_cond_broadcast(&work_cv);
|
|
rt_log("vk_pipeline: prewarm: %zu records in %s: %u pipelines built, %u left to build in the background, %u "
|
|
"failed, %u already built, %u dropped (shader neither cached nor stored); shaders %.0f ms, pipelines %.0f "
|
|
"ms on %d threads, total %.0f ms", n, K.path, K.built, later, K.failed, known, dropped, t1 - t0,
|
|
now_ms() - t1, nt, now_ms() - t0);
|
|
free(K.file);
|
|
K.file = NULL;
|
|
plat_mutex_unlock(&P.lock);
|
|
fb_log("after the boot prewarm");
|
|
vk_mem_stage_log("the recorded pipelines' prebuild");
|
|
vk_pipeline_prewarm_marker_clear(); // the boot's share is done; what the pool builds later runs during play
|
|
lb_start(pr_hist, fs_hist);
|
|
}
|
|
|
|
// ---- boot prebuild crash marker
|
|
// The boot builds the recorded pipelines (warm_builder) before the game starts. A driver whose compiler crashes on one
|
|
// of them would kill every start at the same point, and the preparation journal above does not cover these builds.
|
|
// So <dir>/pipeline-prewarm-running.txt exists while they run: written before the first one, deleted when the boot's
|
|
// share ends (or when the boot stops waiting and lets the game start). A start that finds it left over records this
|
|
// GPU and driver in <dir>/pipeline-prewarm-skip.txt, and starts on them build no recorded pipelines ahead of play:
|
|
// each one is built at its first draw instead, as with NX2NAVI_PIPE_PREBUILD=0. Another driver version, or deleting
|
|
// the file, brings the prebuild back. Only the boot's own share is covered: a crash during play, where the pool also
|
|
// builds what the boot left, says nothing about these builds.
|
|
static int cache_dir(char *dir, size_t n);
|
|
static void gpu_key(char *out, size_t n);
|
|
static char PM_run[PLAT_PATH_MAX * 2 + 48];
|
|
static volatile int32_t pm_on;
|
|
|
|
static int pm_skipped(void) {
|
|
char dir[PLAT_PATH_MAX * 2], sp[sizeof PM_run], key[32];
|
|
if (!cache_dir(dir, sizeof dir)) return 0;
|
|
snprintf(PM_run, sizeof PM_run, "%s/pipeline-prewarm-running.txt", dir);
|
|
snprintf(sp, sizeof sp, "%s/pipeline-prewarm-skip.txt", dir);
|
|
gpu_key(key, sizeof key);
|
|
uint64_t n = 0;
|
|
uint8_t *left = plat_file_read(PM_run, &n);
|
|
if (left) {
|
|
char text[256];
|
|
int len = snprintf(text, sizeof text,
|
|
"# The boot's pipeline prebuild did not finish on this GPU and driver (runtime/vk_pipeline.c "
|
|
"pm_skipped): starts skip it\n# while this line matches. Delete this file to build ahead of play "
|
|
"again.\n%s\n", key);
|
|
if (plat_file_write_atomic(sp, text, (size_t)len)) plat_file_delete(PM_run);
|
|
rt_log("vk_pipeline: the last start died while the boot built pipelines (%s was left); %s now skips that "
|
|
"prebuild on this GPU and driver (%s)", PM_run, sp, key);
|
|
free(left);
|
|
}
|
|
uint8_t *skip = plat_file_read(sp, &n);
|
|
int on = 0;
|
|
for (char *line = (char *)skip, *next; line && *line && !on; line = next) {
|
|
next = strchr(line, '\n');
|
|
if (next) *next++ = 0;
|
|
size_t l = strlen(line);
|
|
while (l && (line[l - 1] == '\r' || line[l - 1] == ' ')) line[--l] = 0;
|
|
on = line[0] != '#' && !strcmp(line, key);
|
|
}
|
|
free(skip);
|
|
if (on)
|
|
rt_log("vk_pipeline: prewarm: %zu recorded pipelines not built ahead of play: an earlier start died while building "
|
|
"them on this GPU and driver (%s lists %s); each builds at its first draw", K.file_n, sp, key);
|
|
return on;
|
|
}
|
|
|
|
static void pm_set(void) {
|
|
if (!PM_run[0]) return;
|
|
char key[32];
|
|
gpu_key(key, sizeof key);
|
|
// straight to the disk: the file has to be there if the driver takes the process down a moment later
|
|
if (plat_file_write_atomic(PM_run, key, strlen(key))) plat_xchg32(&pm_on, 1);
|
|
}
|
|
|
|
void vk_pipeline_prewarm_marker_clear(void) {
|
|
if (plat_xchg32(&pm_on, 0)) plat_file_delete(PM_run);
|
|
}
|
|
|
|
// ensure_init's hook: without a prewarm (NX2NAVI_PIPE_PREBUILD=0, or the device came up first elsewhere) the records
|
|
// are not built. Caller holds no lock.
|
|
static void prebuild_start(void) {
|
|
plat_mutex_lock(&P.lock);
|
|
if (K.file && !K.warm) {
|
|
rt_log("vk_pipeline: prebuild off%s; %zu records not built",
|
|
!cfg_bool("NX2NAVI_PIPE_PREBUILD") ? " (NX2NAVI_PIPE_PREBUILD=0)" : " (no boot prewarm)", K.file_n);
|
|
free(K.file);
|
|
K.file = NULL;
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
}
|
|
|
|
// NX2NAVI_SPV_OVERRIDE=<dir> (test): a compute shader whose SPIR-V hashes (FNV-1a over its bytes, the name vk_dump.c
|
|
// gives it) to a file <dir>/<hash>.spv runs that SPIR-V instead, an edited copy with the same interface, to see a
|
|
// shader's intermediate values in a frame dump. NULL: no override (malloc'd words otherwise).
|
|
static uint32_t *spv_override(const uint32_t *code, uint64_t words, uint64_t *out) {
|
|
const char *dir = cfg_get("NX2NAVI_SPV_OVERRIDE");
|
|
if (!dir || !*dir) return NULL;
|
|
uint64_t h = 0xcbf29ce484222325ull;
|
|
for (uint64_t i = 0; i < words * 4; i++) h = (h ^ ((const uint8_t *)code)[i]) * 0x100000001b3ull;
|
|
char path[600];
|
|
snprintf(path, sizeof path, "%s/%016llx.spv", dir, (unsigned long long)h);
|
|
FILE *f = fopen(path, "rb");
|
|
if (!f) return NULL;
|
|
fseek(f, 0, SEEK_END);
|
|
long n = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
uint32_t *w = n > 20 && n % 4 == 0 ? malloc((size_t)n) : NULL;
|
|
if (w && fread(w, 1, (size_t)n, f) != (size_t)n) free(w), w = NULL;
|
|
fclose(f);
|
|
if (w) *out = (uint64_t)n / 4, rt_log("vk_pipeline: compute shader %016llx replaced by %s", (unsigned long long)h, path);
|
|
return w;
|
|
}
|
|
|
|
// A skipped froxel's exit in the fog's first compute pass: the block at word at (an OpLabel) makes five image writes, four
|
|
// of (0,0,0,1) and last the mask value (1/255,0,0,0), and returns.
|
|
static int fog_skip_block(const uint32_t *w, uint64_t n, uint64_t at, const uint64_t *def, uint32_t bound) {
|
|
int writes = 0;
|
|
for (uint64_t i = at + (w[at] >> 16); i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n) return 0;
|
|
if (op == 253) return writes == 5; // OpReturn
|
|
if (op == 248 || op == 249 || op == 250) return 0; // OpLabel, OpBranch, OpBranchConditional
|
|
if (op == 99) { // OpImageWrite
|
|
if (len < 4 || w[i + 3] >= bound || ++writes > 5) return 0;
|
|
uint64_t c = def[w[i + 3]];
|
|
if (!c || (w[c] & 0xffff) != 80 || w[c] >> 16 != 7) return 0; // OpCompositeConstruct of four
|
|
float v[4];
|
|
for (int k = 0; k < 4; k++) {
|
|
uint32_t id = w[c + 3 + k];
|
|
uint64_t f = id < bound ? def[id] : 0;
|
|
if (!f || (w[f] & 0xffff) != 43 || w[f] >> 16 != 4) return 0; // OpConstant of one word
|
|
memcpy(&v[k], &w[f + 3], 4);
|
|
}
|
|
if (writes == 5 ? !(v[0] > 0.0039f && v[0] < 0.0040f && v[1] == 0 && v[2] == 0 && v[3] == 0)
|
|
: !(v[0] == 0 && v[1] == 0 && v[2] == 0 && v[3] == 1)) return 0;
|
|
}
|
|
i += len;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// The volumetric fog's first compute pass skips a froxel behind a coarse depth table and writes it as empty, and the next
|
|
// pass's mask keeps it empty. That table comes from the frame before, so in the columns a near object has just left, every
|
|
// froxel past the object's old depth came out with no fog: a dark block of the fog grid beside the object, which the fog's
|
|
// history then smeared into a trail, and worst at the screen edges, where near objects move fastest. A froxel behind
|
|
// today's depth is never seen, so computing it only costs a little time: the comparison is made never true (x > x).
|
|
// Returns the patched copy of that shader, NULL for any other. Found by its shape rather than a hash, so it holds across
|
|
// translator updates: a branch whose other side is that exit, taken on !(distance > table) || flag.
|
|
// NX2NAVI_FOG_CULL=1 keeps the game's skip.
|
|
static uint32_t *spirv_fog_keep_froxels(const uint32_t *w, uint64_t n) {
|
|
const char *e = cfg_get("NX2NAVI_FOG_CULL");
|
|
if (n < 6 || (e && !strcmp(e, "1"))) return NULL;
|
|
uint32_t bound = w[3];
|
|
int writes = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t len = w[i] >> 16;
|
|
if (!len || i + len > n) return NULL;
|
|
writes += (w[i] & 0xffff) == 99;
|
|
i += len;
|
|
}
|
|
if (writes < 5) return NULL;
|
|
uint64_t *def = calloc(bound, sizeof *def); // word of each id's definition, for the instructions looked at below
|
|
if (!def) return NULL;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (op == 248 && len >= 2 && w[i + 1] < bound) def[w[i + 1]] = i; // OpLabel
|
|
else if ((op == 43 || op == 80 || op == 166 || op == 168 || op == 186) && len >= 3 && w[i + 2] < bound) def[w[i + 2]] = i;
|
|
i += len;
|
|
}
|
|
uint64_t cmp = 0, prev = 0;
|
|
int found = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (op == 250 && len >= 4 && prev && (w[prev] & 0xffff) == 247 && w[prev + 1] == w[i + 3] && w[i + 1] < bound &&
|
|
w[i + 3] < bound && def[w[i + 3]] && fog_skip_block(w, n, def[w[i + 3]], def, bound)) { // OpSelectionMerge just before
|
|
uint64_t c = def[w[i + 1]];
|
|
for (int k = 0; c && (w[c] & 0xffff) == 166 && k < 2; k++) { // OpLogicalOr
|
|
uint32_t a = w[c + 3 + k];
|
|
uint64_t no = a < bound ? def[a] : 0;
|
|
if (!no || (w[no] & 0xffff) != 168 || w[no + 3] >= bound) continue; // OpLogicalNot
|
|
uint64_t gt = def[w[no + 3]];
|
|
if (gt && (w[gt] & 0xffff) == 186) cmp = gt, found++; // OpFOrdGreaterThan
|
|
}
|
|
}
|
|
prev = i;
|
|
i += len;
|
|
}
|
|
free(def);
|
|
uint32_t *o = found == 1 ? malloc(n * 4) : NULL;
|
|
if (!o) return NULL;
|
|
memcpy(o, w, n * 4);
|
|
o[cmp + 4] = o[cmp + 3];
|
|
LOG_ONCE("vk_pipeline: fog: froxels behind last frame's depth are computed too");
|
|
return o;
|
|
}
|
|
|
|
// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised).
|
|
// cached_only (needs P.cache_control): from P.cache or the driver's own disk cache, never compiled; VK_NULL_HANDLE when
|
|
// neither holds it (the driver returns VK_PIPELINE_COMPILE_REQUIRED, a success code, with no pipeline).
|
|
static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cached_only) {
|
|
uint64_t words;
|
|
const uint32_t *code = gfx_shader_spirv(cs, &words);
|
|
uint64_t ow = 0;
|
|
uint32_t *over = spv_override(code, words, &ow);
|
|
if (over) code = over, words = ow;
|
|
uint32_t *fog = spirv_fog_keep_froxels(code, words);
|
|
if (fog) free(over), over = fog, code = fog;
|
|
VkShaderModule m = make_module(code, words, 0);
|
|
if (!m) return free(over), VK_NULL_HANDLE;
|
|
VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqc;
|
|
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
|
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
|
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
|
.layout = l->layout, .flags = noopt_flag()};
|
|
if (cached_only) ci.flags |= VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
|
|
pin_subgroup32(&ci.stage, &rqc, code, words);
|
|
// Subgroup operations in a workgroup whose width is a whole number of subgroups need every subgroup full: the
|
|
// driver may otherwise run partial ones, and a ballot over the culling's candidates then counted differently from
|
|
// one frame to the next
|
|
uint32_t lx = local_size_x(code, words);
|
|
if (P.f_full_subgroups && vk_caps.sg_size && lx && lx % vk_caps.sg_size == 0 && uses_subgroup_ops(code, words))
|
|
ci.stage.flags |= VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
VkPipelineCreationFeedbackCreateInfo fci;
|
|
VkPipelineCreationFeedback fb;
|
|
ci.pNext = fb_chain(&fci, &fb, ci.pNext);
|
|
crash_note("a compute pipeline", cs, NULL, NULL);
|
|
if (pipe_create(2, &ci, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
|
plat_crash_context(NULL);
|
|
if (pipe) fb_note(FB_COMPUTE, &fb);
|
|
cc_hit = !pipe ? -1 : pb_last ? 1
|
|
: !P.feedback || !P.cache || !(fb.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) ? -1
|
|
: (fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0;
|
|
if (P.feedback >= 2 && (pipe || !cached_only))
|
|
rt_log("vk_pipeline: compute %016llx: %llu words, module %016llx, %s, %.1f ms", (unsigned long long)fnv(FNV0, code, words * 4),
|
|
(unsigned long long)words, (unsigned long long)mm_hash,
|
|
!pipe ? "failed" : pb_last ? "from pipeline-binaries.bin" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled",
|
|
(double)fb.duration / 1e6);
|
|
vkDestroyShaderModule(P.device, m, NULL);
|
|
free(over);
|
|
return pipe;
|
|
}
|
|
|
|
// The layout of a compute shader in *B; NULL with *why on failure. Caller holds P.lock.
|
|
static Layout *compute_layout(const GfxShader *cs, Bindings *B, const char **why) {
|
|
Layout *l = NULL;
|
|
if (gfx_shader_stage(cs) != GFX_STAGE_CS) *why = "not a compute shader";
|
|
else if (!add_stage(B, cs, 0)) *why = "too many bindings";
|
|
else if (!(l = layout_get(B))) *why = "descriptor layout creation failed";
|
|
return l;
|
|
}
|
|
|
|
// ---- stage library prebuild
|
|
// Two places build what a draw would otherwise build while the game waits:
|
|
// - nx2navi.exe --prepare-shaders (lb_warm_prepared), after the translation, on every core: for each prepared variant
|
|
// (shader-prepared.bin) the pre-rasterization library of a vertex shader, the fragment-shader library of a fragment
|
|
// shader and the pipeline of a compute shader, each built from a private shader copy and destroyed at once. What
|
|
// stays is the driver's own disk cache: on AMD a library built again in a later process takes 0.1 ms instead of
|
|
// 55 ms (vertex) / 28 ms (fragment). Our pipeline-cache.bin is not written there:
|
|
// it would grow by about 10 KB per library (~240 MB) and be read into memory at every boot.
|
|
// - boot (lb_worker), after the pipeline-key prewarm, on two threads (lb_start) at PLAT_PRIO_IDLE (the short P.lock
|
|
// sections and shader loads at normal priority, so a preempted thread never holds the lock the replay needs; a
|
|
// compute pipeline's compile below normal priority, lb_compute):
|
|
// compute pipelines of every recorded and prepared compute shader (vk_pipeline_compute builds compute
|
|
// synchronously on the replay thread; 163 prepared): those of the shaders dispatched most recently stay, the others
|
|
// are built into the pipeline cache and destroyed (comment at compute-used.bin). Then the libraries of the first
|
|
// NX2NAVI_LIB_PREBUILD_KEEP variants play recorded (priority 0), kept in libs[] with their shaders. KEEP defaults to
|
|
// 0: a kept library costs 140-460 KB of process memory (10,000 kept took 9 GB in run 1) to save a rebuild the driver
|
|
// cache makes 0.1 ms.
|
|
// Library state: the vertex output union known now (vs_union_peek; the prewarm fills it from pipeline-keys.bin, the
|
|
// preparation has none), the clamp, polygon mode and depth mode most recorded pipelines use (defaults without a key
|
|
// file: clamp 0, fill, -w..w), single-sampled, no dual source; the state dynamic on this GPU is 0 in every key.
|
|
// A draw that meets such a variant in a new pipeline then gets its libraries from libs[] or the driver cache and
|
|
// fast-links (gpl_link). NX2NAVI_LIB_PREBUILD=0 turns the boot part off, =N stops it after N variants;
|
|
// NX2NAVI_LIB_PREBUILD_THREADS sets its thread count, NX2NAVI_LIB_PREBUILD_BIG its compute size limit (LB_BIG_WORDS).
|
|
#define LB_MAX_THREADS 16
|
|
// The boot part builds a compute shader over LB_BIG_WORDS SPIR-V words only from the pipeline caches (compute_create
|
|
// cached_only); one that no cache holds is left to its first dispatch, like any shader outside the warm list. Play
|
|
// recorded four straight-line compute shaders of 0.87-4.4 million words (one function of up to 888K instructions; the
|
|
// median of 3,648 compute shaders is 7,351 words, p99 64,844). NVIDIA 621.01 on an RTX 4090 compiled them on four
|
|
// prebuild threads for minutes at 100% CPU each while the process grew from 12 GB to 28-31 GB, until RAM ran out and the
|
|
// game stuttered and went black a few minutes after boot. None finished, so no cache held one and every boot did it
|
|
// again. NX2NAVI_LIB_PREBUILD_BIG sets the limit in words; 0 builds every size.
|
|
#define LB_BIG_WORDS (512u * 1024)
|
|
|
|
static struct {
|
|
GfxWarmEnt *e;
|
|
size_t n;
|
|
size_t n_keep; // e[0..n_keep): compute shaders whose pipeline stays (cu_keep)
|
|
volatile int64_t next;
|
|
volatile int32_t stop, running;
|
|
int threads;
|
|
PlatThread *th[LB_MAX_THREADS];
|
|
uint8_t clamp, poly, minus_w, samples; // library state (GKey fields)
|
|
double t0, ms[3]; // ms: build time per kind (vertex, fragment, compute)
|
|
// warmed: built for the caches only and destroyed; nolib: vertex and fragment variants skipped with the graphics
|
|
// pipeline library off
|
|
// skipped: variants of the skip list (SK below)
|
|
uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, skipped, done, cs_draw;
|
|
uint64_t big_words, deferred; // LB_BIG_WORDS; compute shaders over it left to their dispatch
|
|
uint64_t big_unused; // compute shaders over it that no run dispatched: not built at all
|
|
double warm_ms[3];
|
|
volatile int32_t kept; // libraries entered in libs[] so far
|
|
int32_t keep_n;
|
|
int idle; // build at PLAT_PRIO_IDLE (boot), normal (preparation)
|
|
PlatMutex stat;
|
|
} LB = {.stat = PLAT_MUTEX_INIT};
|
|
static __thread int lb_me = -1; // on a boot prebuild thread (lb_worker): its index in LB.th
|
|
|
|
// ---- preparation that outlives a driver crash. A bug in the driver's shader compiler can kill the process inside
|
|
// vkCreate*Pipelines: NVIDIA 615.78.08 on an RTX 5080 crashed on the same vertex library at the same point of every
|
|
// preparation, natively and under Wine, so preparation never finished (issue #30 and a report on PR #60).
|
|
// While lb_warm_prepared builds, slot k of <dir>/shader-prepare-journal.bin holds the variant its worker k has handed to
|
|
// the driver (input id and stage; zeros between variants). The writes go straight to the OS, so they outlast the
|
|
// process, and a run that finishes deletes the file. The next preparation or boot (skip_init) moves the variants a
|
|
// leftover journal names into <dir>/shader-prepare-skip.txt with this GPU and driver version, and preparation and the
|
|
// boot prebuild (lb_worker) skip what that file lists for them. The variants the other workers were building at the
|
|
// crash are skipped along with the culprit; like it, they build at their first draw, where the driver can still crash.
|
|
// Another driver version builds them again, and so does deleting the file. tools/prepare_shaders.py runs the build
|
|
// again (--warm-shaders) after a run that left a journal.
|
|
#define JR_SLOT 64 // bytes per journal slot: input id, stage, zeros
|
|
#define JR_SLOTS 64 // lb_warm_prepared's thread cap
|
|
|
|
static struct {
|
|
PlatFile *journal; // while lb_warm_prepared builds
|
|
uint8_t (*iid)[32]; // the skip list's variants for this GPU and driver
|
|
size_t n;
|
|
int loaded;
|
|
} SK;
|
|
|
|
static int cache_dir(char *dir, size_t n) { // the folder of pipeline-cache.bin and shader-prepared.bin
|
|
snprintf(dir, n, "%s", P.cache_path);
|
|
char *cut = plat_last_sep(dir);
|
|
if (!cut) return 0;
|
|
*cut = 0;
|
|
return 1;
|
|
}
|
|
|
|
static void gpu_key(char *out, size_t n) { // vendor:device:driver version, as the skip list names a GPU and driver
|
|
const VkPhysicalDeviceProperties *p = &vk_ctx()->props;
|
|
snprintf(out, n, "%04x:%04x:%08x", p->vendorID, p->deviceID, p->driverVersion);
|
|
}
|
|
|
|
static const char *stage_name(int st) {
|
|
return st == GFX_STAGE_VS ? "vertex" : st == GFX_STAGE_FS ? "fragment" : st == GFX_STAGE_CS ? "compute" : "other";
|
|
}
|
|
|
|
static int hex_digit(int c) {
|
|
return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1;
|
|
}
|
|
|
|
// A journal a crashed run left into the skip list, then the skip list's variants for this GPU and driver into SK.
|
|
static void skip_init(const char *dir) {
|
|
if (SK.loaded) return;
|
|
SK.loaded = 1;
|
|
char jp[PLAT_PATH_MAX * 2 + 48], sp[PLAT_PATH_MAX * 2 + 48], key[32];
|
|
snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir);
|
|
snprintf(sp, sizeof sp, "%s/shader-prepare-skip.txt", dir);
|
|
gpu_key(key, sizeof key);
|
|
uint64_t jn = 0, sn = 0;
|
|
uint8_t *j = plat_file_read(jp, &jn);
|
|
char *s = (char *)plat_file_read(sp, &sn);
|
|
int keep_journal = 0;
|
|
if (j) {
|
|
static const char head[] =
|
|
"# Shader variants the graphics driver was building when the process died during preparation\n"
|
|
"# (runtime/vk_pipeline.c skip_init): input id, stage, GPU vendor:device:driver version. Preparation skips\n"
|
|
"# them on that GPU and driver. Delete this file to build them again.\n";
|
|
size_t cap = (s ? sn : sizeof head) + (jn / JR_SLOT + 1) * 128, len = 0;
|
|
char *out = malloc(cap);
|
|
int added = 0;
|
|
if (out) {
|
|
len = s ? sn : sizeof head - 1;
|
|
memcpy(out, s ? s : head, len);
|
|
if (len && out[len - 1] != '\n') out[len++] = '\n';
|
|
for (uint64_t o = 0; o + 33 <= jn; o += JR_SLOT) {
|
|
int set = 0;
|
|
for (int b = 0; b < 32; b++) set |= j[o + b];
|
|
if (!set) continue;
|
|
char hex[65];
|
|
for (int b = 0; b < 32; b++) snprintf(hex + 2 * b, 3, "%02x", j[o + b]);
|
|
len += (size_t)snprintf(out + len, cap - len, "%s %s %s\n", hex, stage_name(j[o + 32]), key);
|
|
added++;
|
|
}
|
|
out[len] = 0; // parsed below as the list's text
|
|
}
|
|
if (added && plat_file_write_atomic(sp, out, len)) {
|
|
rt_log("vk_pipeline: the last shader preparation died while the driver built %d variants; %s now lists them "
|
|
"and preparation skips them on this GPU and driver (%s)", added, sp, key);
|
|
free(s), s = out, sn = len, out = NULL;
|
|
} else if (added) {
|
|
rt_log("vk_pipeline: cannot write %s; %s stays for the next run", sp, jp);
|
|
keep_journal = 1;
|
|
}
|
|
free(out);
|
|
free(j);
|
|
}
|
|
if (!keep_journal) plat_file_delete(jp);
|
|
for (char *line = s, *next; line && *line; line = next) {
|
|
next = strchr(line, '\n');
|
|
if (next) *next++ = 0;
|
|
char hex[65], st[16], k[32];
|
|
uint8_t iid[32];
|
|
int ok = line[0] != '#' && sscanf(line, "%64s %15s %31s", hex, st, k) == 3 && strlen(hex) == 64 &&
|
|
!strcmp(k, key);
|
|
for (int b = 0; ok && b < 32; b++) {
|
|
int hi = hex_digit(hex[2 * b]), lo = hex_digit(hex[2 * b + 1]);
|
|
if (hi < 0 || lo < 0) ok = 0;
|
|
else iid[b] = (uint8_t)(hi << 4 | lo);
|
|
}
|
|
uint8_t (*grown)[32] = ok ? realloc(SK.iid, (SK.n + 1) * sizeof *SK.iid) : NULL;
|
|
if (grown) memcpy(grown[SK.n++], iid, 32), SK.iid = grown;
|
|
}
|
|
free(s);
|
|
if (SK.n) rt_log("vk_pipeline: %zu shader variants skipped ahead of play on this GPU and driver (%s)", SK.n, sp);
|
|
}
|
|
|
|
static int skip_listed(const uint8_t iid[32]) {
|
|
for (size_t i = 0; i < SK.n; i++)
|
|
if (!memcmp(SK.iid[i], iid, 32)) return 1;
|
|
return 0;
|
|
}
|
|
|
|
static void skip_count(void) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.skipped++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
}
|
|
|
|
// Journal slot `slot`: the variant e its worker hands to the driver next, or zeros (e NULL) when the build returned.
|
|
static void journal_mark(int slot, const GfxWarmEnt *e) {
|
|
if (!SK.journal || slot < 0 || slot >= JR_SLOTS) return;
|
|
uint8_t rec[JR_SLOT] = {0};
|
|
if (e) memcpy(rec, e->iid, 32), rec[32] = e->stage;
|
|
plat_file_pwrite(SK.journal, rec, sizeof rec, (uint64_t)slot * JR_SLOT);
|
|
}
|
|
|
|
// Compute shaders whose pipeline a thread is compiling now (P.lock): a cold driver takes about 2 s for one of Ocarina's
|
|
// large compute shaders. The replay used to compile under P.lock, which held every other pipeline lookup that long, and
|
|
// compiled a second copy of a pipeline the boot prebuild (lb_compute) was already compiling.
|
|
#define CS_FLIGHT 32
|
|
static const GfxShader *cs_flight[CS_FLIGHT];
|
|
|
|
static int cs_flight_has(const GfxShader *cs) {
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (cs_flight[i] == cs) return 1;
|
|
return 0;
|
|
}
|
|
|
|
static volatile uint8_t cs_flight_want[CS_FLIGHT]; // a dispatch waits for the slot's pipeline (vk_pipeline_compute)
|
|
static PlatThread *cs_flight_th[CS_FLIGHT]; // the boot prebuild thread compiling it below normal priority
|
|
|
|
// th: the thread a waiting dispatch raises to normal priority, NULL when it compiles at normal priority already.
|
|
static int cs_flight_add(const GfxShader *cs, PlatThread *th) { // 0 when the table is full (the caller compiles anyway)
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (!cs_flight[i]) return cs_flight[i] = cs, cs_flight_want[i] = 0, cs_flight_th[i] = th, 1;
|
|
return 0;
|
|
}
|
|
|
|
// A dispatch waits for cs's build: a builder that would only have filled the pipeline cache keeps the pipeline. Returns
|
|
// the thread to raise, the first time.
|
|
static PlatThread *cs_flight_wanted(const GfxShader *cs) {
|
|
PlatThread *th = NULL;
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (cs_flight[i] == cs && !cs_flight_want[i]) cs_flight_want[i] = 1, th = cs_flight_th[i];
|
|
return th;
|
|
}
|
|
|
|
// Without P.lock, for the thread that holds cs's slot: whether a dispatch came to wait.
|
|
static int cs_flight_is_wanted(const GfxShader *cs) {
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (__atomic_load_n(&cs_flight[i], __ATOMIC_RELAXED) == cs) return cs_flight_want[i];
|
|
return 0;
|
|
}
|
|
|
|
static int cs_flight_remove(const GfxShader *cs) { // 1 when a dispatch waited for it
|
|
int want = 0;
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (cs_flight[i] == cs) cs_flight[i] = NULL, want |= cs_flight_want[i];
|
|
return want;
|
|
}
|
|
|
|
// ---- compute pipelines the boot keeps (compute-used.bin next to pipeline-keys.bin; NX2NAVI_COMPUTE_KEEP)
|
|
// The boot prebuild (lb_compute) used to build the pipeline of every compute shader of the warm list (the ones play
|
|
// recorded and the prepared ones, shader-prepared.bin) and keep all of them for the process: 4,066 on an installed
|
|
// copy (42,344 variants in its list, 42,142 of them prepared), of which a boot into the first village dispatched 95.
|
|
// What that cost was twenty of them. A copy's list has about twenty compute shaders of 0.9 to 2.0 million SPIR-V
|
|
// words: the game's tensor-core upscaler programs, which the game never dispatches natively (vk_exec_draw.c
|
|
// native_tensor) and which no run of the usage record below ever dispatched. A pipeline of one holds about 100 MiB
|
|
// of device memory from the moment it exists, used or not, and the driver keeps that memory when the pipeline is
|
|
// destroyed. On an RTX 5090 (NVIDIA 621.01) a boot pass that read all 4,066 pipelines from pipeline-cache.bin took the
|
|
// device memory from 386 to 2,398 MiB although it destroyed 3,949 of them at once; the same pass before the twenty
|
|
// were in the cache left it at 386, and building only the twenty took it to 1,370 (--probe-prewarm, 2026-11-10; the
|
|
// game showed 608 to 2,743 MiB over the same stage). The 4,046 others are cheap: all of them kept cost 2 MiB of device
|
|
// memory and 0.7 GB of the process's commit. So a copy with prepared shaders and a filled pipeline cache held 2 GiB
|
|
// for nothing from the title screen on, on cards that need that memory for textures.
|
|
// What the boot does now:
|
|
// - A compute shader over LB.big_words is built ahead only when a run has dispatched it (lb_compute, its keep
|
|
// argument). One that no run dispatched is not created at all: not from the pipeline caches and not by the
|
|
// background builder (bg_main). If the game does dispatch one for the first time, that dispatch builds it, which
|
|
// holds the frame for the compile once (0.65 to 1.9 s measured for such a shader), and the record has it built ahead
|
|
// from the next boot on.
|
|
// - Of the others the boot keeps the pipelines of the NX2NAVI_COMPUTE_KEEP (default 512) dispatched most recently, and
|
|
// builds every other one into the pipeline cache and destroys it at once (lb_compute keep 0), from a copy of its
|
|
// shader that is released again (lb_worker): the cache then holds it, so its first dispatch creates it without
|
|
// compiling. On that machine 94 first dispatches in play took 0.24 ms on average and 1.6 ms at most, 14 of them in
|
|
// the busiest 33 ms; reading the shader from the disk at that moment adds 0.05 to 0.15 ms (--probe-prewarm).
|
|
// The record: one entry per compute shader a run dispatched (its input id) with the number of the last run that did;
|
|
// each run that dispatches anything takes the next number. Kept at boot are the keep_n entries with the highest
|
|
// numbers, so a short run does not push out what a long one used, and nothing is forgotten until newer shaders take
|
|
// its place. Without a record (the first run that has one) nothing is kept and the run pays the first dispatches once.
|
|
// NX2NAVI_COMPUTE_KEEP=all builds and keeps every compute shader of the warm list, the large ones too, as before; 0
|
|
// keeps none.
|
|
#define CU_VERSION 1
|
|
typedef struct { char magic[8]; uint32_t version, rec_size, run, pad; } CuHeader;
|
|
typedef struct { uint8_t iid[32]; uint32_t run, pad; } CuRec;
|
|
static struct {
|
|
PlatMutex lock;
|
|
int opened, dirty;
|
|
char path[PLAT_PATH_MAX * 2 + 32];
|
|
CuRec *rec;
|
|
uint8_t *keep; // per record read at boot: among the keep_n dispatched most recently
|
|
uint32_t n, cap, n_file; // n_file: the records read at boot (keep has one byte for each)
|
|
uint32_t *slot; // open addressing by the id's first bytes: record index + 1, 0 empty
|
|
uint32_t slot_cap;
|
|
uint32_t run; // this run's number
|
|
int quiet; // --probe-prewarm: its dispatches are not the game's, the file stays as it is
|
|
int32_t keep_n; // NX2NAVI_COMPUTE_KEEP; -1: all
|
|
uint32_t kept; // records flagged in keep
|
|
} CU = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
// What the first dispatches of compute shaders cost this run (under P.lock): each creates its pipeline on the replay
|
|
// thread. burst: the creations that fell into one window of CF_WINDOW_MS, the worst such window kept, which is what a
|
|
// player would feel as one long frame when a new area starts.
|
|
#define CF_WINDOW_MS 33.0
|
|
static struct {
|
|
uint64_t n, hit, pre_used; // hit: the pipeline cache held it (cc_hit 1); pre_used: kept from the boot and dispatched
|
|
double ms, max_ms, hit_ms;
|
|
double win_t0, win_ms, worst_ms;
|
|
uint32_t win_n, worst_n;
|
|
} CF;
|
|
|
|
static uint32_t *cu_slot(const uint8_t iid[32]) { // the id's slot, or the empty one it would take. Caller holds CU.lock.
|
|
uint64_t h;
|
|
memcpy(&h, iid, 8);
|
|
for (uint32_t j = (uint32_t)(h * 0x9e3779b97f4a7c15ull >> 32) & (CU.slot_cap - 1);; j = (j + 1) & (CU.slot_cap - 1))
|
|
if (!CU.slot[j] || !memcmp(CU.rec[CU.slot[j] - 1].iid, iid, 32)) return &CU.slot[j];
|
|
}
|
|
|
|
// Room for one more record and a table at most half full. 0 when out of memory. Caller holds CU.lock.
|
|
static int cu_room(void) {
|
|
if (CU.n == CU.cap) {
|
|
uint32_t cap = CU.cap ? CU.cap * 2 : 1024;
|
|
CuRec *t = realloc(CU.rec, cap * sizeof *t);
|
|
if (!t) return 0;
|
|
CU.rec = t, CU.cap = cap;
|
|
}
|
|
if ((CU.n + 1) * 2 > CU.slot_cap) {
|
|
uint32_t cap = CU.slot_cap ? CU.slot_cap * 2 : 4096;
|
|
uint32_t *t = calloc(cap, sizeof *t), *old = CU.slot;
|
|
if (!t) return 0;
|
|
CU.slot = t, CU.slot_cap = cap;
|
|
for (uint32_t i = 0; i < CU.n; i++) *cu_slot(CU.rec[i].iid) = i + 1;
|
|
free(old);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
static int cu_run_cmp(const void *a, const void *b) { // {run, index} pairs, the highest run first
|
|
const uint32_t *x = a, *y = b;
|
|
return x[0] != y[0] ? (x[0] < y[0] ? 1 : -1) : (x[1] > y[1]) - (x[1] < y[1]);
|
|
}
|
|
|
|
// The number of records in a compute-used.bin image (f, size) with its header in *h, or -1 when it is not one.
|
|
static long cu_parse(const uint8_t *f, uint64_t size, CuHeader *h) {
|
|
if (!f || size < sizeof *h) return -1;
|
|
memcpy(h, f, sizeof *h);
|
|
if (memcmp(h->magic, "NX2NCUSE", 8) || h->version != CU_VERSION || h->rec_size != sizeof(CuRec)) return -1;
|
|
return (long)((size - sizeof *h) / sizeof(CuRec));
|
|
}
|
|
|
|
// Reads the record and marks what the boot keeps. Caller holds P.lock (ensure_init), after keys_open.
|
|
static void cu_open(void) {
|
|
if (!A.on || !K.path[0]) return;
|
|
const char *e = cfg_value("NX2NAVI_COMPUTE_KEEP");
|
|
plat_mutex_lock(&CU.lock);
|
|
int count = cfg_int("NX2NAVI_COMPUTE_KEEP"); // a value that is neither "all" nor a number reads as the default
|
|
CU.keep_n = !strcmp(e, "all") ? -1 : count < 0 ? 0 : count;
|
|
snprintf(CU.path, sizeof CU.path, "%s", K.path);
|
|
char *cut = plat_last_sep(CU.path);
|
|
snprintf(cut ? cut + 1 : CU.path, sizeof CU.path - (size_t)(cut ? cut + 1 - CU.path : 0), "compute-used.bin");
|
|
uint64_t size = 0;
|
|
uint8_t *f = plat_file_read(CU.path, &size);
|
|
CuHeader h;
|
|
long n = cu_parse(f, size, &h);
|
|
if (n >= 0) {
|
|
for (long i = 0; i < n && cu_room(); i++) {
|
|
CuRec r;
|
|
memcpy(&r, f + sizeof h + (size_t)i * sizeof r, sizeof r);
|
|
uint32_t *s = cu_slot(r.iid);
|
|
if (*s) continue; // an id twice: the first entry stands
|
|
CU.rec[CU.n] = r, *s = ++CU.n;
|
|
}
|
|
CU.run = h.run;
|
|
} else if (f) {
|
|
rt_log("vk_pipeline: %s has another format or version; it is rewritten", CU.path);
|
|
}
|
|
free(f);
|
|
CU.n_file = CU.n, CU.run++;
|
|
uint32_t (*order)[2] = CU.n ? malloc(CU.n * sizeof *order) : NULL;
|
|
CU.keep = CU.n ? calloc(CU.n, 1) : NULL;
|
|
if (order && CU.keep) {
|
|
for (uint32_t i = 0; i < CU.n; i++) order[i][0] = CU.rec[i].run, order[i][1] = i;
|
|
qsort(order, CU.n, sizeof *order, cu_run_cmp);
|
|
for (uint32_t i = 0; i < CU.n && (CU.keep_n < 0 || i < (uint32_t)CU.keep_n); i++) CU.keep[order[i][1]] = 1, CU.kept++;
|
|
}
|
|
free(order);
|
|
CU.opened = 1;
|
|
rt_log("vk_pipeline: %s: %u compute shaders dispatched in earlier runs; the boot keeps the pipelines of %s%u "
|
|
"dispatched most recently (NX2NAVI_COMPUTE_KEEP %s)", CU.path, CU.n,
|
|
CU.keep_n < 0 ? "every recorded compute shader, among them the " : "the ", CU.kept, e);
|
|
plat_mutex_unlock(&CU.lock);
|
|
}
|
|
|
|
// 1 when the boot keeps the pipeline of the compute shader with this input id.
|
|
static int cu_keep(const uint8_t iid[32]) {
|
|
plat_mutex_lock(&CU.lock);
|
|
int keep = CU.keep_n < 0 || !CU.opened; // without the record (no cache folder): as before, every one
|
|
if (!keep && CU.slot_cap) {
|
|
uint32_t s = *cu_slot(iid);
|
|
keep = s && s <= CU.n_file && CU.keep && CU.keep[s - 1];
|
|
}
|
|
plat_mutex_unlock(&CU.lock);
|
|
return keep;
|
|
}
|
|
|
|
// The first dispatch of cs in this run: its entry takes this run's number. The runtime's own shaders (no input id)
|
|
// are built at their first use every run and are not recorded.
|
|
static void cu_note(const GfxShader *cs) {
|
|
static const uint8_t none[32];
|
|
const uint8_t *iid = gfx_shader_input_id(cs);
|
|
if (!CU.opened || !memcmp(iid, none, 32)) return;
|
|
plat_mutex_lock(&CU.lock);
|
|
if (cu_room()) {
|
|
uint32_t *s = cu_slot(iid);
|
|
if (!*s) {
|
|
memset(&CU.rec[CU.n], 0, sizeof *CU.rec);
|
|
memcpy(CU.rec[CU.n].iid, iid, 32);
|
|
*s = ++CU.n;
|
|
}
|
|
if (CU.rec[*s - 1].run != CU.run) CU.rec[*s - 1].run = CU.run, CU.dirty = 1;
|
|
}
|
|
plat_mutex_unlock(&CU.lock);
|
|
}
|
|
|
|
// Writes the record to path. 1 when written.
|
|
// One writer at a time, from the copy of the record to the end of the file write: the autosave thread and a save at
|
|
// exit can both be here, and the older copy must not land on the newer file.
|
|
static PlatMutex cu_io = PLAT_MUTEX_INIT;
|
|
|
|
static int cu_write(const char *path) {
|
|
plat_mutex_lock(&cu_io);
|
|
plat_mutex_lock(&CU.lock);
|
|
uint32_t n = CU.n;
|
|
plat_mutex_unlock(&CU.lock);
|
|
// allocated outside CU.lock, which a dispatch takes (cu_note); room for the entries that come meanwhile
|
|
size_t cap = sizeof(CuHeader) + ((size_t)n + 256) * sizeof(CuRec), size = 0;
|
|
uint8_t *out = malloc(cap);
|
|
plat_mutex_lock(&CU.lock);
|
|
CuHeader h = {{'N', 'X', '2', 'N', 'C', 'U', 'S', 'E'}, CU_VERSION, sizeof(CuRec), CU.run, 0};
|
|
if (out && sizeof h + CU.n * sizeof(CuRec) <= cap) {
|
|
size = sizeof h + CU.n * sizeof(CuRec);
|
|
memcpy(out, &h, sizeof h);
|
|
if (CU.n) memcpy(out + sizeof h, CU.rec, CU.n * sizeof(CuRec));
|
|
CU.dirty = 0;
|
|
}
|
|
plat_mutex_unlock(&CU.lock);
|
|
int ok = size && plat_file_write_atomic(path, out, size);
|
|
if (!ok) { // still to be written: the next flush tries again
|
|
plat_mutex_lock(&CU.lock);
|
|
CU.dirty = 1;
|
|
plat_mutex_unlock(&CU.lock);
|
|
}
|
|
free(out);
|
|
plat_mutex_unlock(&cu_io);
|
|
return ok;
|
|
}
|
|
|
|
// Writes the record when a dispatch changed it (the autosave thread and the save at exit).
|
|
static void cu_flush(void) {
|
|
if (!CU.opened || CU.quiet || !*(volatile int *)&CU.dirty) return;
|
|
if (!cu_write(CU.path)) LOG_ONCE("vk_pipeline: cannot write %s; the next saves try again", CU.path);
|
|
}
|
|
|
|
// A compute pipeline created at a dispatch took ms. Caller holds P.lock.
|
|
static void cf_note(double ms, int hit) {
|
|
double now = now_ms();
|
|
CF.n++, CF.ms += ms;
|
|
if (ms > CF.max_ms) CF.max_ms = ms;
|
|
if (hit == 1) CF.hit++, CF.hit_ms += ms;
|
|
if (now - CF.win_t0 > CF_WINDOW_MS) CF.win_t0 = now, CF.win_n = 0, CF.win_ms = 0;
|
|
CF.win_n++, CF.win_ms += ms;
|
|
if (CF.win_ms > CF.worst_ms) CF.worst_ms = CF.win_ms, CF.worst_n = CF.win_n;
|
|
}
|
|
|
|
static void cf_log(void) {
|
|
plat_mutex_lock_shared(&P.lock);
|
|
typeof(CF) c = CF; // the line is written after the unlock: every dispatch needs P.lock, and this
|
|
plat_mutex_unlock_shared(&P.lock); // runs on the autosave thread below normal priority
|
|
if (c.n || c.pre_used)
|
|
rt_log("vk_pipeline: compute pipelines created at their first dispatch: %llu (avg %.2f ms, longest %.2f ms; the "
|
|
"pipeline cache held %llu of them, avg %.2f ms); the %.0f ms that held the most of it: %u pipelines, "
|
|
"%.1f ms; of the compute pipelines kept from the boot, %llu were dispatched",
|
|
(unsigned long long)c.n, c.n ? c.ms / c.n : 0.0, c.max_ms, (unsigned long long)c.hit,
|
|
c.hit ? c.hit_ms / c.hit : 0.0, CF_WINDOW_MS, c.worst_n, c.worst_ms, (unsigned long long)c.pre_used);
|
|
}
|
|
|
|
// The background builder of large compute shaders (comment at bg_main).
|
|
static struct {
|
|
const GfxShader **cs;
|
|
uint8_t (*iid)[32];
|
|
uint64_t *words;
|
|
size_t n, cap;
|
|
PlatThread *th;
|
|
const GfxShader *volatile building; // the shader on the driver now, NULL between builds
|
|
volatile int32_t raised; // a dispatch waited for it and raised the thread
|
|
uint64_t built, failed, there;
|
|
double ms;
|
|
} BG;
|
|
|
|
VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) {
|
|
if (info) *info = NULL;
|
|
if (!cs || !ensure_init()) return VK_NULL_HANDLE;
|
|
struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
plat_mutex_lock(&P.lock);
|
|
Pipe *p = pipe_find(&k, sizeof k, h);
|
|
double w0 = !p && cs_flight_has(cs) ? now_ms() : 0;
|
|
int bg = 0;
|
|
while (!p && cs_flight_has(cs)) { // another thread is compiling it: its result, not a second compile
|
|
// a build that only fills the pipeline cache keeps its pipeline for this dispatch, and a boot prebuild thread
|
|
// compiling it below normal priority goes to normal
|
|
PlatThread *builder = cs_flight_wanted(cs);
|
|
if (builder) plat_thread_set_priority(builder, PLAT_PRIO_NORMAL);
|
|
if (BG.building == cs && BG.th && !plat_xchg32(&BG.raised, 1)) // the background builder: normal priority now
|
|
plat_thread_set_priority(BG.th, PLAT_PRIO_NORMAL), bg = 1;
|
|
plat_mutex_unlock(&P.lock);
|
|
plat_sleep_ms(1);
|
|
plat_mutex_lock(&P.lock);
|
|
p = pipe_find(&k, sizeof k, h);
|
|
}
|
|
if (w0 && now_ms() - w0 >= 2.0) {
|
|
static int logs;
|
|
if (logs++ < 16)
|
|
rt_log("vk_exec: a dispatch waited %.0f ms for its compute pipeline (%s)", now_ms() - w0,
|
|
bg ? "built on the background thread" : "built by another thread");
|
|
}
|
|
if (p) {
|
|
P.stats.hits++;
|
|
} else {
|
|
const char *why = NULL;
|
|
Bindings B = {0};
|
|
Layout *l = compute_layout(cs, &B, &why);
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (!why) {
|
|
int flight = cs_flight_add(cs, NULL);
|
|
plat_mutex_unlock(&P.lock); // the compile runs without the lock
|
|
uint64_t p0 = perf_begin();
|
|
double c0 = now_ms();
|
|
pipe = compute_create(cs, l, 0);
|
|
double cms = now_ms() - c0;
|
|
int hit = cc_hit;
|
|
perf_end(PERF_PIPE, p0);
|
|
plat_mutex_lock(&P.lock);
|
|
if (flight) cs_flight_remove(cs);
|
|
if (!pipe) why = "pipeline creation failed";
|
|
else cf_note(cms, hit);
|
|
}
|
|
if ((p = pipe_find(&k, sizeof k, h))) { // another thread added it meanwhile (no flight slot was free)
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
} else {
|
|
if (why) {
|
|
P.stats.failures++;
|
|
rt_log("vk_pipeline: compute pipeline not built: %s", why);
|
|
} else {
|
|
P.stats.compute++;
|
|
LB.cs_draw++;
|
|
}
|
|
p = pipe_add(&k, sizeof k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_COMPUTE, 0);
|
|
}
|
|
}
|
|
if (p && !p->used) { // the first dispatch of this run: the boot of the next runs keeps this shader's pipeline
|
|
p->used = 1;
|
|
CF.pre_used += p->pre;
|
|
if (p->info.pipeline) cu_note(cs);
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
if (!p) return VK_NULL_HANDLE;
|
|
if (info) *info = &p->info;
|
|
return p->info.pipeline;
|
|
}
|
|
|
|
static void lb_lock(void) {
|
|
plat_thread_priority(PLAT_PRIO_NORMAL);
|
|
plat_mutex_lock(&P.lock);
|
|
}
|
|
|
|
static void lb_unlock(void) {
|
|
plat_mutex_unlock(&P.lock);
|
|
if (LB.idle) plat_thread_priority(PLAT_PRIO_IDLE);
|
|
}
|
|
|
|
static void lb_count(int kind, int what, double ms) { // what: 0 built, 1 already there, 2 failed, 3 cache only
|
|
plat_mutex_lock(&LB.stat);
|
|
if (what == 3) LB.warmed[kind]++, LB.warm_ms[kind] += ms;
|
|
else (what == 0 ? LB.built : what == 1 ? LB.have : LB.fail)[kind]++, LB.ms[kind] += ms;
|
|
plat_mutex_unlock(&LB.stat);
|
|
}
|
|
|
|
// vs_union_missing for vs alone, without entering vs into the union table (it holds VS_OUTS_CAP shaders).
|
|
static uint32_t vs_union_peek(const GfxShader *vs, GfxShaderIo fin[32]) {
|
|
GfxShaderIo vout[32];
|
|
uint32_t need = 0, have = gfx_shader_outputs(vs, vout), missing = 0;
|
|
memset(fin, 0, 32 * sizeof *fin);
|
|
plat_mutex_lock_shared(&VO.lock);
|
|
uint32_t j = (uint32_t)(((uintptr_t)vs >> 4) * 0x9e3779b1u) & (VS_OUTS_CAP - 1);
|
|
while (VO.vs[j] && VO.vs[j] != vs) j = (j + 1) & (VS_OUTS_CAP - 1);
|
|
if (VO.vs[j]) need = VO.mask[j], memcpy(fin, VO.io[j], 32 * sizeof *fin);
|
|
plat_mutex_unlock_shared(&VO.lock);
|
|
if (P.vs_outs_all)
|
|
for (int i = 0; i < 32; i++)
|
|
if (!(need >> i & 1)) fin[i] = (GfxShaderIo){GFX_TYPE_FLOAT, 4, 0}, need |= 1u << i;
|
|
for (int i = 0; i < 32; i++)
|
|
if (need >> i & 1 && !(have >> i & 1)) missing |= 1u << i;
|
|
return missing;
|
|
}
|
|
|
|
// A compute shader over LB.big_words that no pipeline cache holds: its first dispatch builds it (vk_pipeline_compute).
|
|
static void lb_defer(const uint8_t iid[32], uint64_t words) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.deferred++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
rt_log("vk_pipeline: stage library prebuild: compute shader %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) %s: "
|
|
"left to its first dispatch (NX2NAVI_LIB_PREBUILD_BIG %llu)", iid[0], iid[1], iid[2], iid[3], iid[4], iid[5],
|
|
iid[6], iid[7], (unsigned long long)words,
|
|
P.cache_control ? "not in the pipeline caches" : "too large to build without a cache probe (no pipelineCreationCacheControl)",
|
|
(unsigned long long)LB.big_words);
|
|
}
|
|
|
|
// ---- large compute shaders on a background thread (NX2NAVI_LIB_PREBUILD_BIG_BG, default 1; 0 leaves them to their
|
|
// first dispatch as before)
|
|
// The boot prebuild leaves a compute shader over LB.big_words that no pipeline cache holds (lb_compute). Its first
|
|
// dispatch then compiled it on the replay thread: frames of 0.65-1.9 s in Kokiri Forest on an RTX 5090 after a
|
|
// translator change. One thread builds them instead, once the boot prebuild is done, one at a time and the smallest
|
|
// first: four compiled side by side grew the process by 16-19 GB on an RTX 4090 (comment at LB_BIG_WORDS). The shader
|
|
// being built is in cs_flight, so a dispatch that needs it waits for this build (vk_pipeline_compute raises the
|
|
// thread to normal priority then) instead of compiling a second copy. What is built is entered for the process, as
|
|
// lb_compute's pipelines are, and lands in pipeline-cache.bin for the next boot.
|
|
|
|
static int bg_on(void) {
|
|
static int on = -1;
|
|
if (on < 0) on = cfg_bool("NX2NAVI_LIB_PREBUILD_BIG_BG");
|
|
return on;
|
|
}
|
|
|
|
// A large compute shader no cache holds goes to the background builder (lb_worker threads; under LB.stat).
|
|
static void bg_queue(const GfxShader *cs, const uint8_t iid[32], uint64_t words) {
|
|
plat_mutex_lock(&LB.stat);
|
|
if (BG.n == BG.cap) {
|
|
size_t cap = BG.cap ? BG.cap * 2 : 16;
|
|
const GfxShader **c = realloc(BG.cs, cap * sizeof *c);
|
|
if (c) BG.cs = c;
|
|
uint8_t (*i)[32] = realloc(BG.iid, cap * sizeof *i);
|
|
if (i) BG.iid = i;
|
|
uint64_t *w = realloc(BG.words, cap * sizeof *w);
|
|
if (w) BG.words = w;
|
|
if (c && i && w) BG.cap = cap;
|
|
}
|
|
int ok = BG.n < BG.cap;
|
|
if (ok) BG.cs[BG.n] = cs, memcpy(BG.iid[BG.n], iid, 32), BG.words[BG.n] = words, BG.n++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
if (!ok) {
|
|
lb_defer(iid, words);
|
|
return;
|
|
}
|
|
rt_log("vk_pipeline: stage library prebuild: compute shader %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) not in "
|
|
"the pipeline caches: built on the background thread after the prebuild (NX2NAVI_LIB_PREBUILD_BIG_BG)",
|
|
iid[0], iid[1], iid[2], iid[3], iid[4], iid[5], iid[6], iid[7], (unsigned long long)words);
|
|
}
|
|
|
|
// NX2NAVI_BIG_BUILD_FREE_MB (default 8192; 0 no check): the system commit headroom a large compute build waits for.
|
|
// A build of one of these shaders took the process up by 1-3 GB on an RTX 5090 and four at once by 16-19 GB on an
|
|
// RTX 4090; started on a machine near its commit limit it can end the game. While the headroom is short the shader
|
|
// waits, and if the game dispatches it first it is built there, as without the background builder.
|
|
static int bg_room(double *used, double *limit) {
|
|
static double need = -1;
|
|
if (need < 0) need = strtod(cfg_value("NX2NAVI_BIG_BUILD_FREE_MB"), NULL);
|
|
double proc;
|
|
plat_commit_info(&proc, used, limit);
|
|
return need <= 0 || *limit < 0 || *limit - *used >= need;
|
|
}
|
|
|
|
// NX2NAVI_BIG_BUILD_MAX_MB (default 4096; 0 no cap): how far the builder may raise the process commit before it stops
|
|
// and leaves the rest to their first dispatch. One of these shaders costs the driver 1-4 GB: on a 31 GB machine four of
|
|
// them took the process from 15 to 29 GB and system commit to 60 of 65 GB, which is where a player feels the game
|
|
// stutter. The builder exists to remove hitches, not to trade them for paging, so a machine that cannot hold them all
|
|
// builds only what it can and pays one hitch per shader later.
|
|
static double bg_growth_cap(void) {
|
|
static double cap = -1;
|
|
if (cap < 0) cap = strtod(cfg_value("NX2NAVI_BIG_BUILD_MAX_MB"), NULL);
|
|
return cap;
|
|
}
|
|
|
|
// The builder compiles into a pipeline cache of its own (own_cache at pipe_create_in), not into P.cache. A create into
|
|
// P.cache holds cache_rw shared from start to end, and the cache is only saved at a moment when nobody holds it: on an
|
|
// RX 7900 XT (26.10.1) the largest of these shaders (4,874,502 SPIR-V words) was still compiling 590 s after the boot,
|
|
// pipeline-cache.bin was not written once in that time, and the game was closed before the build ended, so the next
|
|
// boot found neither that shader nor the pipelines built in play and started the same build again. bg_merge moves the
|
|
// result into P.cache when the build is done, at a moment with no build into P.cache running: the lock is tried as
|
|
// cache_save tries it, because a waiting exclusive request would hold back every new build.
|
|
static void bg_merge(VkPipelineCache own, int built) {
|
|
int locked = 0;
|
|
while (built && !LB.stop && !(locked = plat_mutex_trylock(&cache_rw))) plat_sleep_ms(2);
|
|
if (locked) {
|
|
VkResult r = vkMergePipelineCaches(P.device, P.cache, 1, &own);
|
|
plat_mutex_unlock(&cache_rw);
|
|
if (r == VK_SUCCESS) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED);
|
|
else rt_log("vk_pipeline: background compute build: its cache did not merge into %s (%d)", P.cache_path, (int)r);
|
|
}
|
|
vkDestroyPipelineCache(P.device, own, NULL);
|
|
}
|
|
|
|
static void bg_main(void *arg) {
|
|
(void)arg;
|
|
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
// The queue can grow while this thread builds (bg_queue runs on the replay thread and reallocs these arrays), so
|
|
// the sort and every read below happen under LB.stat; an entry is copied out before its compile so no pointer into
|
|
// the arrays is held across it.
|
|
plat_mutex_lock(&LB.stat);
|
|
for (size_t i = 1; i < BG.n; i++) // smallest first: insertion sort of a handful
|
|
for (size_t j = i; j > 0 && BG.words[j] < BG.words[j - 1]; j--) {
|
|
const GfxShader *c = BG.cs[j];
|
|
BG.cs[j] = BG.cs[j - 1], BG.cs[j - 1] = c;
|
|
uint64_t w = BG.words[j];
|
|
BG.words[j] = BG.words[j - 1], BG.words[j - 1] = w;
|
|
uint8_t t[32];
|
|
memcpy(t, BG.iid[j], 32), memcpy(BG.iid[j], BG.iid[j - 1], 32), memcpy(BG.iid[j - 1], t, 32);
|
|
}
|
|
plat_mutex_unlock(&LB.stat);
|
|
double t_all = now_ms();
|
|
double c_start, c_used0, c_limit0;
|
|
plat_commit_info(&c_start, &c_used0, &c_limit0);
|
|
for (size_t i = 0; !LB.stop; i++) {
|
|
uint8_t idbuf[32];
|
|
uint64_t words = 0;
|
|
const GfxShader *cs;
|
|
plat_mutex_lock(&LB.stat);
|
|
int have = i < BG.n;
|
|
if (have) {
|
|
cs = BG.cs[i];
|
|
memcpy(idbuf, BG.iid[i], 32);
|
|
words = BG.words[i];
|
|
}
|
|
plat_mutex_unlock(&LB.stat);
|
|
if (!have) break;
|
|
const uint8_t *iid = idbuf;
|
|
struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
Bindings B = {0};
|
|
const char *why = NULL;
|
|
double used, limit, c_now;
|
|
plat_commit_info(&c_now, &used, &limit);
|
|
double cap = bg_growth_cap();
|
|
if (cap > 0 && c_now - c_start >= cap) {
|
|
rt_log("vk_pipeline: background compute build: stopping, the process grew %.0f MB in all (NX2NAVI_BIG_BUILD_MAX_MB "
|
|
"%.0f); the shaders left build at their first dispatch", c_now - c_start, cap);
|
|
break;
|
|
}
|
|
int waited = 0, no_room = 0;
|
|
double w0 = now_ms();
|
|
while (!LB.stop && !bg_room(&used, &limit)) { // not under P.lock: a dispatch builds it meanwhile
|
|
if (!waited++)
|
|
rt_log("vk_pipeline: background compute build: waiting for commit headroom (system %.0f of %.0f MB, "
|
|
"NX2NAVI_BIG_BUILD_FREE_MB %s)", used, limit, cfg_value("NX2NAVI_BIG_BUILD_FREE_MB"));
|
|
if (now_ms() - w0 > 30000.0) { // waiting is only useful when the headroom will come back
|
|
rt_log("vk_pipeline: background compute build: no commit headroom after %.0f s; the shaders left build at "
|
|
"their first dispatch", (now_ms() - w0) / 1000.0);
|
|
no_room = 1;
|
|
break;
|
|
}
|
|
plat_sleep_ms(2000);
|
|
}
|
|
if (LB.stop || no_room) break;
|
|
plat_mutex_lock(&P.lock);
|
|
int there = cs_flight_has(cs) || pipe_find(&k, sizeof k, h); // a dispatch got to it first
|
|
Layout *l = there ? NULL : compute_layout(cs, &B, &why);
|
|
int flight = l ? cs_flight_add(cs, NULL) : 0; // vk_pipeline_compute raises this thread through BG.raised
|
|
if (l) BG.building = cs;
|
|
plat_mutex_unlock(&P.lock);
|
|
if (!l) {
|
|
if (there) BG.there++;
|
|
else BG.failed++, rt_log("vk_pipeline: background compute build: %s", why ? why : "no layout");
|
|
continue;
|
|
}
|
|
VkPipelineCache own = VK_NULL_HANDLE;
|
|
VkPipelineCacheCreateInfo oci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO};
|
|
if (P.cache && vkCreatePipelineCache(P.device, &oci, NULL, &own) != VK_SUCCESS) own = VK_NULL_HANDLE;
|
|
double t0 = now_ms();
|
|
own_cache = own; // without one the build goes into P.cache as before
|
|
VkPipeline pipe = compute_create(cs, l, 0);
|
|
own_cache = VK_NULL_HANDLE;
|
|
double ms = now_ms() - t0;
|
|
// The pipeline stays: only a shader a run dispatched is queued here (lb_compute), and it is entered before
|
|
// the merge below, so a dispatch that comes meanwhile finds it.
|
|
plat_mutex_lock(&P.lock);
|
|
if (flight) cs_flight_remove(cs);
|
|
BG.building = NULL;
|
|
Pipe *e = NULL;
|
|
if (pipe && !pipe_find(&k, sizeof k, h) &&
|
|
(e = pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0))) {
|
|
e->pre = 1;
|
|
P.stats.compute++;
|
|
} else if (pipe) { // a dispatch built its own meanwhile (no flight slot was free), or out of memory
|
|
vkDestroyPipeline(P.device, pipe, NULL);
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
if (own) bg_merge(own, pipe != VK_NULL_HANDLE); // after the entry: a dispatch waiting for it does not wait for this
|
|
if (plat_xchg32(&BG.raised, 0)) plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
if (pipe) BG.built++, BG.ms += ms;
|
|
else BG.failed++;
|
|
double c_proc, c_used, c_limit;
|
|
plat_commit_info(&c_proc, &c_used, &c_limit);
|
|
rt_log("vk_pipeline: background compute build: %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) %s in %.0f ms; "
|
|
"commit: process %.0f MB, system %.0f of %.0f MB", iid[0], iid[1], iid[2], iid[3], iid[4], iid[5], iid[6],
|
|
iid[7], (unsigned long long)words, pipe ? "built" : "failed", ms, c_proc, c_used, c_limit);
|
|
}
|
|
rt_log("vk_pipeline: background compute builds %s: %llu built (%.0f ms in all), %llu failed, %llu dispatched before "
|
|
"their turn; %.0f ms after the prebuild", LB.stop ? "stopped" : "done", (unsigned long long)BG.built, BG.ms,
|
|
(unsigned long long)BG.failed, (unsigned long long)BG.there, now_ms() - t_all);
|
|
}
|
|
|
|
// The last boot prebuild worker starts the builder when it queued anything.
|
|
static void bg_start(void) {
|
|
if (!BG.n || LB.stop || BG.th) return;
|
|
if (!(BG.th = plat_thread_new(bg_main, NULL, "big compute")))
|
|
rt_log("vk_pipeline: cannot start the background compute builder; %zu large compute shaders build at their first "
|
|
"dispatch", BG.n);
|
|
}
|
|
|
|
// keep 0: the pipeline is built for the pipeline cache only and destroyed at once, unless a dispatch came to wait for
|
|
// it meanwhile; a shader over LB.big_words is then not built at all (comment at compute-used.bin). cs may be a copy
|
|
// the caller releases afterwards when keep is 0: nothing here holds on to it then.
|
|
static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep) {
|
|
struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
Bindings B = {0};
|
|
const char *why = NULL;
|
|
uint64_t words = 0;
|
|
gfx_shader_spirv(cs, &words);
|
|
int big = LB.big_words && words > LB.big_words; // only from the caches (LB_BIG_WORDS)
|
|
if (big && !keep) { // no run dispatched it: a pipeline of it would hold about 100 MiB of device memory unused
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.big_unused++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
return;
|
|
}
|
|
if (big && !P.cache_control) {
|
|
if (bg_on()) bg_queue(cs, iid, words);
|
|
else lb_defer(iid, words);
|
|
return;
|
|
}
|
|
lb_lock();
|
|
int busy = cs_flight_has(cs); // the replay compiles it now (vk_pipeline_compute)
|
|
Layout *l = busy || pipe_find(&k, sizeof k, h) ? NULL : compute_layout(cs, &B, &why);
|
|
PlatThread *me = LB.idle && lb_me >= 0 ? LB.th[lb_me] : NULL;
|
|
int have = !l && !why, flight = l ? cs_flight_add(cs, me) : 0;
|
|
lb_unlock();
|
|
if (!l) {
|
|
lb_count(2, have ? 1 : 2, 0);
|
|
return;
|
|
}
|
|
double t0 = now_ms();
|
|
// The compile runs below normal priority: after an update or a driver update every one of these pipelines is
|
|
// compiled again (about 140 ms each on an RTX 5090's driver, minutes of two threads for a long play history), and
|
|
// at normal priority those threads took cores from the game's own on a CPU with few of them. While the shader is in
|
|
// flight a dispatch of it waits for this build (vk_pipeline_compute) and raises the thread to normal: a compile left
|
|
// at idle priority under load held that frame for seconds. The dispatch may have come between the unlock and the
|
|
// call below, which would undo its raise, hence the look at the slot after it. Without a slot or a handle nobody
|
|
// could raise the thread, so it compiles at normal priority.
|
|
if (LB.idle && flight && me) {
|
|
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
if (cs_flight_is_wanted(cs)) plat_thread_priority(PLAT_PRIO_NORMAL);
|
|
} else if (LB.idle) {
|
|
plat_thread_priority(PLAT_PRIO_NORMAL);
|
|
}
|
|
VkPipeline pipe = compute_create(cs, l, big);
|
|
double ms = now_ms() - t0;
|
|
int compiled = pipe && cc_hit != 1, kept = 0;
|
|
lb_lock();
|
|
if (flight && cs_flight_remove(cs)) keep = 1; // a dispatch waits for it
|
|
Pipe *e = NULL;
|
|
if (pipe && pipe_find(&k, sizeof k, h)) {
|
|
vkDestroyPipeline(P.device, pipe, NULL);
|
|
have = 1;
|
|
} else if (pipe && keep && (e = pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0))) {
|
|
e->pre = 1, kept = 1;
|
|
P.stats.compute++;
|
|
} else if (pipe) {
|
|
vkDestroyPipeline(P.device, pipe, NULL);
|
|
// P.stats.compute does not count it, so the autosave would not see what its compile added to P.cache
|
|
if (compiled) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED);
|
|
}
|
|
lb_unlock();
|
|
if (!pipe && big && bg_on()) bg_queue(cs, iid, words);
|
|
else if (!pipe && big) lb_defer(iid, words);
|
|
else lb_count(2, !pipe ? 2 : have ? 1 : kept ? 0 : 3, ms);
|
|
}
|
|
|
|
// The pre-rasterization (vertex) or fragment-shader library of one shader with the LB state.
|
|
// keep 0: build into the driver's pipeline cache only (sh is a private copy; the library is destroyed).
|
|
static void lb_library(const GfxShader *sh, int vertex, int keep) {
|
|
int kind = vertex ? LIB_PR : LIB_FS, stat = vertex ? 0 : 1;
|
|
VkPipeState s;
|
|
vk_pipe_state_defaults(&s);
|
|
GKey k;
|
|
memset(&k, 0, sizeof k);
|
|
k.clamp = LB.clamp, k.poly_mode = LB.poly, k.minus_w = LB.minus_w, k.samples = LB.samples;
|
|
PrKey pk;
|
|
FsKey fk;
|
|
memset(&pk, 0, sizeof pk), memset(&fk, 0, sizeof fk);
|
|
const void *key;
|
|
uint32_t size;
|
|
if (vertex) {
|
|
GfxShaderIo fin[32];
|
|
s.vs = sh;
|
|
pk.kind = LIB_PR, pk.vs = sh, pk.clamp = k.clamp, pk.poly_mode = k.poly_mode, pk.minus_w = k.minus_w;
|
|
pk.missing = vs_union_peek(sh, fin);
|
|
for (int i = 0; i < 32; i++)
|
|
if (pk.missing >> i & 1) pk.fin[i] = fin[i];
|
|
key = &pk, size = sizeof pk;
|
|
} else {
|
|
s.fs = sh;
|
|
fk.kind = LIB_FS, fk.fs = sh, fk.samples = k.samples;
|
|
key = &fk, size = sizeof fk;
|
|
}
|
|
uint64_t h = key_hash(key, size);
|
|
Bindings B = {0}, F = {0};
|
|
VkPipelineLayout lay = VK_NULL_HANDLE;
|
|
lb_lock();
|
|
int have = keep && lib_find(h, key, size) != NULL;
|
|
if (!have && add_stage(vertex ? &B : &F, sh, 0)) {
|
|
Layout *l = layout_pair(&B, &F);
|
|
if (l) lay = half_layout(l, vertex ? 0 : 1);
|
|
}
|
|
lb_unlock();
|
|
if (have || !lay) {
|
|
lb_count(stat, have ? 1 : 2, 0);
|
|
return;
|
|
}
|
|
double t0 = now_ms();
|
|
VkPipeline pipe = lib_create(kind, &s, &k, lay, key);
|
|
double ms = now_ms() - t0;
|
|
if (!keep) {
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
lb_count(stat, pipe ? 3 : 2, ms);
|
|
return;
|
|
}
|
|
lb_lock();
|
|
have = lib_find(h, key, size) != NULL;
|
|
VkPipeline got = lib_put(h, key, size, pipe);
|
|
if (!have && pipe) {
|
|
Lib *b = lib_find(h, key, size);
|
|
if (b) b->pre = 1;
|
|
}
|
|
lb_unlock();
|
|
lb_count(stat, !pipe ? 2 : have || got != pipe ? 1 : 0, ms);
|
|
}
|
|
|
|
// A vertex or fragment variant with the graphics pipeline library off: no stage library to build ahead.
|
|
static void lb_skip_lib(void) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.nolib++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
}
|
|
|
|
static void lb_report(const char *how) {
|
|
float mem_used, mem_total, mem_proc;
|
|
plat_mem_info(&mem_used, &mem_total, &mem_proc);
|
|
plat_mutex_lock_shared(&LB.stat);
|
|
rt_log("vk_pipeline: stage library prebuild %s: %llu of %zu variants in %.0f ms; kept: vertex %llu built (avg %.1f ms), "
|
|
"%llu there; fragment %llu built (avg %.1f ms), %llu there; compute %llu built (avg %.1f ms), %llu there; driver "
|
|
"cache only: vertex %llu (avg %.1f ms), fragment %llu (avg %.1f ms), compute %llu (avg %.1f ms); %llu failed, %llu not in the shader cache, "
|
|
"%llu vertex/fragment skipped (graphics pipeline library off), %llu skipped (the driver crashed on them before, "
|
|
"shader-prepare-skip.txt), %llu large compute shaders left to their first dispatch (NX2NAVI_LIB_PREBUILD_BIG), %zu "
|
|
"queued for the background builder (NX2NAVI_LIB_PREBUILD_BIG_BG), %llu large compute shaders not built because "
|
|
"no run has dispatched them; process working set %.0f MB", how, (unsigned long long)LB.done, LB.n,
|
|
now_ms() - LB.t0, (unsigned long long)LB.built[0], LB.built[0] ? LB.ms[0] / LB.built[0] : 0.0,
|
|
(unsigned long long)LB.have[0], (unsigned long long)LB.built[1], LB.built[1] ? LB.ms[1] / LB.built[1] : 0.0,
|
|
(unsigned long long)LB.have[1], (unsigned long long)LB.built[2], LB.built[2] ? LB.ms[2] / LB.built[2] : 0.0,
|
|
(unsigned long long)LB.have[2], (unsigned long long)LB.warmed[0], LB.warmed[0] ? LB.warm_ms[0] / LB.warmed[0] : 0.0,
|
|
(unsigned long long)LB.warmed[1], LB.warmed[1] ? LB.warm_ms[1] / LB.warmed[1] : 0.0,
|
|
(unsigned long long)LB.warmed[2], LB.warmed[2] ? LB.warm_ms[2] / LB.warmed[2] : 0.0,
|
|
(unsigned long long)(LB.fail[0] + LB.fail[1] + LB.fail[2]),
|
|
(unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (unsigned long long)LB.skipped,
|
|
(unsigned long long)LB.deferred, BG.n, (unsigned long long)LB.big_unused, (double)mem_proc);
|
|
plat_mutex_unlock_shared(&LB.stat);
|
|
}
|
|
|
|
// Boot: compute pipelines (the first LB.n_keep entered for the process, the others only built into the pipeline
|
|
// cache) and kept libraries.
|
|
static void lb_worker(void *arg) {
|
|
lb_me = (int)(intptr_t)arg; // its handle is LB.th[lb_me]
|
|
plat_thread_priority(PLAT_PRIO_IDLE);
|
|
for (;;) {
|
|
size_t i = (size_t)plat_inc64(&LB.next) - 1;
|
|
if (i >= LB.n || LB.stop) break;
|
|
if (skip_listed(LB.e[i].iid)) {
|
|
skip_count();
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.done++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
continue;
|
|
}
|
|
// A compute shader whose pipeline does not stay is read as a copy of this thread's own and released after its
|
|
// build: kept in the shader cache's memory, the 3,900 of an installed copy's list were 240 MB of the process's
|
|
// commit for shaders most runs never dispatch. A dispatch that does come reads its shader then (0.05 to
|
|
// 0.15 ms from the files this pass has just read). The shaders of kept pipelines and libraries stay loaded:
|
|
// their pipeline and library entries point at them.
|
|
int keep = i < LB.n_keep, owned = 0;
|
|
plat_thread_priority(PLAT_PRIO_NORMAL); // gfx_shader's cache lock, disk reads
|
|
const GfxShader *sh = keep || LB.e[i].stage != GFX_STAGE_CS ? gfx_shader_load_input(LB.e[i].iid, 0)
|
|
: gfx_shader_load_private(LB.e[i].iid, &owned);
|
|
plat_thread_priority(PLAT_PRIO_IDLE);
|
|
if (!sh) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.noshader++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
} else if (gfx_shader_stage(sh) == GFX_STAGE_CS) {
|
|
lb_compute(sh, LB.e[i].iid, keep);
|
|
} else if (!owned && (gfx_shader_stage(sh) == GFX_STAGE_VS || gfx_shader_stage(sh) == GFX_STAGE_FS)) {
|
|
if (P.gpl) lb_library(sh, gfx_shader_stage(sh) == GFX_STAGE_VS, 1); // a kept library points at sh: never a copy
|
|
else lb_skip_lib();
|
|
}
|
|
if (owned) gfx_shader_release_private(sh);
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.done++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
}
|
|
if (!plat_dec32(&LB.running)) {
|
|
lb_report(LB.stop ? "stopped" : "done");
|
|
fb_log("after the stage library prebuild");
|
|
vk_mem_stage_log("the compute pipeline prebuild");
|
|
bg_start();
|
|
}
|
|
return;
|
|
}
|
|
|
|
// The library state from the key file's records (prewarm); defaults without one.
|
|
static void lb_state(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) {
|
|
// state index: clamp | poly (0 fill, 1 line, 2 point -> VkPolygonMode) << 1 | minus_w << 3 (pr); samples log2 (fs)
|
|
uint32_t best = 0, bf = 0;
|
|
for (uint32_t i = 1; i < 16; i++) best = pr_hist[i] > pr_hist[best] ? i : best;
|
|
for (uint32_t i = 1; i < 4; i++) bf = fs_hist[i] > fs_hist[bf] ? i : bf;
|
|
if (!pr_hist[best]) best = 8; // no key file: NX2navi clips -w..w (all 2706 records of a recorded key file)
|
|
LB.clamp = (uint8_t)(best & 1), LB.poly = (uint8_t)(best >> 1 & 3), LB.minus_w = (uint8_t)(best >> 3 & 1);
|
|
LB.samples = (uint8_t)(1u << bf);
|
|
}
|
|
|
|
// ---- the driver's cache of stage libraries, warmed again during play
|
|
//
|
|
// The preparation (lb_warm_prepared) builds every prepared variant's stage library once, so the driver's disk cache
|
|
// holds it and the library a draw needs later takes a few milliseconds instead of a compile: 3 ms against 100 ms for
|
|
// a vertex and 340 ms for a fragment library on an RX 7900 XT (driver 26.10.1). The driver's cache does not outlive
|
|
// everything, though: a driver update empties it, the AMD driver keeps a limited amount and drops old entries, and a
|
|
// change to what a library is built with (the dynamic states it lists, as when dynamic vertex input was turned off for
|
|
// this driver) gives every library another entry. After any of them each pipeline a draw first needs compiles cold:
|
|
// the draw is skipped for the frames that takes, or waits for it. Measured on that card with the cache cold, a walk
|
|
// through a village not seen before skipped 11,180 draws while their pipelines built and 94 draws waited 5.5 s in
|
|
// all, one of them 1.9 s.
|
|
//
|
|
// So every start checks: a sample of the prepared fragment libraries is built (lw_probe), and when their median time
|
|
// says the cache is cold, the whole prepared list is built again into the driver's cache on a few idle-priority
|
|
// threads while the game runs (lw_worker), as the preparation does but without stopping play. shader-warm.txt beside
|
|
// the caches says where that stands: "warming" while it runs, so a start after a run that ended meanwhile goes on,
|
|
// and "done" with the driver and a count of the times the cache was cold again right after a finished warm. At two in
|
|
// a row the driver does not keep stage libraries at all and warming stops being tried (NX2NAVI_LIB_WARM=0 turns it off;
|
|
// =force warms at every start).
|
|
#define LW_SAMPLE 12
|
|
#define LW_COLD_MS 15.0
|
|
#define LW_MAX_THREADS 6
|
|
static struct {
|
|
GfxWarmEnt *e;
|
|
size_t n;
|
|
volatile int64_t next, done;
|
|
volatile int32_t running, stop, started;
|
|
PlatThread *th[LW_MAX_THREADS + 1];
|
|
int threads, strikes;
|
|
uint32_t vendor; // the GPU's vendor id, for what a cold cache's log line adds (vk_warm_hint.h)
|
|
double t0;
|
|
char stamp[PLAT_PATH_MAX * 2 + 32], id[64];
|
|
} LW;
|
|
|
|
static void lw_stamp(const char *state, int strikes) {
|
|
char line[128];
|
|
int n = snprintf(line, sizeof line, "%s %s %d\n", state, LW.id, strikes);
|
|
if (!plat_file_write_atomic(LW.stamp, line, (size_t)n)) rt_log("vk_pipeline: cannot write %s", LW.stamp);
|
|
}
|
|
|
|
// One prepared vertex or fragment variant's library into the driver's cache; its time in ms, negative when the
|
|
// variant is on the skip list, not in the shader cache or of another stage.
|
|
static double lw_build(const GfxWarmEnt *e) {
|
|
if (e->stage != GFX_STAGE_VS && e->stage != GFX_STAGE_FS) return -1;
|
|
if (skip_listed(e->iid)) return -1;
|
|
int owned = 0;
|
|
const GfxShader *sh = gfx_shader_load_private(e->iid, &owned);
|
|
if (!sh) return -1;
|
|
double t0 = now_ms();
|
|
int st = gfx_shader_stage(sh);
|
|
if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) lb_library(sh, st == GFX_STAGE_VS, 0);
|
|
double ms = now_ms() - t0;
|
|
if (owned) gfx_shader_release_private(sh);
|
|
return ms;
|
|
}
|
|
|
|
static void lw_worker(void *arg) {
|
|
(void)arg;
|
|
plat_thread_priority(PLAT_PRIO_IDLE);
|
|
for (;;) {
|
|
size_t i = (size_t)plat_inc64(&LW.next) - 1;
|
|
if (i >= LW.n || LW.stop) break;
|
|
lw_build(&LW.e[i]);
|
|
int64_t d = plat_inc64(&LW.done);
|
|
if (d % 8000 == 0)
|
|
rt_log("vk_pipeline: library warm: %lld of %zu prepared variants in %.0f s", (long long)d, LW.n,
|
|
(now_ms() - LW.t0) / 1000);
|
|
}
|
|
if (!plat_dec32(&LW.running) && !LW.stop) {
|
|
lw_stamp("done", LW.strikes);
|
|
rt_log("vk_pipeline: library warm: done, %zu prepared variants in %.0f s on %d idle threads; new pipelines "
|
|
"link from the driver's cache from here on", LW.n, (now_ms() - LW.t0) / 1000, LW.threads);
|
|
}
|
|
}
|
|
|
|
// The median time of LW_SAMPLE prepared fragment libraries spread over the list; negative without that many.
|
|
static double lw_probe(void) {
|
|
double ms[LW_SAMPLE];
|
|
int got = 0;
|
|
size_t step = LW.n / LW_SAMPLE ? LW.n / LW_SAMPLE : 1, at = (size_t)(plat_now_ns() / 1000) % step;
|
|
for (size_t i = at; i < LW.n && got < LW_SAMPLE && !LW.stop; i += step)
|
|
for (size_t j = i; j < i + step && j < LW.n; j++) { // the next fragment variant from here
|
|
if (LW.e[j].stage != GFX_STAGE_FS) continue;
|
|
double t = lw_build(&LW.e[j]);
|
|
if (t >= 0) ms[got++] = t;
|
|
break;
|
|
}
|
|
if (got < LW_SAMPLE / 2) return -1;
|
|
for (int i = 1; i < got; i++)
|
|
for (int j = i; j > 0 && ms[j] < ms[j - 1]; j--) {
|
|
double t = ms[j];
|
|
ms[j] = ms[j - 1], ms[j - 1] = t;
|
|
}
|
|
return ms[got / 2];
|
|
}
|
|
|
|
static void lw_main(void *arg) {
|
|
(void)arg;
|
|
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
char was[16] = "", id[64] = "";
|
|
int strikes = 0;
|
|
uint64_t size = 0;
|
|
char *f = (char *)plat_file_read(LW.stamp, &size);
|
|
if (f && size < 200) {
|
|
char line[200];
|
|
memcpy(line, f, size), line[size] = 0;
|
|
if (sscanf(line, "%15s %63s %d", was, id, &strikes) < 2) was[0] = 0;
|
|
}
|
|
free(f);
|
|
int same = !strcmp(id, LW.id), resume = same && !strcmp(was, "warming");
|
|
const char *mode = cfg_value("NX2NAVI_LIB_WARM");
|
|
double median = resume ? 0 : lw_probe();
|
|
if (LW.stop) return;
|
|
int cold = median > LW_COLD_MS || !strcmp(mode, "force");
|
|
if (!resume && !cold) {
|
|
if (same && strikes) lw_stamp("done", 0); // the driver kept them this time
|
|
rt_log("vk_pipeline: library warm: the driver's cache has the prepared libraries (a sample's median %.1f ms)",
|
|
median);
|
|
return;
|
|
}
|
|
if (!resume && same && strikes >= 2 && strcmp(mode, "force")) {
|
|
rt_log("vk_pipeline: library warm: the prepared libraries are cold again (a sample's median %.0f ms) after two "
|
|
"finished warms on this driver: it does not keep them, so pipelines compile when a draw first needs "
|
|
"them (NX2NAVI_LIB_WARM=force warms anyway).%s", median,
|
|
lw_cold_hint(LW.vendor, LW_COLD_GAVE_UP, LW_ON_WINDOWS));
|
|
return;
|
|
}
|
|
LW.strikes = resume ? strikes : same && !strcmp(was, "done") ? strikes + 1 : 0;
|
|
lw_stamp("warming", LW.strikes);
|
|
int cpus = (int)plat_cpu_usable(), t = cpus / 4;
|
|
t = t < 2 ? 2 : t > LW_MAX_THREADS ? LW_MAX_THREADS : t;
|
|
if (resume)
|
|
rt_log("vk_pipeline: library warm: going on with the warm an earlier run began: %zu prepared variants on %d "
|
|
"idle threads", LW.n, t);
|
|
else // cold right after a finished warm on this driver says more than a first cold (vk_warm_hint.h)
|
|
rt_log("vk_pipeline: library warm: the driver's cache does not have the prepared libraries (a sample's median "
|
|
"%.0f ms; a driver update or a change of what they are built with): building %zu prepared variants "
|
|
"into it on %d idle threads while the game runs. Until it is done, a pipeline a draw first needs may "
|
|
"compile, which skips that draw for some frames or makes it wait.%s", median, LW.n, t,
|
|
lw_cold_hint(LW.vendor,
|
|
median <= LW_COLD_MS ? LW_WARM_FORCED // warm, but lib_warm = force
|
|
: same && !strcmp(was, "done") ? LW_COLD_AFTER_WARM : LW_COLD_FIRST,
|
|
LW_ON_WINDOWS));
|
|
LW.t0 = now_ms();
|
|
LW.running = t;
|
|
int started = 0;
|
|
for (int i = 0; i < t; i++)
|
|
if ((LW.th[1 + started] = plat_thread_new(lw_worker, NULL, "lib warm"))) started++;
|
|
LW.threads = started;
|
|
if (plat_add32(&LW.running, started - t) == 0 && !started) rt_log("vk_pipeline: library warm: no thread started");
|
|
}
|
|
|
|
// Boot, once the library state is known (lb_state): the prepared list is read and the check starts on its own thread.
|
|
static void lw_start(void) {
|
|
const char *mode = cfg_value("NX2NAVI_LIB_WARM");
|
|
if (!P.gpl || !strcmp(mode, "0") || plat_xchg32(&LW.started, 1)) return;
|
|
char dir[PLAT_PATH_MAX * 2], path[PLAT_PATH_MAX * 2 + 32];
|
|
if (!cache_dir(dir, sizeof dir)) return;
|
|
snprintf(path, sizeof path, "%s/shader-prepared.bin", dir);
|
|
snprintf(LW.stamp, sizeof LW.stamp, "%s/shader-warm.txt", dir);
|
|
uint64_t size = 0;
|
|
uint8_t *f = plat_file_read(path, &size);
|
|
size_t n = f && size >= 16 ? (size_t)(size - 16) / 36 : 0;
|
|
LW.e = n ? calloc(n, sizeof *LW.e) : NULL;
|
|
for (size_t i = 0; LW.e && i < n; i++)
|
|
memcpy(LW.e[i].iid, f + 16 + 36 * i + 4, 32), LW.e[i].stage = f[16 + 36 * i], LW.e[i].prio = f[16 + 36 * i + 1];
|
|
free(f);
|
|
if (!LW.e) return; // nothing prepared (a build without the preparation step)
|
|
LW.n = n;
|
|
VkPhysicalDeviceProperties pp;
|
|
vkGetPhysicalDeviceProperties(vk_ctx()->phys, &pp);
|
|
snprintf(LW.id, sizeof LW.id, "%04x:%04x:%08x", pp.vendorID, pp.deviceID, pp.driverVersion);
|
|
LW.vendor = pp.vendorID;
|
|
LW.th[0] = plat_thread_new(lw_main, NULL, "lib warm check");
|
|
}
|
|
|
|
static void lw_stop(void) {
|
|
plat_xchg32(&LW.stop, 1);
|
|
for (int i = 0; i <= LW_MAX_THREADS; i++)
|
|
if (LW.th[i]) plat_thread_join(LW.th[i]), LW.th[i] = NULL;
|
|
}
|
|
|
|
// Boot: compute shaders of the warm list (recorded and prepared), those whose pipeline stays first, then the first
|
|
// KEEP recorded graphics variants.
|
|
static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) {
|
|
const char *e = cfg_value("NX2NAVI_LIB_PREBUILD"), *ke = cfg_get("NX2NAVI_LIB_PREBUILD_KEEP");
|
|
const char *te = cfg_get("NX2NAVI_LIB_PREBUILD_THREADS"); // no fixed default: 2 when unset (1 under 4 CPUs)
|
|
const char *be = cfg_get("NX2NAVI_LIB_PREBUILD_BIG"); // SPIR-V words; LB_BIG_WORDS when unset, 0 no limit
|
|
if (!A.on || !cfg_bool("NX2NAVI_LIB_PREBUILD")) return;
|
|
lb_state(pr_hist, fs_hist); // the libraries' state, also for lw_start
|
|
lw_start();
|
|
LB.big_words = be && *be ? strtoull(be, NULL, 10) : LB_BIG_WORDS;
|
|
char dir[PLAT_PATH_MAX * 2];
|
|
if (cache_dir(dir, sizeof dir)) skip_init(dir);
|
|
GfxWarmEnt *all = NULL;
|
|
size_t total = gfx_shader_warm_list(&all), n = 0, n_cs = 0;
|
|
LB.keep_n = ke && *ke ? atoi(ke) : 0;
|
|
LB.e = malloc((total + 1) * sizeof *LB.e);
|
|
// the compute shaders whose pipeline stays come first: they are the ones the game is about to dispatch
|
|
uint8_t *kf = total ? malloc(total) : NULL;
|
|
for (size_t i = 0; LB.e && kf && i < total; i++)
|
|
if (all[i].stage == GFX_STAGE_CS && (kf[i] = (uint8_t)cu_keep(all[i].iid))) LB.e[n++] = all[i];
|
|
LB.n_keep = n;
|
|
for (size_t i = 0; LB.e && i < total; i++)
|
|
if (all[i].stage == GFX_STAGE_CS && !(kf && kf[i])) LB.e[n++] = all[i];
|
|
free(kf);
|
|
n_cs = n;
|
|
for (size_t i = 0; LB.e && i < total && (int32_t)(n - n_cs) < LB.keep_n; i++)
|
|
if (all[i].stage != GFX_STAGE_CS && all[i].prio == 0) LB.e[n++] = all[i];
|
|
free(all);
|
|
if (atoi(e) > 1 && (size_t)atoi(e) < n) n = (size_t)atoi(e); // a count above 1 caps the list; 1 is plain on
|
|
if (LB.n_keep > n) LB.n_keep = n;
|
|
if (!n) {
|
|
free(LB.e), LB.e = NULL;
|
|
return;
|
|
}
|
|
LB.n = n;
|
|
LB.idle = 1;
|
|
int cpus = (int)plat_cpu_usable();
|
|
// Two threads by default, not half the CPUs. On a 32-thread CPU with NVIDIA's driver, 16 builders kept the driver's
|
|
// lock busy: 77% of the process's running time was a spin loop in nvoglv64.dll (about 8 cores), the prebuild had not
|
|
// finished after 34,000 frames, and lib_prebuild = 0 took it to 5% and the game to its frame limit (RTX 4080 SUPER,
|
|
// driver 617.14, 2026-11-08). With the driver's disk cache warm (--prepare-shaders, --warm-shaders) a build takes
|
|
// well under a millisecond, so two threads still finish the list quickly.
|
|
int t = te && *te ? atoi(te) : (cpus < 4 ? 1 : 2); // idle priority; a cold driver takes ~2 s per large compute shader
|
|
t = t < 1 ? 1 : t > LB_MAX_THREADS ? LB_MAX_THREADS : t;
|
|
if ((size_t)t > n) t = (int)n;
|
|
LB.t0 = now_ms();
|
|
LB.running = t;
|
|
int started = 0;
|
|
for (int i = 0; i < t; i++)
|
|
if ((LB.th[started] = plat_thread_new(lb_worker, (void *)(intptr_t)started, "lib prebuild"))) started++;
|
|
plat_add32(&LB.running, started - t);
|
|
LB.threads = started;
|
|
char big[128];
|
|
if (!LB.big_words) snprintf(big, sizeof big, "compute shaders of every size");
|
|
else snprintf(big, sizeof big, "compute shaders over %llu SPIR-V words %s", (unsigned long long)LB.big_words,
|
|
P.cache_control ? "only from the pipeline caches" : "left to their first dispatch");
|
|
rt_log("vk_pipeline: stage library prebuild: %zu compute pipelines, of which %zu stay (the shaders dispatched most "
|
|
"recently, NX2NAVI_COMPUTE_KEEP) and the others are built into the pipeline cache and destroyed, to be created "
|
|
"at their first dispatch; %zu recorded variants' libraries (NX2NAVI_LIB_PREBUILD_KEEP %d); on %d idle-priority "
|
|
"threads; library state clamp %u, polygon mode %u, depth -w..w %u, samples %u; %s (NX2NAVI_LIB_PREBUILD_BIG)",
|
|
n_cs < n ? n_cs : n, LB.n_keep, n > n_cs ? n - n_cs : 0, LB.keep_n, started, LB.clamp, LB.poly, LB.minus_w,
|
|
LB.samples, big);
|
|
}
|
|
|
|
static void lb_stop(void) {
|
|
lw_stop();
|
|
plat_xchg32(&LB.stop, 1);
|
|
for (int i = 0; i < LB.threads; i++)
|
|
if (LB.th[i]) plat_thread_join(LB.th[i]), LB.th[i] = NULL;
|
|
LB.threads = 0;
|
|
if (BG.th) plat_thread_join(BG.th), BG.th = NULL; // after its current build, if one runs
|
|
}
|
|
|
|
// Preparation: one prepared variant at a time into the driver's disk cache (private shader copy, nothing kept). arg is
|
|
// the worker's journal slot.
|
|
static void lb_warm_worker(void *arg) {
|
|
int slot = (int)(intptr_t)arg;
|
|
for (;;) {
|
|
size_t i = (size_t)plat_inc64(&LB.next) - 1;
|
|
if (i >= LB.n) return;
|
|
int owned = 0, skip = skip_listed(LB.e[i].iid);
|
|
const GfxShader *sh = skip ? NULL : gfx_shader_load_private(LB.e[i].iid, &owned);
|
|
int st = sh ? gfx_shader_stage(sh) : -1;
|
|
if (sh) journal_mark(slot, &LB.e[i]); // from here to its clearing below, a crash names this variant
|
|
if (skip) {
|
|
skip_count();
|
|
} else if (!sh) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.noshader++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
} else if (st == GFX_STAGE_CS) {
|
|
Bindings B = {0};
|
|
const char *why = NULL;
|
|
lb_lock();
|
|
Layout *l = compute_layout(sh, &B, &why);
|
|
lb_unlock();
|
|
double t0 = now_ms();
|
|
VkPipeline pipe = l ? compute_create(sh, l, 0) : VK_NULL_HANDLE;
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
lb_count(2, pipe ? 3 : 2, now_ms() - t0);
|
|
} else if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) {
|
|
if (P.gpl) lb_library(sh, st == GFX_STAGE_VS, 0);
|
|
else lb_skip_lib();
|
|
}
|
|
if (sh) journal_mark(slot, NULL);
|
|
if (owned) gfx_shader_release_private(sh);
|
|
plat_mutex_lock(&LB.stat);
|
|
uint64_t d = ++LB.done;
|
|
plat_mutex_unlock(&LB.stat);
|
|
if (d % 4000 == 0) lb_report("preparation progress");
|
|
}
|
|
}
|
|
|
|
// Preparation: every variant of <dir>/shader-prepared.bin on every core at normal priority.
|
|
static void lb_warm_prepared(const char *dir) {
|
|
char path[PLAT_PATH_MAX * 2 + 32];
|
|
snprintf(path, sizeof path, "%s/shader-prepared.bin", dir);
|
|
uint64_t size = 0;
|
|
uint8_t *f = plat_file_read(path, &size);
|
|
size_t n = f && size >= 16 ? (size_t)(size - 16) / 36 : 0;
|
|
LB.e = n ? calloc(n, sizeof *LB.e) : NULL;
|
|
for (size_t i = 0; LB.e && i < n; i++)
|
|
memcpy(LB.e[i].iid, f + 16 + 36 * i + 4, 32), LB.e[i].stage = f[16 + 36 * i], LB.e[i].prio = f[16 + 36 * i + 1];
|
|
free(f);
|
|
if (!LB.e) return;
|
|
skip_init(dir);
|
|
char jp[PLAT_PATH_MAX * 2 + 48];
|
|
snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir);
|
|
SK.journal = plat_fs_create(jp, NULL); // skip_init removed a leftover one
|
|
if (!SK.journal) rt_log("vk_pipeline: cannot create %s: a driver crash in this run will not be recorded", jp);
|
|
LB.n = n, LB.idle = 0, LB.t0 = now_ms();
|
|
static const uint32_t no_pr[16], no_fs[4];
|
|
lb_state(no_pr, no_fs);
|
|
int cpus = (int)plat_cpu_usable(), k = 0;
|
|
PlatThread *t[JR_SLOTS];
|
|
for (int i = 0; i < cpus && i < JR_SLOTS; i++)
|
|
if ((t[k] = plat_thread_new(lb_warm_worker, (void *)(intptr_t)k, "lib warm"))) k++;
|
|
if (!k) lb_warm_worker((void *)(intptr_t)0);
|
|
// every 2 s a count on stdout for tools/make_game.py (step_shaders), as gfx_shader.c prepare_files prints for the
|
|
// translation: a driver building for a long time still shows the step moving
|
|
for (int i = 0; i < k; i++)
|
|
while (!plat_thread_join_ns(t[i], 2000000000u)) {
|
|
plat_mutex_lock(&LB.stat);
|
|
uint64_t d = LB.done;
|
|
plat_mutex_unlock(&LB.stat);
|
|
printf("prepare-shaders: progress build %llu %zu\n", (unsigned long long)d, n);
|
|
fflush(stdout);
|
|
}
|
|
printf("prepare-shaders: progress build %zu %zu\n", n, n);
|
|
fflush(stdout);
|
|
if (SK.journal) plat_file_close(SK.journal), SK.journal = NULL, plat_file_delete(jp); // every build returned
|
|
lb_report("preparation done");
|
|
// with gpl off only compute variants are built: say so, rather than count the skipped ones as built
|
|
unsigned long long skipped = LB.skipped;
|
|
size_t tried = n - (size_t)skipped;
|
|
if (LB.nolib)
|
|
printf("prepare-shaders: %zu variants on %d threads in %.0f ms, %llu vertex/fragment variants skipped (graphics "
|
|
"pipeline library off, they build at their first draw)\n", tried, k, now_ms() - LB.t0,
|
|
(unsigned long long)LB.nolib);
|
|
else
|
|
printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", tried, k,
|
|
now_ms() - LB.t0);
|
|
if (skipped) // installer/steps.py Prepare reads this line
|
|
printf("prepare-shaders: %llu variants skipped: the graphics driver crashed while building them in an earlier "
|
|
"run (shader-prepare-skip.txt); they build at their first draw\n", skipped);
|
|
free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0, LB.skipped = 0;
|
|
}
|
|
|
|
// nx2navi.exe --prepare-shaders (tools/prepare_shaders.py): the device sets the translator's host answers (they are in every
|
|
// cache key), the seed next to nx2navi.exe is translated on every core, then every prepared variant's stage library or
|
|
// compute pipeline is built once into the driver's disk cache (lb_warm_prepared). No game runs.
|
|
int vk_pipeline_prepare_shaders(void) {
|
|
if (!ensure_init()) {
|
|
fprintf(stderr, "prepare-shaders: no Vulkan device\n");
|
|
return 1;
|
|
}
|
|
char dir[PLAT_PATH_MAX * 2];
|
|
if (!cache_dir(dir, sizeof dir)) return 1;
|
|
int cpus = (int)plat_cpu_usable();
|
|
const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset
|
|
int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2;
|
|
int rc = gfx_shader_prepare_seed(dir, cpus, procs < 1 ? 1 : procs);
|
|
if (!rc) lb_warm_prepared(dir);
|
|
return rc;
|
|
}
|
|
|
|
// nx2navi.exe --warm-shaders: every variant of shader-prepared.bin built into the driver's disk cache again from the shader
|
|
// cache, without translating the seed. The AMD cache is keyed to the exe path (a relink keeps it: 29,587 variants in
|
|
// 1.8 s) and holds a limited amount: preparing another build dir evicted this one's libraries, and a library a draw
|
|
// needed then took 40-220 ms in play instead of 0.2 ms. Seconds when the libraries are still there.
|
|
int vk_pipeline_warm_shaders(void) {
|
|
if (!ensure_init()) {
|
|
fprintf(stderr, "warm-shaders: no Vulkan device\n");
|
|
return 1;
|
|
}
|
|
char dir[PLAT_PATH_MAX * 2];
|
|
if (!cache_dir(dir, sizeof dir)) return 1;
|
|
char prp[PLAT_PATH_MAX * 2 + 32];
|
|
snprintf(prp, sizeof prp, "%s/shader-prepared.bin", dir);
|
|
if (!plat_file_exists(prp)) {
|
|
fprintf(stderr, "warm-shaders: no %s (tools/prepare_shaders.py writes it)\n", prp);
|
|
return 1;
|
|
}
|
|
lb_warm_prepared(dir);
|
|
return 0;
|
|
}
|
|
|
|
// nx2navi.exe --probe-prewarm: the boot's shader and pipeline stages without the game, from the caches next to
|
|
// nx2navi.exe (pipeline-cache.bin, pipeline-keys.bin, shader-answers.bin, the shader cache): the pipeline cache load,
|
|
// the shader prewarm, the recorded pipelines and the compute pipeline prebuild, each followed by vk_mem_stage_log's
|
|
// line, so the log tells what every stage leaves in device memory and in the process's commit. Then up to
|
|
// PROBE_DISPATCHES compute shaders whose pipeline the boot did not keep are asked for as a dispatch would
|
|
// (gfx_shader_load_input for the shader, then vk_pipeline_compute), which gives what a first dispatch costs on this
|
|
// machine and how many of them the pipeline cache held. Fails when one of those pipelines is not created, when the
|
|
// prebuild kept a pipeline it should not have, when a large compute shader went to the background builder although no
|
|
// run dispatched it, or when the record of dispatched shaders, written to a file of the probe's own and read back,
|
|
// does not hold the probe's dispatches. compute-used.bin itself stays as it is; the pipeline cache is saved as after
|
|
// a boot.
|
|
#define PROBE_DISPATCHES 256
|
|
int vk_pipeline_probe_prewarm(void) {
|
|
vk_pipeline_async_enable();
|
|
CU.quiet = 1;
|
|
vk_pipeline_prewarm(); // brings the device up itself: it has to own the key file before ensure_init runs
|
|
if (!P.ready) {
|
|
printf("probe-prewarm: SKIP (the boot prewarm brought no Vulkan device up: none on this machine, NX2NAVI_GPU=null "
|
|
"or NX2NAVI_PIPE_SYNC=1)\n");
|
|
return 0;
|
|
}
|
|
while (__atomic_load_n(&LB.running, __ATOMIC_ACQUIRE) > 0) plat_sleep_ms(50);
|
|
if (!LB.n) {
|
|
printf("probe-prewarm: SKIP (no recorded compute shaders next to the executable: %s)\n", P.cache_path);
|
|
return 0;
|
|
}
|
|
VkPipeStats st;
|
|
vk_pipeline_stats(&st);
|
|
plat_mutex_lock_shared(&LB.stat);
|
|
uint64_t kept = LB.built[2], cache_only = LB.warmed[2], failed = LB.fail[2], there = LB.have[2];
|
|
uint64_t big_unused = LB.big_unused;
|
|
size_t queued = BG.n;
|
|
plat_mutex_unlock_shared(&LB.stat);
|
|
printf("probe-prewarm: %zu compute shaders, %zu to keep: %llu kept, %llu built into the pipeline cache only, %llu "
|
|
"there already, %llu failed, %llu large ones not built (no run dispatched them), %zu large ones for the "
|
|
"background builder; %llu graphics pipelines\n", LB.n, LB.n_keep, (unsigned long long)kept,
|
|
(unsigned long long)cache_only, (unsigned long long)there, (unsigned long long)failed,
|
|
(unsigned long long)big_unused, queued, (unsigned long long)st.graphics);
|
|
int fails = kept > LB.n_keep; // a dispatch never waits in this mode, so nothing else may have been kept
|
|
if (fails) printf("probe-prewarm: FAIL: %llu pipelines kept, %zu expected at most\n", (unsigned long long)kept, LB.n_keep);
|
|
if (queued > LB.n_keep) // only a shader a run dispatched may be built ahead when it is large
|
|
fails++, printf("probe-prewarm: FAIL: %zu large compute shaders queued for the background builder, %zu kept "
|
|
"shaders at most could be among them\n", queued, LB.n_keep);
|
|
uint32_t asked = 0, got = 0;
|
|
double t_all = 0, t_max = 0, t_load = 0;
|
|
for (size_t i = LB.n_keep; i < LB.n && asked < PROBE_DISPATCHES; i++) {
|
|
if (LB.e[i].stage != GFX_STAGE_CS) continue;
|
|
double l0 = now_ms();
|
|
const GfxShader *sh = gfx_shader_load_input(LB.e[i].iid, 0); // from the disk: the boot released its copy
|
|
double load = now_ms() - l0;
|
|
uint64_t words = 0;
|
|
if (!sh || gfx_shader_stage(sh) != GFX_STAGE_CS) continue;
|
|
gfx_shader_spirv(sh, &words);
|
|
if (LB.big_words && words > LB.big_words) continue; // a first dispatch of one compiles it: not what is timed here
|
|
const VkPipeInfo *info = NULL;
|
|
double t0 = now_ms();
|
|
VkPipeline p = vk_pipeline_compute(sh, &info);
|
|
double ms = now_ms() - t0;
|
|
asked++, got += p != VK_NULL_HANDLE;
|
|
t_all += ms, t_load += load;
|
|
if (ms > t_max) t_max = ms;
|
|
}
|
|
vk_mem_stage_log("the probe's first dispatches");
|
|
plat_mutex_lock_shared(&P.lock);
|
|
uint64_t hit = CF.hit;
|
|
plat_mutex_unlock_shared(&P.lock);
|
|
printf("probe-prewarm: %u first dispatches of pipelines the boot did not keep: %u created, avg %.2f ms, longest %.2f "
|
|
"ms, the pipeline cache held %llu of them; reading their shaders took avg %.2f ms\n", asked, got,
|
|
asked ? t_all / asked : 0.0, t_max, (unsigned long long)hit, asked ? t_load / asked : 0.0);
|
|
if (got != asked) fails++, printf("probe-prewarm: FAIL: %u pipelines were not created\n", asked - got);
|
|
if (CU.opened) {
|
|
char rp[sizeof CU.path + 8];
|
|
snprintf(rp, sizeof rp, "%s.probe", CU.path);
|
|
uint64_t rsize = 0;
|
|
CuHeader rh = {0};
|
|
uint32_t mine = 0;
|
|
int wrote = cu_write(rp);
|
|
uint8_t *rf = wrote ? plat_file_read(rp, &rsize) : NULL;
|
|
long rn = cu_parse(rf, rsize, &rh);
|
|
for (long i = 0; i < rn; i++) {
|
|
CuRec r;
|
|
memcpy(&r, rf + sizeof rh + (size_t)i * sizeof r, sizeof r);
|
|
mine += r.run == CU.run;
|
|
}
|
|
free(rf);
|
|
plat_file_delete(rp);
|
|
printf("probe-prewarm: the record of dispatched compute shaders: %ld entries, %u of this run (run %u)\n", rn, mine,
|
|
CU.run);
|
|
if (!wrote || rn < 0 || rh.run != CU.run || mine != got)
|
|
fails++, printf("probe-prewarm: FAIL: the record does not hold the %u shaders dispatched here\n", got);
|
|
}
|
|
lb_stop();
|
|
vk_pipeline_save();
|
|
printf("probe-prewarm: %s\n", fails ? "FAILED" : "passed");
|
|
return fails ? 1 : 0;
|
|
}
|
|
|
|
// ---- dynamic state
|
|
|
|
void vk_pipeline_set_dynamic(VkCommandBuffer cb, const VkPipeState *s) {
|
|
VkDynShadow sh;
|
|
uint64_t skipped = 0;
|
|
sh.have = 0; // nothing known: every call is recorded
|
|
vk_pipeline_set_dynamic_delta(cb, s, &sh, 0, &skipped);
|
|
}
|
|
|
|
// Every graphics pipeline the replay binds lists the same dynamic states (P.dyn; mesh pipelines leave out topology,
|
|
// primitive restart and vertex input), so a value recorded into a command buffer holds across pipeline binds until it
|
|
// is set again. Each call is recorded only when its value differs from the one *sh says the command buffer has: a new
|
|
// pipeline usually changes the shaders and a field or two, not the twenty states this used to record with every bind.
|
|
int vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s, VkDynShadow *sh, int mesh, uint64_t *skipped) {
|
|
if (!ensure_init()) return 0;
|
|
const poly_st *pl = &s->poly;
|
|
const ds_st *d = &s->ds;
|
|
int core = (sh->have & VK_DYN_CORE) != 0, vtx = !mesh && (sh->have & VK_DYN_VTX), blend = (sh->have & VK_DYN_BLEND) != 0;
|
|
int n = 0, left = 0;
|
|
#define DYN(known, field, val, call) \
|
|
do { \
|
|
if ((known) && sh->field == (val)) left++; \
|
|
else sh->field = (val), call, n++; \
|
|
} while (0)
|
|
// The front face maps unchanged: NX2NAVI sets the upper-left window origin, and the y-flipped viewport (y_flip) puts
|
|
// NDC +y at row 0 as NVN does, so the framebuffer winding Vulkan tests is the one NVN tests
|
|
// (a lower-left origin would need it mirrored).
|
|
int front_cw = pl->front == NVN_FRONT_FACE_CW;
|
|
VkCullModeFlags cull = (VkCullModeFlags)(pl->cull & 3); // NVN_FACE_* has VkCullModeFlagBits' values
|
|
VkFrontFace front = front_cw ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE;
|
|
DYN(core, cull, cull, vkCmdSetCullMode(cb, cull));
|
|
DYN(core, front, front, vkCmdSetFrontFace(cb, front));
|
|
if (!mesh) {
|
|
VkPrimitiveTopology topo = vk_pipeline_topology(s->prim);
|
|
DYN(vtx, topo, topo, vkCmdSetPrimitiveTopology(cb, topo));
|
|
}
|
|
DYN(core, dt, (VkBool32)d->dt, vkCmdSetDepthTestEnable(cb, d->dt));
|
|
DYN(core, dw, (VkBool32)d->dw, vkCmdSetDepthWriteEnable(cb, d->dw));
|
|
DYN(core, dop, compare_op(d->df), vkCmdSetDepthCompareOp(cb, compare_op(d->df)));
|
|
DYN(core, st, (VkBool32)d->st, vkCmdSetStencilTestEnable(cb, d->st));
|
|
for (int f = 0; f < 2; f++) {
|
|
VkStencilOp o[3] = {stencil_op(f ? d->fail_b : d->fail_f), stencil_op(f ? d->zpass_b : d->zpass_f),
|
|
stencil_op(f ? d->zfail_b : d->zfail_f)};
|
|
VkCompareOp c = compare_op(f ? d->sf_b : d->sf_f);
|
|
if (core && sh->scmp[f] == c && !memcmp(sh->sop[f], o, sizeof o)) {
|
|
left++;
|
|
continue;
|
|
}
|
|
vkCmdSetStencilOp(cb, f ? VK_STENCIL_FACE_BACK_BIT : VK_STENCIL_FACE_FRONT_BIT, o[0], o[1], o[2], c);
|
|
sh->scmp[f] = c, memcpy(sh->sop[f], o, sizeof o), n++;
|
|
}
|
|
// NVN_POLYGON_OFFSET_ENABLE_* bit of the polygon mode in use: point 1, line 2, fill 4
|
|
unsigned mode = pl->mode <= NVN_POLYGON_MODE_FILL ? pl->mode : NVN_POLYGON_MODE_FILL;
|
|
VkBool32 bias_en = (pl->offset_en >> mode & 1) != 0, discard = s->raster_discard != 0, restart = s->prim_restart != 0;
|
|
DYN(core, bias_en, bias_en, vkCmdSetDepthBiasEnable(cb, bias_en));
|
|
DYN(core, discard, discard, vkCmdSetRasterizerDiscardEnable(cb, discard));
|
|
if (!mesh) DYN(vtx, restart, restart, vkCmdSetPrimitiveRestartEnable(cb, restart));
|
|
if (!mesh && P.dyn_vi && s->vs) {
|
|
VtxInput v;
|
|
vertex_input(s, &v);
|
|
if (vtx && sh->vi.n_attr == v.n_attr && sh->vi.n_bind == v.n_bind &&
|
|
!memcmp(sh->vi.a, v.a, sizeof v.a[0] * v.n_attr) && !memcmp(sh->vi.b, v.b, sizeof v.b[0] * v.n_bind)) {
|
|
left++;
|
|
} else {
|
|
rec_vertex_input(P.set_vi, cb, v.n_bind, v.b, v.n_attr, v.a);
|
|
sh->vi = v, n++;
|
|
}
|
|
} else if (!mesh) {
|
|
sh->vi.n_attr = UINT32_MAX; // not recorded: unknown to the next draw
|
|
}
|
|
if (P.dyn_clamp) {
|
|
VkBool32 clamp = s->depth_clamp != 0;
|
|
DYN(core, clamp, clamp, rec_bool(P.set_clamp, cb, clamp));
|
|
}
|
|
if (P.dyn_poly) {
|
|
VkPolygonMode pm = polygon_mode(pl->mode);
|
|
DYN(core, poly, pm, rec_polygon_mode(P.set_poly, cb, pm));
|
|
}
|
|
if (P.dyn_a2c) { // the same signature as the depth clamp call
|
|
VkBool32 a2c = s->ms.a2c != 0;
|
|
DYN(core, a2c, a2c, rec_bool((PFN_vkCmdSetDepthClampEnableEXT)P.set_a2c, cb, a2c));
|
|
}
|
|
#undef DYN
|
|
sh->have |= VK_DYN_CORE;
|
|
if (mesh) sh->have &= ~(uint32_t)VK_DYN_VTX;
|
|
else sh->have |= VK_DYN_VTX;
|
|
int nt = target_count(s);
|
|
if (nt) {
|
|
VkBool32 en[8];
|
|
VkColorBlendEquationEXT eq[8];
|
|
VkColorComponentFlags wm[8];
|
|
for (int i = 0; i < nt; i++) {
|
|
const blend_st *b = &s->blend[i];
|
|
en[i] = (s->color.blend_en >> i & 1) && blendable(target_format(s->color_fmt[i], NULL, NULL));
|
|
eq[i] = (VkColorBlendEquationEXT){blend_factor(b->src_rgb), blend_factor(b->dst_rgb), blend_op(b->eq_rgb),
|
|
blend_factor(b->src_a), blend_factor(b->dst_a), blend_op(b->eq_a)};
|
|
wm[i] = target_wmask(s, i);
|
|
}
|
|
// The calls write targets [0, nt); targets past nt keep what the command buffer had.
|
|
int known = blend && (uint32_t)nt <= sh->nt;
|
|
if (P.dyn_blend_en) {
|
|
if (known && !memcmp(sh->en, en, sizeof en[0] * nt)) left++;
|
|
else rec_blend_enable(P.set_blend_en, cb, 0, (uint32_t)nt, en), n++;
|
|
}
|
|
if (P.dyn_blend_eq) {
|
|
if (known && !memcmp(sh->eq, eq, sizeof eq[0] * nt)) left++;
|
|
else rec_blend_equation(P.set_blend_eq, cb, 0, (uint32_t)nt, eq), n++;
|
|
}
|
|
if (P.dyn_wmask) {
|
|
if (known && !memcmp(sh->wm, wm, sizeof wm[0] * nt)) left++;
|
|
else rec_write_mask(P.set_wmask, cb, 0, (uint32_t)nt, wm), n++;
|
|
}
|
|
memcpy(sh->en, en, sizeof en[0] * nt), memcpy(sh->eq, eq, sizeof eq[0] * nt), memcpy(sh->wm, wm, sizeof wm[0] * nt);
|
|
sh->nt = blend && sh->nt > (uint32_t)nt ? sh->nt : (uint32_t)nt;
|
|
sh->have |= VK_DYN_BLEND;
|
|
}
|
|
*skipped += (uint64_t)left;
|
|
return n;
|
|
}
|
|
|
|
// ---- statistics, persistence, teardown
|
|
|
|
void vk_pipeline_stats(VkPipeStats *s) {
|
|
plat_mutex_lock_shared(&P.lock);
|
|
*s = P.stats;
|
|
plat_mutex_unlock_shared(&P.lock);
|
|
}
|
|
|
|
static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache
|
|
|
|
// Copies P.cache's data into pipeline-cache.bin, with cache_rw held exclusively for the copy only (not the file write).
|
|
// The lock is only ever tried, never waited for: a waiting exclusive request would hold back every new build
|
|
// (a shared request queues behind it), so a pending save could stall the replay. wait_ms: how long to keep trying
|
|
// while builds run (0: one try). Returns 0 when no build-free moment came, 1 otherwise (saved, or nothing to save).
|
|
static int cache_save(uint32_t wait_ms) {
|
|
if (!P.ready || !P.cache || !P.cache_path[0]) return 1;
|
|
// Not under P.lock: holding P.lock through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped
|
|
// every pipeline lookup of the replay, a freeze at each 20 s autosave.
|
|
plat_mutex_lock(&save_lock);
|
|
if (!P.cache) { // vk_pipeline_shutdown destroyed it
|
|
plat_mutex_unlock(&save_lock);
|
|
return 1;
|
|
}
|
|
uint64_t until = plat_now_ms() + wait_ms;
|
|
while (!plat_mutex_trylock(&cache_rw)) {
|
|
if (plat_now_ms() >= until) {
|
|
plat_mutex_unlock(&save_lock);
|
|
return 0;
|
|
}
|
|
plat_sleep_ms(2);
|
|
}
|
|
size_t size = 0;
|
|
void *data = NULL;
|
|
int got = vkGetPipelineCacheData(P.device, P.cache, &size, NULL) == VK_SUCCESS && size && (data = malloc(size)) &&
|
|
vkGetPipelineCacheData(P.device, P.cache, &size, data) == VK_SUCCESS;
|
|
plat_mutex_unlock(&cache_rw);
|
|
if (got) {
|
|
if (!plat_file_write_atomic(P.cache_path, data, size)) rt_log("vk_pipeline: cannot write %s", P.cache_path);
|
|
else if (P.feedback) {
|
|
char when[96];
|
|
snprintf(when, sizeof when, "at a save of %.1f MB (%+.1f MB since the boot)", size / 1048576.0,
|
|
((double)size - (double)P.cache_loaded) / 1048576.0);
|
|
fb_log(when);
|
|
}
|
|
}
|
|
free(data);
|
|
plat_mutex_unlock(&save_lock);
|
|
return 1;
|
|
}
|
|
|
|
// Builds still running after this long at a save that waits (exit, the shader preparation) leave the file as it was.
|
|
#define SAVE_WAIT_MS 5000
|
|
|
|
static void save_with(uint32_t wait_ms) {
|
|
if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log();
|
|
if (P.ready && P.gpl) {
|
|
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws "
|
|
"lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued "
|
|
"low after their fast link), %llu drawing-thread links over 5 ms; %llu waiting draws built their libraries "
|
|
"(avg %.2f ms, max %.2f ms)",
|
|
(unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, P.g.links ? P.g.link_ms / P.g.links : 0.0,
|
|
P.g.link_max, (unsigned long long)P.g.link_fail, (unsigned long long)P.g.defer,
|
|
(unsigned long long)P.g.swaps, (unsigned long long)P.g.requeued, (unsigned long long)P.g.slow,
|
|
(unsigned long long)P.g.dlinks, P.g.dlinks ? P.g.dlink_ms / P.g.dlinks : 0.0, P.g.dlink_max);
|
|
if (U.waits || U.queued)
|
|
rt_log("vk_pipeline: gpl: library jobs for waiting draws: %llu queued, %llu built by a worker, %llu by the "
|
|
"thread that needed them (no worker free), %llu waits ended at their limit; %llu fast links on the %d "
|
|
"library workers",
|
|
(unsigned long long)U.queued, (unsigned long long)U.by_worker, (unsigned long long)U.stolen,
|
|
(unsigned long long)U.timeouts, (unsigned long long)U.links, U.lib_workers);
|
|
static const char *const kn[4] = {"vertex input", "pre-rasterization", "fragment shader", "fragment output"};
|
|
for (int i = 0; i < 4; i++)
|
|
rt_log("vk_pipeline: gpl: %s libraries %llu (avg %.2f ms, max %.2f ms)", kn[i], (unsigned long long)P.g.lib_n[i],
|
|
P.g.lib_n[i] ? P.g.lib_ms[i] / P.g.lib_n[i] : 0.0, P.g.lib_max[i]);
|
|
if (LB.n)
|
|
rt_log("vk_pipeline: gpl: prebuilt libraries a draw used: pre-rasterization %llu, fragment shader %llu; "
|
|
"compute pipelines built on the replay thread %llu", (unsigned long long)P.g.pre_used[LIB_PR],
|
|
(unsigned long long)P.g.pre_used[LIB_FS], (unsigned long long)LB.cs_draw);
|
|
}
|
|
gfx_shader_record_flush(1);
|
|
if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below
|
|
if (!cache_save(wait_ms) && wait_ms)
|
|
rt_log("vk_pipeline: %s not saved: pipeline builds kept running for %u ms", P.cache_path, (unsigned)wait_ms);
|
|
}
|
|
|
|
// The save at exit and after the shader preparation: it waits up to SAVE_WAIT_MS for a moment with no build running.
|
|
void vk_pipeline_save(void) {
|
|
save_with(SAVE_WAIT_MS);
|
|
}
|
|
|
|
// The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it
|
|
// runs: at most every 20 s, and only after new pipelines were built. New pipeline keys are appended every 2 s.
|
|
static void autosave_once(void) {
|
|
static uint64_t last_n, last_ms, last_keys;
|
|
if (!P.ready) return;
|
|
uint64_t now = plat_now_ms();
|
|
if (now - last_keys >= 2000) {
|
|
last_keys = now;
|
|
if (*(volatile uint32_t *)&K.pend_n) keys_flush();
|
|
pb_flush();
|
|
cu_flush();
|
|
}
|
|
gfx_shader_record_flush(0);
|
|
if (now - last_ms < 20000) return;
|
|
VkPipeStats s;
|
|
vk_pipeline_stats(&s);
|
|
uint64_t n = s.graphics + s.compute + cache_merges; // a merge comes after its pipeline was counted (bg_main)
|
|
if (n == last_n) return;
|
|
// Only at a moment with no build running (cache_save): while one runs the save waits for the next tick, 500 ms
|
|
// on, before it logs or flushes anything, so a busy stretch does not repeat the statistics lines every tick.
|
|
if (P.cache && P.cache_path[0]) {
|
|
if (!plat_mutex_trylock(&cache_rw)) return;
|
|
plat_mutex_unlock(&cache_rw);
|
|
}
|
|
last_n = n, last_ms = now;
|
|
save_with(0);
|
|
}
|
|
|
|
// On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to
|
|
// the disk each) ran on the present thread, and the render thread waited for it to take the next frame.
|
|
static void autosave_main(void *arg) {
|
|
(void)arg;
|
|
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
for (;;) {
|
|
plat_sleep_ms(500);
|
|
autosave_once();
|
|
}
|
|
}
|
|
|
|
void vk_pipeline_autosave(void) {
|
|
static int32_t started;
|
|
if (!P.ready || plat_xchg32(&started, 1)) return;
|
|
if (!plat_thread_spawn(autosave_main, NULL, "pipe autosave", 0))
|
|
rt_log("vk_pipeline: cannot start the autosave thread; the cache is saved at exit only");
|
|
}
|
|
|
|
void vk_pipeline_shutdown(void) {
|
|
if (!P.ready) return;
|
|
lb_stop();
|
|
workers_drain(); // jobs point at layouts and Pipe entries freed below
|
|
vk_pipeline_save();
|
|
plat_mutex_lock(&P.lock);
|
|
vkDeviceWaitIdle(P.device);
|
|
for (int i = 0; i < PIPE_BUCKETS; i++) {
|
|
for (Pipe *p = pipes[i], *n; p; p = n) {
|
|
n = p->next;
|
|
if (p->info.pipeline) vkDestroyPipeline(P.device, p->info.pipeline, NULL);
|
|
if (p->fast && p->fast != p->info.pipeline) vkDestroyPipeline(P.device, p->fast, NULL);
|
|
free(p->bindings), free(p);
|
|
}
|
|
pipes[i] = NULL;
|
|
}
|
|
libs_destroy();
|
|
for (int i = 0; i < LAYOUT_BUCKETS; i++) {
|
|
for (Layout *l = layouts[i], *n; l; l = n) {
|
|
n = l->next;
|
|
for (int k = 0; k < 2; k++)
|
|
if (l->tpl[k]) P.tpl_destroy(P.device, l->tpl[k], NULL);
|
|
vkDestroyPipelineLayout(P.device, l->layout, NULL);
|
|
for (int h = 0; h < 2; h++)
|
|
if (l->half[h]) vkDestroyPipelineLayout(P.device, l->half[h], NULL);
|
|
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
|
|
if (l->set1) vkDestroyDescriptorSetLayout(P.device, l->set1, NULL);
|
|
free(l);
|
|
}
|
|
layouts[i] = NULL;
|
|
}
|
|
plat_mutex_lock(&save_lock); // not while the autosave thread copies it
|
|
if (P.cache) vkDestroyPipelineCache(P.device, P.cache, NULL);
|
|
P.cache = VK_NULL_HANDLE;
|
|
plat_mutex_unlock(&save_lock);
|
|
memset(&P.stats, 0, sizeof P.stats);
|
|
plat_xchg32(&P.ready, 0);
|
|
plat_mutex_unlock(&P.lock);
|
|
}
|
|
|
|
// ---- self-test: census shaders (probe_census_dir, tools/shader_census.py) through gfx_shader_get into pipelines.
|
|
// Graphics pairs are a fragment shader and a vertex shader of the same source file, spread over the census; compute
|
|
// shaders are spread the same way. Every pipeline is built twice (the second call must hit the cache) and the
|
|
// dynamic state of the first graphics pipeline is recorded into a throwaway command buffer.
|
|
|
|
typedef struct { const char *key, *c1, *src; size_t c1_len, src_len; int stage; uint32_t local[3]; } CensusEntry;
|
|
|
|
static const char *st_field(const char *from, const char *to, const char *name, size_t *len) {
|
|
char pat[32];
|
|
snprintf(pat, sizeof pat, "\"%s\": \"", name);
|
|
const char *p = strstr(from, pat);
|
|
if (!p || p >= to) return NULL;
|
|
p += strlen(pat);
|
|
const char *e = memchr(p, '"', (size_t)(to - p));
|
|
if (!e) return NULL;
|
|
*len = (size_t)(e - p);
|
|
return p;
|
|
}
|
|
|
|
static int census_cmp(const void *x, const void *y) {
|
|
const CensusEntry *a = x, *b = y;
|
|
size_t n = a->src_len < b->src_len ? a->src_len : b->src_len;
|
|
int c = memcmp(a->src, b->src, n);
|
|
if (c) return c;
|
|
if (a->src_len != b->src_len) return a->src_len < b->src_len ? -1 : 1;
|
|
return a->stage - b->stage;
|
|
}
|
|
|
|
static char *census_read(const char *root, size_t *size) {
|
|
char path[PLAT_PATH_MAX * 2 + 64];
|
|
snprintf(path, sizeof path, "%s/jobs.json", root);
|
|
FILE *f = fopen(path, "rb");
|
|
if (!f) return NULL;
|
|
fseek(f, 0, SEEK_END);
|
|
long n = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
char *b = malloc((size_t)n + 1);
|
|
if (b && fread(b, 1, (size_t)n, f) != (size_t)n) free(b), b = NULL;
|
|
fclose(f);
|
|
if (b) b[n] = 0, *size = (size_t)n;
|
|
return b;
|
|
}
|
|
|
|
static int census_parse(const char *jobs, CensusEntry *out, int max) {
|
|
int n = 0;
|
|
for (const char *p = jobs; n < max && (p = strstr(p, "\"job\": {")); p += 8) {
|
|
const char *end = strstr(p, "\"origin\": ");
|
|
if (!end) break;
|
|
size_t kl, sl, cl, rl;
|
|
const char *k = st_field(p, end, "key", &kl), *s = st_field(p, end, "stage", &sl), *c = st_field(p, end, "c1", &cl),
|
|
*r = st_field(p, end, "source", &rl);
|
|
if (!k || !s || !c || !r || kl != 64) continue;
|
|
CensusEntry *e = &out[n++];
|
|
e->key = k, e->c1 = c, e->c1_len = cl / 2, e->src = r, e->src_len = rl;
|
|
e->stage = !strncmp(s, "vs", sl) ? GFX_STAGE_VS : !strncmp(s, "ps", sl) ? GFX_STAGE_FS : GFX_STAGE_CS;
|
|
e->local[0] = e->local[1] = e->local[2] = 1;
|
|
const char *l = strstr(p, "\"local\": [");
|
|
if (l && l < end) {
|
|
l += 10;
|
|
for (int i = 0; i < 3; i++) {
|
|
while (*l == ' ' || *l == '\n' || *l == '\r' || *l == ',') l++;
|
|
e->local[i] = (uint32_t)strtoul(l, (char **)&l, 10);
|
|
}
|
|
}
|
|
}
|
|
return n;
|
|
}
|
|
|
|
static int hexv(char c) { return c <= '9' ? c - '0' : (c | 32) - 'a' + 10; }
|
|
|
|
// The program the game would hand NVN for a census entry, translated with default answers (as gfx_shader_probe.c).
|
|
static const GfxShader *census_shader(const char *root, const CensusEntry *e) {
|
|
char path[PLAT_PATH_MAX * 2 + 128];
|
|
static uint8_t c1[8192];
|
|
uint8_t control[0x880] = {0};
|
|
if (e->c1_len > sizeof c1) return NULL;
|
|
for (size_t b = 0; b < e->c1_len; b++) c1[b] = (uint8_t)(hexv(e->c1[b * 2]) << 4 | hexv(e->c1[b * 2 + 1]));
|
|
snprintf(path, sizeof path, "%s/code/%.64s.code", root, e->key);
|
|
FILE *f = fopen(path, "rb");
|
|
if (!f) return NULL;
|
|
fseek(f, 0, SEEK_END);
|
|
long body = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
size_t skip = e->stage == GFX_STAGE_CS ? 0 : 0x30, total = skip + (size_t)body;
|
|
uint8_t *code = calloc(1, total);
|
|
int ok = code && fread(code + skip, 1, (size_t)body, f) == (size_t)body;
|
|
fclose(f);
|
|
if (!ok) return free(code), NULL;
|
|
uint32_t c1_off = (uint32_t)(total - e->c1_len), length = (uint32_t)total;
|
|
memcpy(control + 0x700, &c1_off, 4), memcpy(control + 0x704, &length, 4);
|
|
GfxShaderAnswers a = {0};
|
|
if (e->stage == GFX_STAGE_CS) for (int k = 0; k < 3; k++) a.local[k] = (uint16_t)e->local[k];
|
|
const GfxShader *s = gfx_shader_get(e->stage, control, code, total, c1, &a);
|
|
free(code);
|
|
return s;
|
|
}
|
|
|
|
// A state for a census pair: one target per fragment output (RGBA8 of the output's type), D32S8 depth with the test on,
|
|
// alpha blending on target 0, one RGBA32 attribute per vertex input, interleaved in stream 0.
|
|
static void selftest_state(VkPipeState *s, const GfxShader *vs, const GfxShader *fs) {
|
|
vk_pipe_state_defaults(s);
|
|
s->vs = vs, s->fs = fs;
|
|
GfxShaderIo io[32];
|
|
uint32_t outs = gfx_shader_outputs(fs, io);
|
|
for (int i = 0; i < 8; i++)
|
|
if (outs >> i & 1)
|
|
s->color_fmt[i] = io[i].type == GFX_TYPE_UINT ? NVN_FORMAT_RGBA8UI : io[i].type == GFX_TYPE_SINT ? NVN_FORMAT_RGBA8I
|
|
: NVN_FORMAT_RGBA8;
|
|
s->depth_fmt = NVN_FORMAT_DEPTH32F_STENCIL8;
|
|
s->ds.dt = 1;
|
|
s->color.blend_en = 1;
|
|
s->blend[0].src_rgb = NVN_BLEND_FUNC_SRC_ALPHA, s->blend[0].dst_rgb = NVN_BLEND_FUNC_ONE_MINUS_SRC_ALPHA;
|
|
s->poly.cull = NVN_FACE_BACK;
|
|
uint32_t ins = gfx_shader_inputs(vs, io), off = 0;
|
|
for (int l = 0; l < 16; l++)
|
|
if (ins >> l & 1) {
|
|
va_st a = {io[l].type == GFX_TYPE_UINT ? NVN_FORMAT_RGBA32UI : io[l].type == GFX_TYPE_SINT ? NVN_FORMAT_RGBA32I
|
|
: NVN_FORMAT_RGBA32F,
|
|
off, 0};
|
|
memcpy(&s->vattr[l], &a, sizeof a);
|
|
off += 16;
|
|
}
|
|
s->vstream[0].stride = off ? off : 16;
|
|
}
|
|
|
|
// Records the dynamic state of a pipeline into a command buffer that is never submitted (validation checks the calls).
|
|
static int selftest_record(const VkPipeState *s, VkPipeline pipe) {
|
|
VkCtx *c = vk_ctx();
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .queueFamilyIndex = c->gfx_family};
|
|
VkCommandPool pool;
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &pool) != VK_SUCCESS) return 0;
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
VkCommandBuffer cb;
|
|
int ok = vkAllocateCommandBuffers(c->device, &ai, &cb) == VK_SUCCESS;
|
|
if (ok) {
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(cb, &bi);
|
|
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, pipe);
|
|
vk_pipeline_set_dynamic(cb, s);
|
|
ok = vkEndCommandBuffer(cb) == VK_SUCCESS;
|
|
}
|
|
vkDestroyCommandPool(c->device, pool, NULL);
|
|
return ok;
|
|
}
|
|
|
|
int vk_pipeline_selftest(void) {
|
|
enum { MAXN = 20000, WANT_PAIRS = 64, WANT_CS = 16 };
|
|
char root[PLAT_PATH_MAX * 2];
|
|
if (!vk_init() || !ensure_init()) { printf("vk_pipeline selftest: FAILED: no Vulkan device\n"); return 1; }
|
|
if (!probe_census_dir(root, sizeof root)) { // the census folder, build/<version>/shaders
|
|
printf("vk_pipeline selftest: FAILED: no build/v%d/shaders/jobs.json above the exe\n", rt_game_version);
|
|
return 1;
|
|
}
|
|
size_t size;
|
|
char *jobs = census_read(root, &size);
|
|
CensusEntry *e = malloc(sizeof *e * MAXN);
|
|
int n = jobs && e ? census_parse(jobs, e, MAXN) : 0;
|
|
if (!n) { printf("vk_pipeline selftest: FAILED: no census entries\n"); free(jobs), free(e); return 1; }
|
|
qsort(e, (size_t)n, sizeof *e, census_cmp);
|
|
long err0 = vk_validation_errors_total();
|
|
VkPipeStats st0;
|
|
vk_pipeline_stats(&st0);
|
|
|
|
// candidate pairs: each fragment shader of a source file with one of the file's vertex shaders (round robin)
|
|
int (*pairs)[2] = malloc(sizeof *pairs * (size_t)n), np = 0, *cs = malloc(sizeof *cs * (size_t)n), nc = 0;
|
|
for (int i = 0; i < n;) {
|
|
int j = i, v0 = -1, nv = 0, nf = 0;
|
|
for (; j < n && e[j].src_len == e[i].src_len && !memcmp(e[j].src, e[i].src, e[i].src_len); j++) {
|
|
if (e[j].stage == GFX_STAGE_VS && v0 < 0) v0 = j; // sorted by stage within a file: vertex shaders first
|
|
nv += e[j].stage == GFX_STAGE_VS;
|
|
if (e[j].stage == GFX_STAGE_CS) cs[nc++] = j;
|
|
}
|
|
for (int k = i; k < j && nv; k++)
|
|
if (e[k].stage == GFX_STAGE_FS) pairs[np][0] = v0 + nf++ % nv, pairs[np][1] = k, np++;
|
|
i = j;
|
|
}
|
|
int tried = 0, built = 0, cached = 1, shader_fail = 0, ctried = 0, cbuilt = 0, recorded = 0, varying_fix = 0;
|
|
for (int t = 0; t < WANT_PAIRS && t < np; t++) {
|
|
const int *pr = pairs[(long long)t * np / (np < WANT_PAIRS ? np : WANT_PAIRS)];
|
|
tried++;
|
|
const GfxShader *vs = census_shader(root, &e[pr[0]]), *fs = census_shader(root, &e[pr[1]]);
|
|
if (!vs || !fs) { shader_fail++; continue; }
|
|
if (gfx_shader_inputs(fs, NULL) & ~gfx_shader_outputs(vs, NULL)) varying_fix++;
|
|
VkPipeState s;
|
|
selftest_state(&s, vs, fs);
|
|
GfxShaderAnswers fa = {0};
|
|
vk_pipeline_answers(&s, NULL, &fa);
|
|
const VkPipeInfo *info;
|
|
VkPipeline p = vk_pipeline_graphics(&s, &info);
|
|
if (!p) continue;
|
|
built++;
|
|
const VkPipeInfo *again;
|
|
if (vk_pipeline_graphics(&s, &again) != p || again != info) cached = 0;
|
|
if (!recorded) recorded = selftest_record(&s, p);
|
|
}
|
|
for (int t = 0; t < WANT_CS && t < nc; t++) {
|
|
const CensusEntry *c = &e[cs[(long long)t * nc / (nc < WANT_CS ? nc : WANT_CS)]];
|
|
ctried++;
|
|
const GfxShader *s = census_shader(root, c);
|
|
if (!s) { shader_fail++; continue; }
|
|
const VkPipeInfo *info;
|
|
VkPipeline p = vk_pipeline_compute(s, &info);
|
|
if (!p) continue;
|
|
cbuilt++;
|
|
const VkPipeInfo *again;
|
|
if (vk_pipeline_compute(s, &again) != p || again != info) cached = 0;
|
|
}
|
|
vk_pipeline_save();
|
|
VkPipeStats st;
|
|
vk_pipeline_stats(&st);
|
|
long errors = vk_validation_errors_total() - err0;
|
|
printf("vk_pipeline selftest: graphics %d/%d census pairs (%d needed vertex outputs added), compute %d/%d, "
|
|
"%d shaders failed to translate; layouts %llu (%llu push); cache hits %llu; dynamic state recorded %d; "
|
|
"validation errors %ld\n",
|
|
built, tried, varying_fix, cbuilt, ctried, shader_fail, (unsigned long long)(st.layouts - st0.layouts),
|
|
(unsigned long long)(st.push_layouts - st0.push_layouts), (unsigned long long)(st.hits - st0.hits), recorded,
|
|
errors);
|
|
free(pairs), free(cs), free(e), free(jobs);
|
|
int ok = built > 0 && cbuilt > 0 && cached && recorded && errors == 0;
|
|
printf("vk_pipeline selftest: %s\n", ok ? "passed" : "FAILED");
|
|
return ok ? 0 : 1;
|
|
}
|