mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
677a69378f
A mesh pipeline whose fragment stage reads a location the mesh program never writes now gets a fragment module where that input is a private variable set to (0, 0, 0, 1) at entry, or to (s, t, 0, 1) of gl_PointCoord when the mesh program draws points. The fragment shaders recorded with two of the game's point-drawing mesh programs read location 5, which those programs never write; Vulkan leaves such an input undefined. Checks: tests/fs_fill_inputs_check.py passes; 237 recorded fragment modules rewritten offline pass spirv-val; a 60 s Kakariko run with the change shows the area as before with no Vulkan errors. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
7466 lines
408 KiB
C
7466 lines
408 KiB
C
// Graphics and compute pipelines and descriptor layouts of the Vulkan replay. The op-stream replay (vk_exec.c)
|
|
// collects NVN state into a VkPipeState and asks for the pipeline of each draw; this file maps that state to Vulkan,
|
|
// numbers the descriptor bindings and keeps the caches.
|
|
//
|
|
// Binding model. The translator (tools/xlate/Program.cs, a SPIR-V back end) decorates every resource of a
|
|
// shader with DescriptorSet 0 and numbers the bindings of each shader from 1 in the order it meets them (Accessor
|
|
// Create*Binding); binding 0 is its support buffer (ResourceManager: SupportBuffer.Binding, a std140 uniform
|
|
// buffer every shader declares). The numbering is per shader, so the vertex and fragment stages of one program collide.
|
|
// The scheme here:
|
|
// - one descriptor set, set 0, per pipeline; no push constants (the translator's SPIR-V uses none);
|
|
// - binding 0: the support buffer, one uniform buffer shared by all stages of the pipeline;
|
|
// - vertex stage: its own bindings 1..n unchanged; fragment stage: its bindings b >= 1 move to b + n (the SPIR-V
|
|
// Binding decorations are rewritten when the module is created); compute: its own bindings;
|
|
// - descriptor types from the reflection: constant buffers UNIFORM_BUFFER, storage buffers STORAGE_BUFFER, textures
|
|
// COMBINED_IMAGE_SAMPLER (UNIFORM_TEXEL_BUFFER for buffer textures, SAMPLED_IMAGE for separate ones), images
|
|
// STORAGE_IMAGE (STORAGE_TEXEL_BUFFER for buffer images), array lengths as descriptor counts;
|
|
// - set 0 is a push-descriptor set (VK_KHR_push_descriptor) when its descriptors fit maxPushDescriptors (32 on the RX
|
|
// 9070; a pipeline layout may hold one push set only, which rules out a set per stage); larger ones (a vertex and a
|
|
// fragment stage near the census maxima of 15 and 27) are ordinary sets the replay allocates (VkPipeInfo.push 0).
|
|
// A fixed layout per stage combination (a binding per constant buffer, storage buffer, texture
|
|
// and image slot per stage) would hold about 150 descriptors and could never be pushed, so layouts are made per distinct
|
|
// binding list and shared by every pipeline with that list; VkPipeInfo.bindings tells the replay which NVN slot feeds
|
|
// each binding.
|
|
//
|
|
// Fragment inputs the vertex stage does not write (stages are translated separately) get
|
|
// outputs added to the vertex module, as Vulkan rejects such pairs, initialised to (0, 0, 0, 1) as Maxwell reads an
|
|
// unwritten output (docs/maxwell.md 1.2). The unwritten components of an output the vertex stage does write
|
|
// start at (0, 0, 0, 1) in the translator (tools/xlate/patches/0006).
|
|
//
|
|
// Pipeline key. Everything Vulkan 1.3 can take as dynamic state is dynamic: viewports and scissors with count, depth
|
|
// bias, blend constants, stencil masks and references, cull mode, front face, topology (within its class), depth and
|
|
// stencil tests and ops, depth bias enable, rasterizer discard; vertex input (VK_EXT_vertex_input_dynamic_state, else
|
|
// binding strides only); and, where VK_EXT_extended_dynamic_state3 offers them, depth clamp, polygon mode, blend enables,
|
|
// equations, write masks and alpha to coverage. The key holds the rest: shaders, target formats, sample count, topology
|
|
// class, depth clip mode, logic op and what is not dynamic on the GPU. The VkPipelineCache persists to
|
|
// pipeline-cache.bin next to nx2navi.exe (NX2NAVI_PIPELINE_CACHE overrides the path).
|
|
//
|
|
// The vertex format table has a 3-component fallback. The pipeline key holds the program, target formats, blend and
|
|
// write masks, vertex layout and sample count. The binding conventions are those of the shader translator.
|
|
// Contains code Copyright (c) 2026 Peiikai, Copyright (c) 2026 jhaizhou (MIT; see THIRD_PARTY_NOTICES.md).
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "gfx.h"
|
|
#include "host.h"
|
|
#include "nvn_hmap.h"
|
|
#include "platform/platform.h"
|
|
#include "prp_file.h"
|
|
#include "tex.h"
|
|
#include "vk_build_gate.h"
|
|
#include "vk_cache_file.h"
|
|
#include "vk_save_retry.h"
|
|
#include "vk_warm_hint.h"
|
|
|
|
#define LOG_ONCE(...) do { static volatile int32_t once_; if (!plat_xchg32(&once_, 1)) rt_log(__VA_ARGS__); } while (0)
|
|
|
|
enum { SPV_OP_ENTRY_POINT = 15, SPV_OP_TYPE_INT = 21, SPV_OP_TYPE_FLOAT = 22, SPV_OP_TYPE_VECTOR = 23,
|
|
SPV_OP_TYPE_POINTER = 32, SPV_OP_CONSTANT = 43, SPV_OP_CONSTANT_COMPOSITE = 44,
|
|
SPV_OP_CONSTANT_NULL = 46, SPV_OP_FUNCTION = 54, SPV_OP_VARIABLE = 59,
|
|
SPV_OP_DECORATE = 71, SPV_DEC_BUILTIN = 11, SPV_DEC_LOCATION = 30, SPV_DEC_INDEX = 32, SPV_DEC_BINDING = 33, SPV_DEC_DESCRIPTOR_SET = 34,
|
|
SPV_SC_OUTPUT = 3 };
|
|
|
|
#define MAX_BINDINGS 128
|
|
#define PIPE_BUCKETS 4096
|
|
#define LAYOUT_BUCKETS 256
|
|
|
|
// ---- device capabilities and the pipeline cache (set up on first use, under P.lock)
|
|
|
|
enum { NP_16 = 1, NP_32 = 2, NP_64 = 4 }; // float widths in P.nan_preserve and spirv_nan_preserve
|
|
|
|
static struct {
|
|
PlatMutex lock;
|
|
volatile int32_t ready;
|
|
VkDevice device;
|
|
VkPipelineCache cache;
|
|
uint32_t max_push;
|
|
int indep; // stage-independent layouts (vk_caps.gpl): vertex set 0, fragment set 1
|
|
int push_stage; // GFX_STAGE_* whose set is pushed with independent layouts
|
|
int gpl; // draws whose pipeline is building get a fast-linked one (vk_caps.gpl)
|
|
struct { uint64_t links, fb_links, wlinks, flinks, dlinks, pr_added, lib_slow[4], ready_n[6], ready_nofast, ready_mesh, ready_recorded, ready_fresh, ready_logged, lib_logged, hints, hint_libs, requeued, link_fail, defer, lib_n[4], lib_fail, swaps, slow, pre_used[4]; double pr_added_ms, lib_slow_ms[4], ready_ms, ready_max, link_ms, link_max, lib_ms[4], lib_max[4], dlink_ms, dlink_max; } g;
|
|
int vs_outs_all; // NX2NAVI_VS_OUTS: 0 union of paired fragment inputs, 1 (all) all 32 output locations
|
|
int dyn_vi, dyn_clamp, dyn_poly, dyn_blend_en, dyn_blend_eq, dyn_wmask, dyn_a2c;
|
|
int f_logic_op, f_depth_clamp, f_fill_non_solid, f_dual_src, f_full_subgroups;
|
|
int cache_control; // pipelineCreationCacheControl: a create can ask for the pipeline caches only
|
|
int feedback; // NX2NAVI_PIPE_FEEDBACK: 1 count pipeline cache hits, 2 also log each compute and graphics build
|
|
uint64_t cache_loaded; // bytes of pipeline-cache.bin the driver took at boot
|
|
VkPipelineCache play; // what this run compiles (pipeline-cache-play.bin); VK_NULL_HANDLE: all goes into cache
|
|
uint64_t play_loaded; // bytes of pipeline-cache-play.bin the driver took at boot
|
|
uint64_t saved[2]; // the data size of cache [0] and play [1] that their files hold; 0: not known
|
|
int fold; // the play file's content was merged into cache: saving cache removes the play file
|
|
volatile uint64_t play_made; // pipelines this run compiled into play
|
|
int pv_last; // provokingVertexLast enabled: NVN's default is the last vertex
|
|
VkShaderStageFlags sg32_stages; // stages that accept requiredSubgroupSize = 32 (0: not available)
|
|
int reconverge; // modules with subgroup ops get MaximallyReconvergesKHR (spirv_reconverge)
|
|
uint32_t nan_preserve; // float widths that get SignedZeroInfNanPreserve (spirv_nan_preserve): NP_16|NP_32|NP_64
|
|
int nan_bits; // NaN tests as integer bit tests (spirv_nan_bits)
|
|
volatile uint32_t nan_bits_tests, nan_bits_selects, nan_bits_modules; // rewritten so far (the first log line)
|
|
VkDynamicState dyn[32];
|
|
uint32_t n_dyn;
|
|
PFN_vkCmdPushDescriptorSetKHR push;
|
|
int tpl; // descriptor update templates on (make_tpl)
|
|
PFN_vkCreateDescriptorUpdateTemplate tpl_create;
|
|
PFN_vkDestroyDescriptorUpdateTemplate tpl_destroy;
|
|
PFN_vkUpdateDescriptorSetWithTemplate tpl_update;
|
|
PFN_vkCmdPushDescriptorSetWithTemplateKHR tpl_push;
|
|
PFN_vkCmdSetVertexInputEXT set_vi;
|
|
PFN_vkCmdSetDepthClampEnableEXT set_clamp;
|
|
PFN_vkCmdSetPolygonModeEXT set_poly;
|
|
PFN_vkCmdSetColorBlendEnableEXT set_blend_en;
|
|
PFN_vkCmdSetColorBlendEquationEXT set_blend_eq;
|
|
PFN_vkCmdSetColorWriteMaskEXT set_wmask;
|
|
PFN_vkCmdSetAlphaToCoverageEnableEXT set_a2c;
|
|
char cache_path[PLAT_PATH_MAX * 2];
|
|
char play_path[PLAT_PATH_MAX * 2 + 8]; // pipeline-cache-play.bin beside cache_path
|
|
VkPipeStats stats;
|
|
} P = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
typedef struct Layout {
|
|
struct Layout *next;
|
|
uint64_t hash;
|
|
VkDescriptorSetLayout set;
|
|
VkPipelineLayout layout;
|
|
int push;
|
|
uint32_t n;
|
|
VkDescriptorSetLayout set1; // independent layouts: the fragment set (set 1); VK_NULL_HANDLE otherwise
|
|
int push1; // set 1 is a push-descriptor set
|
|
uint32_t n0; // independent layouts: b[0..n0) are set 0, the rest set 1
|
|
VkPipelineLayout half[2]; // GPL libraries: {set 0, null}, {null, set 1} (made on first use)
|
|
VkDescriptorUpdateTemplate tpl[2]; // per set (make_tpl); VK_NULL_HANDLE: written with write structs
|
|
VkDescriptorSetLayoutBinding b[];
|
|
} Layout;
|
|
|
|
typedef struct Pipe {
|
|
struct Pipe *next;
|
|
uint64_t hash;
|
|
VkPipeInfo info;
|
|
VkPipeBinding *bindings;
|
|
int building; // a background job owns info.pipeline (under P.lock)
|
|
struct Job *job; // that job while building
|
|
const struct Layout *lay; // graphics: the layout it was entered with
|
|
VkPipeline fast; // GPL: the fast-linked pipeline (kept until shutdown: recorded draws use it)
|
|
uint8_t fast_tried;
|
|
uint8_t link_queued; // a waiting draw queued its fast link to the workers (LibJob p), not yet run
|
|
uint8_t used; // compute: a dispatch of this run asked for it (vk_pipeline_compute)
|
|
uint8_t pre; // compute: built and kept by the boot prebuild (lb_compute) or bg_main
|
|
uint64_t t_miss; // graphics: when a draw first missed it (ns), 0 once it is usable (ready_note)
|
|
size_t key_size;
|
|
uint8_t key[];
|
|
} Pipe;
|
|
|
|
static Layout *layouts[LAYOUT_BUCKETS];
|
|
static Pipe *pipes[PIPE_BUCKETS];
|
|
|
|
static uint64_t fnv(uint64_t h, const void *p, size_t n) {
|
|
const uint8_t *b = p;
|
|
for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 0x100000001b3ull;
|
|
return h;
|
|
}
|
|
#define FNV0 0xcbf29ce484222325ull
|
|
static __thread uint64_t mm_hash; // FNV-1a of the module make_module last created on this thread (PIPE_FEEDBACK 2)
|
|
|
|
// Hash of a pipeline table key, 8 bytes per step: a draw hashes its whole GKey, and byte-wise FNV over it cost more
|
|
// than the lookup. Only the in-memory table uses it; the key file keeps fnv.
|
|
static uint64_t key_hash(const void *k, size_t n) { return nvn_hm_hash(k, (uint32_t)n); }
|
|
|
|
// The cache file lives next to nx2navi.exe, so every build dir (the main build and each private fork) has its own:
|
|
// forks that shared build\v121's cache broke each other's runs (2026-10-22).
|
|
static void cache_path_init(void) {
|
|
const char *cfg_path = cfg_get("NX2NAVI_PIPELINE_CACHE");
|
|
if (cfg_path && *cfg_path && strlen(cfg_path) < sizeof P.cache_path) {
|
|
strcpy(P.cache_path, cfg_path);
|
|
return;
|
|
}
|
|
P.cache_path[0] = 0;
|
|
char exe[PLAT_PATH_MAX * 2];
|
|
int n = plat_exe_path(exe, sizeof exe);
|
|
if (!n || n >= sizeof exe) return;
|
|
char *cut = plat_last_sep(exe);
|
|
if (!cut) return;
|
|
*cut = 0;
|
|
snprintf(P.cache_path, sizeof P.cache_path, "%s/pipeline-cache.bin", exe);
|
|
}
|
|
|
|
// ---- pipeline binaries (VK_KHR_pipeline_binary; NX2NAVI_PIPE_BINARIES, default 0 while it is being tested; 1 on)
|
|
// pipeline-cache.bin does not carry compiled pipelines from one executable to another on NVIDIA 621.01: two
|
|
// byte-identical exes under different names, every compute module hash-identical, and the second compiled 317 of the
|
|
// 323 compute pipelines the first had read from the file (2026-11-09). A pipeline binary made from one exe's pipeline
|
|
// creates the same pipeline in another in 0.5 ms instead of 26 ms. So where the driver has the extension, every
|
|
// pipeline, stage library and compute pipeline is created through pipe_create: its driver pipeline key
|
|
// (vkGetPipelineKeyKHR, computed from the create info) is looked up in pipeline-binaries.bin next to nx2navi.exe and
|
|
// the pipeline is made from the stored binaries; a key not there is compiled with its data captured, and its binaries
|
|
// are appended to the file (autosave, every 2 s). pipeline-cache.bin is then neither read nor written: the driver's
|
|
// own disk cache serves an unchanged exe, and the binaries serve a renamed or updated one.
|
|
// Each binary is stored once under its own binary key: on NVIDIA 621.01 nearly every graphics pipeline has, besides
|
|
// its own binaries of 20-200 KB, one 2 MiB binary that is byte-identical across pipelines (2,506 copies in a 4.7 GB
|
|
// first-version file).
|
|
// File: header {"NX2NPBIN", version, the driver's global key size and bytes} (another global key: another driver or
|
|
// GPU, so the file starts empty); then records {u32 bytes after this field, u32 type, u32 key size, 32 key bytes,
|
|
// payload}: type PB_BLOB, a binary under its binary key, payload its data; type PB_PIPE, a pipeline under its pipeline
|
|
// key, payload {u32 count, count x {u32 key size, 32 key bytes}} naming its binaries. Only an index of the records
|
|
// stays in memory; a pipeline's records are read from the file when it is created. NX2NAVI_PIPE_BINARIES_MAX_MB
|
|
// (default 1024, 0 no limit): a larger file starts empty.
|
|
#define PB_VERSION 2
|
|
// 64-bit file offsets: long is 32 bits on Windows, and a file over 2 GB read as cut short.
|
|
#ifdef _WIN32
|
|
#define pb_seek _fseeki64
|
|
#define pb_tell _ftelli64
|
|
#else
|
|
#define pb_seek fseeko
|
|
#define pb_tell ftello
|
|
#endif
|
|
#define PB_MAX_BIN 8
|
|
#define PB_KEY VK_MAX_PIPELINE_BINARY_KEY_SIZE_KHR
|
|
enum { PB_BLOB = 1, PB_PIPE = 2 };
|
|
#define PB_HEAD (4 + 4 + PB_KEY) // a record's bytes after its size field and before its payload
|
|
typedef struct { char magic[8]; uint32_t version, gkey_size; uint8_t gkey[PB_KEY]; } PbHeader;
|
|
typedef struct { uint64_t h, off; uint32_t size; } PbSlot; // off: file offset of the record's size field
|
|
static struct {
|
|
int on;
|
|
PFN_vkCreatePipelineBinariesKHR create;
|
|
PFN_vkDestroyPipelineBinaryKHR destroy;
|
|
PFN_vkGetPipelineKeyKHR key;
|
|
PFN_vkGetPipelineBinaryDataKHR data;
|
|
PFN_vkReleaseCapturedPipelineDataKHR release;
|
|
char path[PLAT_PATH_MAX * 2 + 32];
|
|
PlatMutex lock; // the index, the pending records and rf
|
|
FILE *rf; // reads records
|
|
PbSlot *slot;
|
|
uint32_t cap, n, pipes, blobs;
|
|
uint8_t *pend; // records not written yet; they follow the file's end in this order
|
|
size_t pend_n, pend_cap;
|
|
uint64_t file_size; // bytes on disk
|
|
int no_capture[3]; // kind (pipe_create) whose create refused the capture flag
|
|
uint64_t from_file, load_fail, compiled, captured, capture_fail, blob_new, blob_shared;
|
|
double from_ms, compile_ms;
|
|
} PB = {.lock = PLAT_MUTEX_INIT};
|
|
static __thread int pb_last; // 1: the last pipe_create on this thread made its pipeline from binaries
|
|
static __thread int cc_hit; // compute_create's last pipeline on this thread: 1 the pipeline cache held it
|
|
// (P.cache: read from pipeline-cache.bin, or built into it earlier in this
|
|
// run) or it was made from pipeline binaries, 0 not, -1 the driver did not say
|
|
|
|
static uint64_t pb_hash(uint32_t type, const uint8_t *key, uint32_t n) {
|
|
uint64_t h = fnv(fnv(FNV0, &type, 4), key, n);
|
|
return h ? h : 1;
|
|
}
|
|
|
|
static PbSlot *pb_find(uint64_t h) { // caller holds PB.lock
|
|
if (!PB.cap) return NULL;
|
|
for (uint32_t j = (uint32_t)h & (PB.cap - 1);; j = (j + 1) & (PB.cap - 1)) {
|
|
if (!PB.slot[j].h) return NULL;
|
|
if (PB.slot[j].h == h) return &PB.slot[j];
|
|
}
|
|
}
|
|
|
|
// Enters a record (a hash match is taken as the key: 64 bits). 0 when one with the key is there already or out of
|
|
// memory. Caller holds PB.lock.
|
|
static int pb_insert(uint64_t h, uint64_t off, uint32_t size) {
|
|
if ((PB.n + 1) * 2 > PB.cap) {
|
|
uint32_t cap = PB.cap ? PB.cap * 2 : 4096;
|
|
PbSlot *t = calloc(cap, sizeof *t);
|
|
if (!t) return 0;
|
|
for (uint32_t i = 0; i < PB.cap; i++) {
|
|
if (!PB.slot[i].h) continue;
|
|
uint32_t j = (uint32_t)PB.slot[i].h & (cap - 1);
|
|
while (t[j].h) j = (j + 1) & (cap - 1);
|
|
t[j] = PB.slot[i];
|
|
}
|
|
free(PB.slot);
|
|
PB.slot = t, PB.cap = cap;
|
|
}
|
|
if (pb_find(h)) return 0;
|
|
uint32_t j = (uint32_t)h & (PB.cap - 1);
|
|
while (PB.slot[j].h) j = (j + 1) & (PB.cap - 1);
|
|
PB.slot[j] = (PbSlot){h, off, size};
|
|
PB.n++;
|
|
return 1;
|
|
}
|
|
|
|
static void pb_rewrite(const PbHeader *hd) { // an empty file with this driver's header
|
|
FILE *f = fopen(PB.path, "wb");
|
|
if (f) fwrite(hd, sizeof *hd, 1, f), fclose(f);
|
|
PB.file_size = f ? sizeof *hd : 0;
|
|
}
|
|
|
|
// Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache).
|
|
static void pb_init(VkCtx *c) {
|
|
if (!c->has_pipeline_binary || !P.cache_path[0]) return;
|
|
#define PBPROC(field, name) (PB.field = (void *)vk_dpa(c->device, name))
|
|
if (!PBPROC(create, "vkCreatePipelineBinariesKHR") || !PBPROC(destroy, "vkDestroyPipelineBinaryKHR") ||
|
|
!PBPROC(key, "vkGetPipelineKeyKHR") || !PBPROC(data, "vkGetPipelineBinaryDataKHR") ||
|
|
!PBPROC(release, "vkReleaseCapturedPipelineDataKHR")) {
|
|
rt_log("vk_pipeline: pipeline binaries: the driver lacks a VK_KHR_pipeline_binary function; using pipeline-cache.bin");
|
|
return;
|
|
}
|
|
#undef PBPROC
|
|
PbHeader want = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, PB_VERSION, 0, {0}};
|
|
VkPipelineBinaryKeyKHR gk = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
|
if (PB.key(c->device, NULL, &gk) != VK_SUCCESS || !gk.keySize || gk.keySize > PB_KEY) {
|
|
rt_log("vk_pipeline: pipeline binaries: no global key from the driver; using pipeline-cache.bin");
|
|
return;
|
|
}
|
|
want.gkey_size = gk.keySize;
|
|
memcpy(want.gkey, gk.key, gk.keySize);
|
|
snprintf(PB.path, sizeof PB.path, "%s", P.cache_path);
|
|
char *cut = plat_last_sep(PB.path);
|
|
snprintf(cut ? cut + 1 : PB.path, sizeof PB.path - (size_t)(cut ? cut + 1 - PB.path : 0), "pipeline-binaries.bin");
|
|
uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPE_BINARIES_MAX_MB"), NULL, 10) << 20;
|
|
FILE *f = fopen(PB.path, "rb");
|
|
PbHeader hd;
|
|
const char *why = NULL;
|
|
uint64_t good = 0, len = 0;
|
|
if (f) {
|
|
pb_seek(f, 0, SEEK_END);
|
|
len = (uint64_t)pb_tell(f);
|
|
pb_seek(f, 0, SEEK_SET);
|
|
if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version";
|
|
else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB";
|
|
else {
|
|
good = sizeof hd;
|
|
for (;;) { // index: per record its type, key and offset
|
|
uint32_t head[3];
|
|
uint8_t key[PB_KEY];
|
|
if (fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || head[0] < PB_HEAD ||
|
|
(head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || good + 4 + head[0] > len)
|
|
break;
|
|
if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) {
|
|
if (head[1] == PB_PIPE) PB.pipes++;
|
|
else PB.blobs++;
|
|
}
|
|
good += 4 + (uint64_t)head[0];
|
|
if (pb_seek(f, (int64_t)good, SEEK_SET)) break;
|
|
}
|
|
if (good != len) why = "cut short (a record half written)";
|
|
}
|
|
fclose(f);
|
|
}
|
|
if (why || !f) { // start empty: the index is dropped and the file rewritten
|
|
if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0);
|
|
free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
|
|
pb_rewrite(&want);
|
|
} else {
|
|
PB.file_size = len;
|
|
}
|
|
PB.rf = fopen(PB.path, "rb");
|
|
PB.on = PB.rf != NULL && PB.file_size >= sizeof want;
|
|
rt_log("vk_pipeline: pipeline binaries %s: %s, %u pipelines, %u binaries, %.1f MB (NX2NAVI_PIPE_BINARIES=0 goes back "
|
|
"to pipeline-cache.bin)", PB.on ? "on" : "off (cannot open the file)", PB.path, PB.pipes, PB.blobs,
|
|
PB.file_size / 1048576.0);
|
|
}
|
|
|
|
// The payload of the record of (type, key), malloc'd (*size bytes), or NULL. Caller holds PB.lock.
|
|
static uint8_t *pb_read(uint32_t type, const uint8_t *key, uint32_t ks, uint32_t *size) {
|
|
PbSlot *s = pb_find(pb_hash(type, key, ks));
|
|
if (!s || s->size < PB_HEAD) return NULL;
|
|
uint32_t n = s->size - PB_HEAD;
|
|
uint8_t *head = malloc(PB_HEAD + (size_t)n), *out = NULL;
|
|
if (!head) return NULL;
|
|
int ok;
|
|
if (s->off >= PB.file_size) { // still pending, at its place in pend
|
|
size_t at = (size_t)(s->off - PB.file_size) + 4;
|
|
ok = at + s->size <= PB.pend_n;
|
|
if (ok) memcpy(head, PB.pend + at, s->size);
|
|
} else {
|
|
ok = !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1;
|
|
}
|
|
uint32_t t, k;
|
|
if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4);
|
|
if (ok && t == type && k == ks && !memcmp(head + 8, key, ks)) { // the key itself, not only its hash
|
|
memmove(head, head + PB_HEAD, n);
|
|
out = head, *size = n;
|
|
} else {
|
|
free(head);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// The binaries stored for pipeline key k, created in bins (*n of them). 0 when none are stored or they fail.
|
|
static int pb_load(const VkPipelineBinaryKeyKHR *k, VkPipelineBinaryKHR bins[PB_MAX_BIN], uint32_t *n) {
|
|
VkPipelineBinaryKeyKHR keys[PB_MAX_BIN];
|
|
VkPipelineBinaryDataKHR data[PB_MAX_BIN];
|
|
uint8_t *blob[PB_MAX_BIN] = {0};
|
|
uint32_t nb = 0, size = 0;
|
|
plat_mutex_lock(&PB.lock);
|
|
uint8_t *pipe = pb_read(PB_PIPE, k->key, k->keySize, &size);
|
|
int ok = pipe && size >= 4;
|
|
if (ok) memcpy(&nb, pipe, 4);
|
|
if (!nb || nb > PB_MAX_BIN || size < 4 + nb * (4 + PB_KEY)) ok = 0;
|
|
for (uint32_t i = 0; ok && i < nb; i++) {
|
|
const uint8_t *e = pipe + 4 + i * (4 + PB_KEY);
|
|
uint32_t ks, ds = 0;
|
|
memcpy(&ks, e, 4);
|
|
keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR, .keySize = ks};
|
|
memcpy(keys[i].key, e + 4, PB_KEY);
|
|
if (ks > PB_KEY || !(blob[i] = pb_read(PB_BLOB, keys[i].key, ks, &ds))) ok = 0;
|
|
else data[i] = (VkPipelineBinaryDataKHR){.dataSize = ds, .pData = blob[i]};
|
|
}
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (ok) {
|
|
VkPipelineBinaryKeysAndDataKHR kd = {.binaryCount = nb, .pPipelineBinaryKeys = keys, .pPipelineBinaryData = data};
|
|
VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pKeysAndDataInfo = &kd};
|
|
VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR,
|
|
.pipelineBinaryCount = nb, .pPipelineBinaries = bins};
|
|
ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS;
|
|
*n = nb;
|
|
}
|
|
for (uint32_t i = 0; i < PB_MAX_BIN; i++) free(blob[i]);
|
|
free(pipe);
|
|
return ok;
|
|
}
|
|
|
|
// Appends one record to pend and the index. Caller holds PB.lock.
|
|
static int pb_put(uint32_t type, const uint8_t *key, uint32_t ks, const void *a, size_t an, const void *b, size_t bn) {
|
|
size_t size = PB_HEAD + an + bn;
|
|
if (size >= (1u << 31)) return 0;
|
|
if (PB.pend_n + 4 + size > PB.pend_cap) {
|
|
size_t cap = PB.pend_cap ? PB.pend_cap * 2 : (size_t)4 << 20;
|
|
while (cap < PB.pend_n + 4 + size) cap *= 2;
|
|
uint8_t *t = realloc(PB.pend, cap);
|
|
if (!t) return 0;
|
|
PB.pend = t, PB.pend_cap = cap;
|
|
}
|
|
if (!pb_insert(pb_hash(type, key, ks), PB.file_size + PB.pend_n, (uint32_t)size)) return 0;
|
|
uint8_t *o = PB.pend + PB.pend_n;
|
|
uint32_t s32 = (uint32_t)size;
|
|
memcpy(o, &s32, 4), memcpy(o + 4, &type, 4), memcpy(o + 8, &ks, 4);
|
|
memset(o + 12, 0, PB_KEY), memcpy(o + 12, key, ks);
|
|
if (an) memcpy(o + 4 + PB_HEAD, a, an);
|
|
if (bn) memcpy(o + 4 + PB_HEAD + an, b, bn);
|
|
PB.pend_n += 4 + size;
|
|
if (type == PB_PIPE) PB.pipes++;
|
|
else PB.blobs++;
|
|
return 1;
|
|
}
|
|
|
|
static void pb_flush(void);
|
|
|
|
// Stores the binaries of pipeline p (created with its data captured) under pipeline key k, then frees the data. A
|
|
// binary whose key is stored already is not stored again.
|
|
static void pb_capture(VkPipeline p, const VkPipelineBinaryKeyKHR *k) {
|
|
VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pipeline = p};
|
|
VkPipelineBinaryKHR bins[PB_MAX_BIN];
|
|
VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR};
|
|
uint32_t nb = 0;
|
|
int ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS && hi.pipelineBinaryCount &&
|
|
hi.pipelineBinaryCount <= PB_MAX_BIN;
|
|
if (ok) {
|
|
hi.pPipelineBinaries = bins;
|
|
ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS;
|
|
nb = ok ? hi.pipelineBinaryCount : 0;
|
|
}
|
|
VkPipelineBinaryKeyKHR keys[PB_MAX_BIN];
|
|
uint8_t list[4 + PB_MAX_BIN * (4 + PB_KEY)] = {0};
|
|
memcpy(list, &nb, 4);
|
|
for (uint32_t i = 0; ok && i < nb; i++) { // each binary's key, and its data unless stored already
|
|
VkPipelineBinaryDataInfoKHR di = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_DATA_INFO_KHR, .pipelineBinary = bins[i]};
|
|
keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
|
size_t ds = 0;
|
|
ok = PB.data(P.device, &di, &keys[i], &ds, NULL) == VK_SUCCESS && keys[i].keySize && keys[i].keySize <= PB_KEY;
|
|
if (!ok) break;
|
|
memcpy(list + 4 + i * (4 + PB_KEY), &keys[i].keySize, 4), memcpy(list + 8 + i * (4 + PB_KEY), keys[i].key, keys[i].keySize);
|
|
plat_mutex_lock(&PB.lock);
|
|
int have = pb_find(pb_hash(PB_BLOB, keys[i].key, keys[i].keySize)) != NULL;
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (have) {
|
|
PB.blob_shared++;
|
|
continue;
|
|
}
|
|
uint8_t *d = malloc(ds ? ds : 1);
|
|
ok = d && PB.data(P.device, &di, &keys[i], &ds, d) == VK_SUCCESS;
|
|
if (ok) {
|
|
plat_mutex_lock(&PB.lock);
|
|
if (pb_put(PB_BLOB, keys[i].key, keys[i].keySize, d, ds, NULL, 0)) PB.blob_new++;
|
|
plat_mutex_unlock(&PB.lock); // a 0 from pb_put: another thread stored the same binary meanwhile
|
|
}
|
|
free(d);
|
|
}
|
|
for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL);
|
|
VkReleaseCapturedPipelineDataInfoKHR ri = {.sType = VK_STRUCTURE_TYPE_RELEASE_CAPTURED_PIPELINE_DATA_INFO_KHR, .pipeline = p};
|
|
PB.release(P.device, &ri, NULL);
|
|
plat_mutex_lock(&PB.lock);
|
|
if (ok && pb_put(PB_PIPE, k->key, k->keySize, list, 4 + nb * (4 + PB_KEY), NULL, 0)) PB.captured++;
|
|
else if (!ok) PB.capture_fail++;
|
|
int full = PB.pend_n >= ((size_t)64 << 20);
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (full) pb_flush(); // a cold boot captures thousands before the autosave thread runs: keep the buffer small
|
|
}
|
|
|
|
// Appends the pending records (the autosave thread every 2 s, and the save at exit).
|
|
static void pb_flush(void) {
|
|
if (!PB.on) return;
|
|
plat_mutex_lock(&PB.lock);
|
|
if (PB.pend_n) {
|
|
FILE *f = fopen(PB.path, "ab");
|
|
int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1;
|
|
if (f) ok = !fclose(f) && ok;
|
|
if (ok) PB.file_size += PB.pend_n;
|
|
else rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory", PB.path);
|
|
if (ok) PB.pend_n = 0;
|
|
}
|
|
plat_mutex_unlock(&PB.lock);
|
|
}
|
|
|
|
static void pb_log(const char *when) {
|
|
if (!PB.on) return;
|
|
plat_mutex_lock(&PB.lock);
|
|
rt_log("vk_pipeline: pipeline binaries %s: %llu made from pipeline-binaries.bin (avg %.2f ms), %llu compiled (avg %.1f "
|
|
"ms), %llu captured, %llu could not be stored, %llu stored ones failed; binaries %llu new, %llu shared; %u pipelines and "
|
|
"%u binaries in the file, %.1f MB", when,
|
|
(unsigned long long)PB.from_file, PB.from_file ? PB.from_ms / PB.from_file : 0.0,
|
|
(unsigned long long)PB.compiled, PB.compiled ? PB.compile_ms / PB.compiled : 0.0,
|
|
(unsigned long long)PB.captured, (unsigned long long)PB.capture_fail, (unsigned long long)PB.load_fail,
|
|
(unsigned long long)PB.blob_new, (unsigned long long)PB.blob_shared, PB.pipes, PB.blobs,
|
|
(PB.file_size + PB.pend_n) / 1048576.0);
|
|
plat_mutex_unlock(&PB.lock);
|
|
}
|
|
|
|
// Creates one pipeline: kind 0 a graphics pipeline, 1 a graphics pipeline library, 2 a compute pipeline; ci its
|
|
// Vk*PipelineCreateInfo. With binaries on: from the stored binaries of its pipeline key when there are any, else
|
|
// compiled with P.cache and its binaries captured. The create info is restored before returning.
|
|
typedef struct { VkStructureType sType; const void *pNext; VkPipelineCreateFlags flags; } PipeCiHead;
|
|
|
|
// Builds into P.cache against reads of its data. Every create with P.cache holds this shared (pipe_create_in); a save
|
|
// holds it exclusively for the copy of the data and only takes it when no build is running. The Vulkan rules allow
|
|
// reading a cache created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT while pipelines are built into
|
|
// it, but Intel's Windows driver (101.8991 on an Arc 140V) crashed inside vkGetPipelineCacheData during the copy,
|
|
// reading through a null entry, with compute and library builds running beside the autosave. So no copy is made while
|
|
// a build runs, on any driver.
|
|
static PlatMutex cache_rw = PLAT_MUTEX_INIT;
|
|
// Set while the save at exit waits for cache_rw: the builds already in the driver finish, and new ones wait in
|
|
// cache_build_enter until the copy is made. Without it the boot's prewarm workers started builds back to back, no moment
|
|
// without one came within the save's 5 s, and the pipelines built in that run were not saved (a window closed during
|
|
// the prewarm, 2026-11-11). The autosave never sets it, so it never holds a build back.
|
|
static volatile int32_t cache_saving_at_exit;
|
|
|
|
// Takes cache_rw shared for a build into P.cache; while the save at exit waits for its copy, first waits for that.
|
|
static void cache_build_enter(void) {
|
|
for (;;) {
|
|
while (__atomic_load_n(&cache_saving_at_exit, __ATOMIC_ACQUIRE)) plat_sleep_ms(1);
|
|
plat_mutex_lock_shared(&cache_rw);
|
|
if (!__atomic_load_n(&cache_saving_at_exit, __ATOMIC_ACQUIRE)) return;
|
|
plat_mutex_unlock_shared(&cache_rw);
|
|
}
|
|
}
|
|
|
|
// A pipeline cache of this thread's own that its creates build into in place of P.cache; VK_NULL_HANDLE (every thread
|
|
// but the background builder of large compute shaders) for P.cache itself. A create into it does not hold cache_rw, so
|
|
// a build of minutes does not keep pipeline-cache.bin from being saved; bg_merge moves its content into P.cache.
|
|
static __thread VkPipelineCache own_cache;
|
|
// 1 on a thread whose creates take no pipeline cache of ours at all (the library warm in play, lw_worker and lw_probe):
|
|
// what it builds is wanted in the driver's own disk cache only. Built into P.cache, the 37,572 stage libraries of a
|
|
// warm grew it by 1.6 GB (2026-11-11: a save of 2026.7 MB in play, +1612 MB since the boot); the file was then over
|
|
// NX2NAVI_PIPELINE_CACHE_MAX_MB, so the next start dropped it whole and compiled the pipelines it had held.
|
|
static __thread int no_cache;
|
|
// New content in P.cache that P.stats does not count, for autosave_once to save: bg_merge's merges, and the compiles
|
|
// of pipelines built for the cache only (lb_compute keep 0).
|
|
static volatile uint32_t cache_merges;
|
|
|
|
// ---- the play cache (NX2NAVI_PIPELINE_CACHE_PLAY, default 1; needs pipelineCreationCacheControl)
|
|
// pipeline-cache.bin holds what the preparation built and what earlier runs made: 300 MB and more on an install. One
|
|
// file can only be written whole, so a single new pipeline in play had the whole of it copied out of the driver and
|
|
// written again (2026-11-11: saves of 358.1 MB for 0.3 MB of new data, and 302 to 306 MB written in each of seven
|
|
// sessions that added about 0.5 MB each). So what a run compiles goes into a second cache, P.play, with a file of its
|
|
// own, pipeline-cache-play.bin beside the first: a create asks P.cache with
|
|
// VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT, which gives the pipeline when that cache or the driver's
|
|
// own has it and VK_PIPELINE_COMPILE_REQUIRED when not, and only then builds into P.play (pipe_create_in). In play
|
|
// only the play file is saved; pipeline-cache.bin is not written.
|
|
// The play file is read back at the next start, into P.play and merged into P.cache in memory (cache_load), so the
|
|
// first create finds its pipelines. Once
|
|
// it has grown to NX2NAVI_PIPELINE_CACHE_PLAY_MB (default 64) it is merged into P.cache at the start, before play: the
|
|
// autosave thread writes pipeline-cache.bin once and removes the play file (cache_save). The preparation modes
|
|
// (--prepare-shaders, --warm-shaders, --prepare-pipelines) merge it at any size and build into P.cache alone, as before.
|
|
// A pipeline the driver's own cache gives to the first create enters P.cache in memory only: it is in a file again
|
|
// when the play file is next folded in, or when the driver no longer has it and it is compiled.
|
|
// Without the device feature, with pipeline binaries on, or with the setting 0, P.play stays VK_NULL_HANDLE and
|
|
// everything goes into P.cache as before.
|
|
static int prep_mode; // a preparation mode (set before ensure_init): no play cache
|
|
|
|
static VkResult pipe_create_one(int kind, void *ci, VkPipeline *pipe, VkPipelineCache cache) {
|
|
return kind == 2 ? vkCreateComputePipelines(P.device, cache, 1, ci, NULL, pipe)
|
|
: vkCreateGraphicsPipelines(P.device, cache, 1, ci, NULL, pipe);
|
|
}
|
|
|
|
static VkResult pipe_create_in(int kind, void *ci, VkPipeline *pipe, VkPipelineCache cache) {
|
|
if (no_cache) cache = VK_NULL_HANDLE;
|
|
int shared = cache && !own_cache;
|
|
if (cache && own_cache) cache = own_cache;
|
|
uint64_t t0 = perf_mark_begin(); // NX2NAVI_TRACE=perf: the create on any thread, its wait for cache_rw included
|
|
if (shared) cache_build_enter();
|
|
VkResult r;
|
|
if (shared && P.play) {
|
|
PipeCiHead *hd = ci;
|
|
VkPipelineCreateFlags flags = hd->flags;
|
|
hd->flags = flags | VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
|
|
r = pipe_create_one(kind, ci, pipe, cache);
|
|
hd->flags = flags;
|
|
if (r == VK_PIPELINE_COMPILE_REQUIRED) {
|
|
// Not in P.cache nor in the driver's own cache: from P.play, or compiled into it. A caller that asked for
|
|
// cached pipelines only (its own FAIL_ON_PIPELINE_COMPILE_REQUIRED) gets the play cache's answer.
|
|
r = pipe_create_one(kind, ci, pipe, P.play);
|
|
if (r == VK_SUCCESS && *pipe) __atomic_fetch_add(&P.play_made, 1, __ATOMIC_RELAXED);
|
|
}
|
|
} else {
|
|
r = pipe_create_one(kind, ci, pipe, cache);
|
|
}
|
|
if (shared) plat_mutex_unlock_shared(&cache_rw);
|
|
perf_mark_end(PERF_M_PIPE, t0, 0);
|
|
return r;
|
|
}
|
|
|
|
// ---- pipelines of a library that draws on our device (FSR 3.1: vk_ffx.cpp vk_ffx_proc_addr)
|
|
// Such a library creates its pipelines itself and passes no pipeline cache, so nothing of ours kept them: every
|
|
// context it made (one per render and output size, and one for frame generation) compiled its passes again unless the
|
|
// driver's own disk cache still had them, which a renamed or updated exe on NVIDIA and a driver without a disk cache
|
|
// do not. vk_ffx.cpp hands the library these creates in the place of vkCreateComputePipelines and
|
|
// vkCreateGraphicsPipelines; a create that names no cache goes through ours as pipe_create_in does: asked of P.cache
|
|
// first without compiling, and compiled into the play cache when neither it nor the driver has the pipeline, so the
|
|
// file a run writes (pipeline-cache-play.bin, or pipeline-cache.bin without a play cache) holds it for the next start.
|
|
// Before the first pipeline of ours (P.ready 0) and with pipeline binaries on there is no cache to give: the create
|
|
// goes to the driver as the library asked. infos: n VkComputePipelineCreateInfo (compute 1) or
|
|
// VkGraphicsPipelineCreateInfo. Any thread.
|
|
static struct { volatile uint64_t made, from_cache, compiled, uncached; } FP;
|
|
|
|
void vk_pipeline_foreign_counts(VkForeignPipes *o) {
|
|
o->made = __atomic_load_n(&FP.made, __ATOMIC_RELAXED), o->from_cache = __atomic_load_n(&FP.from_cache, __ATOMIC_RELAXED);
|
|
o->compiled = __atomic_load_n(&FP.compiled, __ATOMIC_RELAXED), o->uncached = __atomic_load_n(&FP.uncached, __ATOMIC_RELAXED);
|
|
}
|
|
|
|
static VkResult foreign_one(VkDevice dev, int compute, VkPipelineCache cache, uint32_t n, const void *ci,
|
|
const VkAllocationCallbacks *alloc, VkPipeline *out) {
|
|
return compute ? vkCreateComputePipelines(dev, cache, n, ci, alloc, out)
|
|
: vkCreateGraphicsPipelines(dev, cache, n, ci, alloc, out);
|
|
}
|
|
|
|
VkResult vk_pipeline_create_foreign(VkDevice dev, int compute, uint32_t n, const void *infos,
|
|
const VkAllocationCallbacks *alloc, VkPipeline *out) {
|
|
VkPipelineCache cache = __atomic_load_n(&P.ready, __ATOMIC_ACQUIRE) && dev == P.device ? P.cache : VK_NULL_HANDLE;
|
|
if (!cache) {
|
|
VkResult r = foreign_one(dev, compute, VK_NULL_HANDLE, n, infos, alloc, out);
|
|
if (r == VK_SUCCESS) __atomic_fetch_add(&FP.made, n, __ATOMIC_RELAXED), __atomic_fetch_add(&FP.uncached, n, __ATOMIC_RELAXED);
|
|
return r;
|
|
}
|
|
VkResult r = VK_SUCCESS;
|
|
cache_build_enter();
|
|
if (!P.play) { // one cache: whether it held them is not known here; its save looks at the data's size
|
|
r = foreign_one(dev, compute, cache, n, infos, alloc, out);
|
|
if (r == VK_SUCCESS) __atomic_fetch_add(&FP.made, n, __ATOMIC_RELAXED), __atomic_fetch_add(&cache_merges, n, __ATOMIC_RELAXED);
|
|
}
|
|
for (uint32_t i = 0; P.play && i < n; i++) {
|
|
// a copy of the create info: the flags are ours to change, the library's struct is not
|
|
VkComputePipelineCreateInfo cc;
|
|
VkGraphicsPipelineCreateInfo gc;
|
|
void *ci;
|
|
if (compute) cc = ((const VkComputePipelineCreateInfo *)infos)[i], ci = &cc;
|
|
else gc = ((const VkGraphicsPipelineCreateInfo *)infos)[i], ci = &gc;
|
|
PipeCiHead *hd = ci;
|
|
// Flags given in a VkPipelineCreateFlags2CreateInfo replace the struct's own, so the question "without
|
|
// compiling" cannot be put through hd->flags: such a create goes to the play cache at once.
|
|
int flags2 = 0;
|
|
for (const VkBaseInStructure *b = hd->pNext; b && !flags2; b = b->pNext)
|
|
flags2 = b->sType == VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR;
|
|
VkPipelineCreateFlags flags = hd->flags;
|
|
VkResult ri = VK_PIPELINE_COMPILE_REQUIRED;
|
|
out[i] = VK_NULL_HANDLE;
|
|
if (!flags2) {
|
|
hd->flags = flags | VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
|
|
ri = foreign_one(dev, compute, cache, 1, ci, alloc, &out[i]);
|
|
hd->flags = flags;
|
|
if (ri == VK_SUCCESS) __atomic_fetch_add(&FP.from_cache, 1, __ATOMIC_RELAXED);
|
|
}
|
|
if (ri == VK_PIPELINE_COMPILE_REQUIRED) {
|
|
ri = foreign_one(dev, compute, P.play, 1, ci, alloc, &out[i]);
|
|
if (ri == VK_SUCCESS) {
|
|
__atomic_fetch_add(&FP.compiled, 1, __ATOMIC_RELAXED);
|
|
__atomic_fetch_add(&P.play_made, 1, __ATOMIC_RELAXED);
|
|
__atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED); // autosave_once: the play file is due
|
|
}
|
|
}
|
|
if (ri == VK_SUCCESS) __atomic_fetch_add(&FP.made, 1, __ATOMIC_RELAXED);
|
|
else if (r == VK_SUCCESS) r = ri; // the first that failed, as a create of several reports one result
|
|
if (ri != VK_SUCCESS) out[i] = VK_NULL_HANDLE;
|
|
}
|
|
plat_mutex_unlock_shared(&cache_rw);
|
|
return r;
|
|
}
|
|
|
|
static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) {
|
|
PipeCiHead *hd = ci;
|
|
VkResult r;
|
|
pb_last = 0;
|
|
#define PIPE_CREATE(cache) pipe_create_in(kind, ci, pipe, cache)
|
|
if (!PB.on) return PIPE_CREATE(P.cache);
|
|
VkPipelineCreateInfoKHR kci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_INFO_KHR, .pNext = ci};
|
|
VkPipelineBinaryKeyKHR key = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
|
int keyed = PB.key(P.device, &kci, &key) == VK_SUCCESS && key.keySize && key.keySize <= PB_KEY;
|
|
const void *next = hd->pNext;
|
|
if (keyed) {
|
|
VkPipelineBinaryKHR bins[PB_MAX_BIN];
|
|
uint32_t nb = 0;
|
|
uint64_t t0 = plat_now_ns();
|
|
if (pb_load(&key, bins, &nb)) {
|
|
VkPipelineBinaryInfoKHR bi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_INFO_KHR, .pNext = next,
|
|
.binaryCount = nb, .pPipelineBinaries = bins};
|
|
hd->pNext = &bi;
|
|
r = PIPE_CREATE(VK_NULL_HANDLE); // a pipeline made from binaries takes no pipeline cache
|
|
hd->pNext = next;
|
|
for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL);
|
|
plat_mutex_lock(&PB.lock);
|
|
if (r == VK_SUCCESS) PB.from_file++, PB.from_ms += (plat_now_ns() - t0) / 1e6;
|
|
else PB.load_fail++;
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (r == VK_SUCCESS) return pb_last = 1, r;
|
|
}
|
|
}
|
|
int capture = keyed && !PB.no_capture[kind];
|
|
VkPipelineCreateFlags2CreateInfoKHR f2 = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR, .pNext = next,
|
|
.flags = (VkPipelineCreateFlags2KHR)hd->flags | VK_PIPELINE_CREATE_2_CAPTURE_DATA_BIT_KHR};
|
|
uint64_t t0 = plat_now_ns();
|
|
if (capture) hd->pNext = &f2;
|
|
r = PIPE_CREATE(P.cache);
|
|
hd->pNext = next;
|
|
if (capture && r < 0) { // the driver refuses the capture for this kind: create without it from now on
|
|
PB.no_capture[kind] = 1;
|
|
rt_log("vk_pipeline: pipeline binaries: a %s with its data captured failed (%d); such pipelines are not stored",
|
|
kind == 2 ? "compute pipeline" : kind ? "pipeline library" : "graphics pipeline", (int)r);
|
|
capture = 0;
|
|
r = PIPE_CREATE(P.cache);
|
|
}
|
|
#undef PIPE_CREATE
|
|
if (r == VK_SUCCESS && *pipe) {
|
|
plat_mutex_lock(&PB.lock);
|
|
PB.compiled++, PB.compile_ms += (plat_now_ns() - t0) / 1e6;
|
|
plat_mutex_unlock(&PB.lock);
|
|
if (capture) pb_capture(*pipe, &key);
|
|
}
|
|
return r;
|
|
}
|
|
|
|
// The data of the pipeline cache file at path for vkCreatePipelineCache (malloc'd, its size in *size), NULL when the
|
|
// run starts without it, which the log says with the file's size. Cache data whose header names another device or
|
|
// driver is dropped before Vulkan sees it.
|
|
// NX2NAVI_PIPELINE_CACHE_MAX_MB (default 512; 0 no limit): a larger file starts the run empty, and the run's
|
|
// saves replace it. The file only grows: entries of earlier translators and builds stay, and on NVIDIA
|
|
// (621.01) an entry another executable wrote is neither used nor written again, so a renamed or updated exe
|
|
// compiles while the file keeps its size (2026-11-09: 650 MB, 15-18 % of the pipelines read from it). The
|
|
// whole file sits in process memory twice at the boot (this read and the driver's copy) and is copied out
|
|
// at every save of it, which a 16 GB machine near its commit limit cannot spare. The size is checked before the
|
|
// read, so a file that is dropped never takes its size in memory. Sizes are 64-bit all the way
|
|
// (vk_cache_file.h cache_file_read): a file past 2 GB is read with the limit raised or off, and refused by
|
|
// its real size otherwise.
|
|
static int cache_over_cap; // the last file cache_data_read was asked for is over the limit
|
|
static void *cache_data_read(VkCtx *c, const char *path, size_t *size) {
|
|
*size = 0;
|
|
cache_over_cap = 0;
|
|
if (!path[0]) return NULL;
|
|
uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_MAX_MB"), NULL, 10) << 20, len = 0;
|
|
int why = CACHE_FILE_NONE;
|
|
void *data = cache_file_read(path, sizeof(VkPipelineCacheHeaderVersionOne), cap, &len, &why);
|
|
if (!data) {
|
|
if (why == CACHE_FILE_OVER_CAP)
|
|
cache_over_cap = 1,
|
|
rt_log("vk_pipeline: %s holds %.0f MB, over NX2NAVI_PIPELINE_CACHE_MAX_MB %llu: starting empty; it is "
|
|
"replaced when the play file is next folded in, at the next start that finds one", path, len / 1048576.0, (unsigned long long)(cap >> 20));
|
|
else if (why != CACHE_FILE_NONE)
|
|
rt_log("vk_pipeline: %s (%llu bytes, %.1f MB) is not read: %s; starting empty", path,
|
|
(unsigned long long)len, len / 1048576.0, cache_file_why(why));
|
|
return NULL;
|
|
}
|
|
VkPipelineCacheHeaderVersionOne h;
|
|
memcpy(&h, data, sizeof h);
|
|
if (h.headerVersion != VK_PIPELINE_CACHE_HEADER_VERSION_ONE || h.vendorID != c->props.vendorID ||
|
|
h.deviceID != c->props.deviceID || memcmp(h.pipelineCacheUUID, c->props.pipelineCacheUUID, VK_UUID_SIZE)) {
|
|
rt_log("vk_pipeline: %s (%.1f MB) was written by another device or driver; starting empty", path,
|
|
len / 1048576.0);
|
|
free(data);
|
|
return NULL;
|
|
}
|
|
*size = (size_t)len;
|
|
return data;
|
|
}
|
|
|
|
// A pipeline cache from data (may be NULL: empty); VK_NULL_HANDLE when the driver makes none. *kept: the size of
|
|
// the data it holds afterwards. A driver may drop initial data it does not accept without failing the create, which
|
|
// leaves every build of the run compiling while the cache file looks valid.
|
|
static VkPipelineCache cache_make(VkCtx *c, const void *data, size_t size, size_t *kept) {
|
|
VkPipelineCache cache = VK_NULL_HANDLE;
|
|
VkPipelineCacheCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, .initialDataSize = size,
|
|
.pInitialData = size ? data : NULL};
|
|
if (vkCreatePipelineCache(c->device, &ci, NULL, &cache) != VK_SUCCESS) {
|
|
ci.initialDataSize = 0, ci.pInitialData = NULL;
|
|
if (vkCreatePipelineCache(c->device, &ci, NULL, &cache) != VK_SUCCESS) cache = VK_NULL_HANDLE;
|
|
}
|
|
*kept = 0;
|
|
if (cache && vkGetPipelineCacheData(c->device, cache, kept, NULL) != VK_SUCCESS) *kept = 0;
|
|
return cache;
|
|
}
|
|
|
|
static void cache_load(VkCtx *c) {
|
|
cache_path_init();
|
|
pb_init(c);
|
|
if (PB.on) { // the binaries carry compiled pipelines across runs; P.cache stays VK_NULL_HANDLE
|
|
rt_log("vk_pipeline: %s is not read or written while pipeline binaries are on", P.cache_path);
|
|
return;
|
|
}
|
|
size_t size = 0, kept = 0;
|
|
void *data = cache_data_read(c, P.cache_path, &size);
|
|
int base_over = cache_over_cap; // an oversized file is never read again: the play file takes its place at once
|
|
P.cache = cache_make(c, data, size, &kept);
|
|
free(data);
|
|
P.cache_loaded = kept;
|
|
// The file holds this data when the driver took all of it: a save is due only once the cache has grown (cache_save).
|
|
P.saved[0] = size && kept == size ? kept : 0;
|
|
if (size)
|
|
rt_log("vk_pipeline: %s: %.1f MB read, the driver holds %.1f MB of it", P.cache_path, size / 1048576.0,
|
|
kept / 1048576.0);
|
|
if (!P.cache || !P.cache_path[0]) return;
|
|
|
|
// The play cache (see above pipe_create_in).
|
|
snprintf(P.play_path, sizeof P.play_path, "%s", P.cache_path);
|
|
size_t n = strlen(P.play_path);
|
|
if (n > 4 && !strcmp(P.play_path + n - 4, ".bin")) n -= 4;
|
|
snprintf(P.play_path + n, sizeof P.play_path - n, "-play.bin");
|
|
int want = cfg_bool("NX2NAVI_PIPELINE_CACHE_PLAY") && !prep_mode, on = want && P.cache_control;
|
|
size_t psize = 0, pkept = 0;
|
|
void *pdata = cache_data_read(c, P.play_path, &psize);
|
|
VkPipelineCache play = psize || on ? cache_make(c, pdata, psize, &pkept) : VK_NULL_HANDLE;
|
|
free(pdata);
|
|
uint64_t fold_at = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_PLAY_MB"), NULL, 10) << 20;
|
|
int fold = play && psize && (!on || psize >= fold_at || base_over);
|
|
if (fold) {
|
|
// Into P.cache now; the file is written by the first save, which then removes the play file. A run that ends
|
|
// before that save finds both files as they were and merges again.
|
|
VkResult r = vkMergePipelineCaches(c->device, P.cache, 1, &play);
|
|
vkDestroyPipelineCache(c->device, play, NULL);
|
|
play = VK_NULL_HANDLE;
|
|
if (r == VK_SUCCESS) {
|
|
P.fold = 1, P.saved[0] = 0;
|
|
rt_log("vk_pipeline: %s: %.1f MB read, the driver holds %.1f MB of it; merged into the cache of %s, which is "
|
|
"written once now (%s)", P.play_path, psize / 1048576.0, pkept / 1048576.0, P.cache_path,
|
|
prep_mode ? "a preparation builds into that file alone"
|
|
: !want ? "NX2NAVI_PIPELINE_CACHE_PLAY=0"
|
|
: !P.cache_control ? "no pipelineCreationCacheControl on this device"
|
|
: "it reached the size at which it is folded in, before play");
|
|
if (on) play = cache_make(c, NULL, 0, &pkept), psize = 0;
|
|
} else {
|
|
rt_log("vk_pipeline: %s did not merge into the cache of %s (%d); it is left as it is and not used in this run",
|
|
P.play_path, P.cache_path, (int)r);
|
|
on = 0;
|
|
}
|
|
} else if (psize) {
|
|
// Kept as the play cache, so its file goes on holding what earlier runs made, and merged into P.cache in
|
|
// memory, where the first create of pipe_create_in finds its pipelines. Without the merge that create took
|
|
// them from the driver's own cache when it had them, at 8 ms a pipeline against 0.4 ms from ours.
|
|
VkResult r = vkMergePipelineCaches(c->device, P.cache, 1, &play);
|
|
rt_log("vk_pipeline: %s: %.1f MB read, the driver holds %.1f MB of it%s", P.play_path, psize / 1048576.0,
|
|
pkept / 1048576.0, r == VK_SUCCESS ? "" : "; it did not merge into the other cache in memory");
|
|
}
|
|
P.play = on ? play : VK_NULL_HANDLE;
|
|
P.play_loaded = P.play ? pkept : 0;
|
|
// An empty play cache is not written (its data is a header): its size counts as saved, as a whole file's does.
|
|
P.saved[1] = P.play && (!psize || pkept == psize) ? pkept : 0;
|
|
if (P.play)
|
|
rt_log("vk_pipeline: what this run compiles goes into %s; %s is not written in play", P.play_path, P.cache_path);
|
|
else if (want && !P.cache_control)
|
|
rt_log("vk_pipeline: no play cache (no pipelineCreationCacheControl on this device): a new pipeline has %s "
|
|
"written whole", P.cache_path);
|
|
}
|
|
|
|
// The float widths spirv_nan_preserve declares on this device (P.nan_preserve), from NX2NAVI_VK_NAN_PRESERVE: 0 off,
|
|
// 1 on every driver; unset, on every driver except NVIDIA's own, on which the game draws right without it. Only widths
|
|
// the driver can preserve (VkPhysicalDeviceFloatControlsProperties) are declared, and nothing without 32-bit floats.
|
|
static void nan_preserve_init(VkCtx *c) {
|
|
VkPhysicalDeviceDriverProperties drv = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES};
|
|
VkPhysicalDeviceFloatControlsProperties fc = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES,
|
|
.pNext = &drv};
|
|
VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &fc};
|
|
vkGetPhysicalDeviceProperties2(c->phys, &p2);
|
|
const char *e = cfg_get("NX2NAVI_VK_NAN_PRESERVE"); // no fixed default: it depends on the driver, see above
|
|
int set = e && *e, want = set ? strcmp(e, "0") != 0 : drv.driverID != VK_DRIVER_ID_NVIDIA_PROPRIETARY;
|
|
uint32_t can = (fc.shaderSignedZeroInfNanPreserveFloat16 ? NP_16 : 0) |
|
|
(fc.shaderSignedZeroInfNanPreserveFloat32 ? NP_32 : 0) |
|
|
(fc.shaderSignedZeroInfNanPreserveFloat64 ? NP_64 : 0);
|
|
P.nan_preserve = want && (can & NP_32) ? can : 0;
|
|
// NX2NAVI_VK_NAN_BITS: 1 every driver, 0 none; unset: AMD's driver for RDNA 1 and 2 (spirv_nan_bits)
|
|
const char *nb = cfg_get("NX2NAVI_VK_NAN_BITS");
|
|
P.nan_bits = nb && *nb ? strcmp(nb, "0") != 0 : vk_caps.amd_sc;
|
|
if (P.nan_bits)
|
|
rt_log("vk_pipeline: game shaders test for NaN on the bits of the number, not as a float (%s)",
|
|
nb && *nb ? "NX2NAVI_VK_NAN_BITS=1" : "AMD's driver for RDNA 1 and 2");
|
|
if (P.nan_preserve)
|
|
rt_log("vk_pipeline: game shaders keep NaN, infinity and signed zero (SignedZeroInfNanPreserve for %s32%s-bit "
|
|
"floats%s)", can & NP_16 ? "16, " : "", can & NP_64 ? ", 64" : "", set ? ", NX2NAVI_VK_NAN_PRESERVE=1" : "");
|
|
else
|
|
rt_log("vk_pipeline: game shaders without SignedZeroInfNanPreserve (%s)",
|
|
!want ? (set ? "NX2NAVI_VK_NAN_PRESERVE=0" : "NVIDIA's driver") : "the driver cannot keep them in 32-bit floats");
|
|
}
|
|
|
|
static void keys_open(void);
|
|
static void cu_open(void);
|
|
static void prebuild_start(void);
|
|
|
|
static int ensure_init(void) {
|
|
if (P.ready) return 1;
|
|
if (!vk_init()) return 0;
|
|
plat_mutex_lock(&P.lock);
|
|
if (P.ready) {
|
|
plat_mutex_unlock(&P.lock);
|
|
return 1;
|
|
}
|
|
VkCtx *c = vk_ctx();
|
|
P.device = c->device;
|
|
VkPhysicalDevicePushDescriptorPropertiesKHR pd = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR};
|
|
VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &pd};
|
|
vkGetPhysicalDeviceProperties2(c->phys, &p2);
|
|
P.max_push = pd.maxPushDescriptors;
|
|
P.indep = vk_caps.gpl;
|
|
P.gpl = vk_caps.gpl;
|
|
rt_log(P.gpl ? "vk_pipeline: graphics pipeline library on: a draw whose pipeline is building gets a fast-linked one, "
|
|
"the optimised build replaces it when done"
|
|
: "vk_pipeline: graphics pipeline library off: draws wait for or skip pipelines that are building");
|
|
const char *ps = cfg_get("NX2NAVI_PUSH_STAGE"), *vo = cfg_get("NX2NAVI_VS_OUTS");
|
|
P.push_stage = ps && !strcmp(ps, "fs") ? GFX_STAGE_FS : GFX_STAGE_VS;
|
|
P.vs_outs_all = vo && !strcmp(vo, "all");
|
|
if (P.indep) rt_log("vk_pipeline: stage-independent layouts: vertex set 0, fragment set 1, %s set pushed; vertex outputs: %s",
|
|
P.push_stage == GFX_STAGE_VS ? "vertex" : "fragment", P.vs_outs_all ? "all 32 locations" : "union of paired fragment inputs");
|
|
// vk_device.c enables every feature the GPU reports in this chain, so the query tells what is enabled.
|
|
VkPhysicalDeviceExtendedDynamicState3FeaturesEXT e3 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_3_FEATURES_EXT};
|
|
VkPhysicalDeviceProvokingVertexFeaturesEXT pvf = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_FEATURES_EXT,
|
|
.pNext = c->has_eds3 ? &e3 : NULL};
|
|
VkPhysicalDeviceVulkan13Features v13 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES,
|
|
.pNext = c->has_provoking_vertex ? (void *)&pvf : (c->has_eds3 ? (void *)&e3 : NULL)};
|
|
VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &v13};
|
|
vkGetPhysicalDeviceFeatures2(c->phys, &f2);
|
|
P.pv_last = c->has_provoking_vertex && pvf.provokingVertexLast;
|
|
if (!P.pv_last) rt_log("vk_pipeline: provokingVertexLast unavailable: flat outputs come from the first vertex (NVN: last)");
|
|
// Maxwell warps have 32 lanes; the translator assumes it (maxwell.md 3.4). Pin 32 where the driver allows it.
|
|
if (v13.subgroupSizeControl) {
|
|
VkPhysicalDeviceVulkan13Properties p13 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES};
|
|
VkPhysicalDeviceProperties2 q2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &p13};
|
|
vkGetPhysicalDeviceProperties2(c->phys, &q2);
|
|
P.sg32_stages = vk_caps.sg_pin_stages; // vk_caps.c: 32 unless NX2NAVI_VK_SUBGROUP; 0 without the pin
|
|
rt_log("vk_pipeline: subgroup size %u-%u, requiredSubgroupSizeStages %#x, pinning %u on %#x", p13.minSubgroupSize,
|
|
p13.maxSubgroupSize, (unsigned)p13.requiredSubgroupSizeStages, vk_caps.sg_size, (unsigned)P.sg32_stages);
|
|
}
|
|
// A warp reconverges where the guest compiler put it back together; Vulkan guarantees that only with maximal
|
|
// reconvergence (spirv_reconverge). NX2NAVI_VK_RECONVERGE=0 leaves the modules as translated.
|
|
P.reconverge = c->has_maximal_reconvergence && rt_knob("VK_RECONVERGE", "1") == 1;
|
|
rt_log("vk_pipeline: maximal reconvergence for shaders with subgroup ops: %s",
|
|
P.reconverge ? "on" : c->has_maximal_reconvergence ? "off (NX2NAVI_VK_RECONVERGE=0)" : "not supported by the driver");
|
|
nan_preserve_init(c);
|
|
P.f_full_subgroups = v13.computeFullSubgroups;
|
|
P.cache_control = v13.pipelineCreationCacheControl;
|
|
// Pipeline creation feedback is core in Vulkan 1.3; the driver reports per pipeline whether P.cache held it.
|
|
P.feedback = c->props.apiVersion >= VK_API_VERSION_1_3 ? atoi(cfg_value("NX2NAVI_PIPE_FEEDBACK")) : 0;
|
|
P.f_logic_op = f2.features.logicOp;
|
|
P.f_depth_clamp = f2.features.depthClamp;
|
|
P.f_fill_non_solid = f2.features.fillModeNonSolid;
|
|
P.f_dual_src = f2.features.dualSrcBlend;
|
|
#define PROC(field, name) (P.field = (void *)vk_dpa(c->device, name))
|
|
PROC(push, "vkCmdPushDescriptorSetKHR");
|
|
// Descriptor update templates (core 1.1; the push variant comes with VK_KHR_push_descriptor on 1.1): the driver
|
|
// walks a layout prepared when the layout is made instead of parsing write structs every draw.
|
|
// NX2NAVI_DESC_TEMPLATE=0 keeps the write structs.
|
|
if (!cfg_bool("NX2NAVI_DESC_TEMPLATE")) {
|
|
rt_log("vk_pipeline: descriptor update templates off (NX2NAVI_DESC_TEMPLATE=0)");
|
|
} else if (c->props.apiVersion >= VK_API_VERSION_1_1 && PROC(tpl_create, "vkCreateDescriptorUpdateTemplate") &&
|
|
PROC(tpl_destroy, "vkDestroyDescriptorUpdateTemplate") && PROC(tpl_update, "vkUpdateDescriptorSetWithTemplate")) {
|
|
P.tpl = 1;
|
|
if (!PROC(tpl_push, "vkCmdPushDescriptorSetWithTemplateKHR"))
|
|
rt_log("vk_pipeline: vkCmdPushDescriptorSetWithTemplateKHR missing: push sets are written with write structs");
|
|
} else {
|
|
rt_log("vk_pipeline: descriptor update templates unavailable: sets are written with write structs");
|
|
}
|
|
// NX2NAVI_VK_DYN_VERTEX=0/1 forces dynamic vertex input off or on. Default off on two drivers:
|
|
// - Intel's frozen Windows driver for its older integrated GPUs (UHD 630, Ice Lake: Vulkan 1.3.215 and older), which
|
|
// draws nothing with it.
|
|
// - AMD's driver for RDNA 3 and newer (vk_caps.amd_llpc), which compiles a pipeline with dynamic vertex input again
|
|
// at every boot: neither pipeline-cache.bin nor the driver's own disk cache gives it back. Measured on an
|
|
// RX 7900 XT (26.10.1) with NX2NAVI_PIPE_FEEDBACK=2, booting one save twice from an empty cache: the second boot
|
|
// compiled all 622 recorded non-mesh pipelines again at 83 ms each, with the module and state hashes of the first
|
|
// boot (the same with NX2NAVI_GPL=0, so it is not the pipeline libraries or their layouts), while the mesh
|
|
// pipelines, which have no vertex input, came from the file. With the vertex layout in the pipeline the second
|
|
// boot built 787 pipelines in 170 ms and the third 898 of 898 from the file in 233 ms, with no build left for the
|
|
// scene. A player's file of 3,979 records used the whole boot budget (PREWARM_PIPE_MS, 20 s) at every boot; with
|
|
// the vertex layout in the pipeline its third boot built 4,273 pipelines in 0.9 s. The price is more pipelines,
|
|
// one per vertex layout (787 records against 692 for that boot); records written with dynamic vertex input still
|
|
// prebuild, each for the layout of the draw that recorded it. The RDNA 1 and 2 branch (vk_caps.amd_sc) is not
|
|
// measured and keeps dynamic vertex input.
|
|
const char *dv = cfg_get("NX2NAVI_VK_DYN_VERTEX"); // no fixed default: it depends on the GPU, see above
|
|
int intel_legacy = c->props.vendorID == 0x8086 && c->props.apiVersion <= VK_MAKE_API_VERSION(0, 1, 3, 215);
|
|
int dv_on = dv && *dv ? strcmp(dv, "0") != 0 : !intel_legacy && !vk_caps.amd_llpc;
|
|
P.dyn_vi = dv_on && c->has_vertex_input_dynamic_state && PROC(set_vi, "vkCmdSetVertexInputEXT");
|
|
if (!dv_on && c->has_vertex_input_dynamic_state)
|
|
rt_log("vk_pipeline: dynamic vertex input off (%s)",
|
|
dv && *dv ? "NX2NAVI_VK_DYN_VERTEX=0"
|
|
: intel_legacy ? "Intel's older Windows driver draws nothing with it"
|
|
: "AMD's driver compiles pipelines that have it again at every boot");
|
|
if (c->has_eds3) {
|
|
P.dyn_clamp = e3.extendedDynamicState3DepthClampEnable && P.f_depth_clamp && PROC(set_clamp, "vkCmdSetDepthClampEnableEXT");
|
|
P.dyn_poly = e3.extendedDynamicState3PolygonMode && PROC(set_poly, "vkCmdSetPolygonModeEXT");
|
|
P.dyn_blend_en = e3.extendedDynamicState3ColorBlendEnable && PROC(set_blend_en, "vkCmdSetColorBlendEnableEXT");
|
|
P.dyn_blend_eq = e3.extendedDynamicState3ColorBlendEquation && PROC(set_blend_eq, "vkCmdSetColorBlendEquationEXT");
|
|
P.dyn_wmask = e3.extendedDynamicState3ColorWriteMask && PROC(set_wmask, "vkCmdSetColorWriteMaskEXT");
|
|
P.dyn_a2c = e3.extendedDynamicState3AlphaToCoverageEnable && PROC(set_a2c, "vkCmdSetAlphaToCoverageEnableEXT");
|
|
}
|
|
// NX2NAVI_VK_DYN_A2C=0/1 forces dynamic alpha to coverage off or on. Default off on AMD's Windows driver branch for
|
|
// RDNA 2 and older (vk_caps.amd_sc), which applies alpha to coverage while the dynamic state says off: on an RX 590
|
|
// (24.9.1) NX2navi, which never turns it on, drew Link and the NPCs with depth but no colour (black silhouettes) and
|
|
// dithered other surfaces into checkerboards (issue #67, RX 470/570/580). Taken out of the dynamic state, it goes
|
|
// into the pipeline key and the characters draw. With the texture pool marks it stays out of the dynamic state
|
|
// whatever the setting: a marked fragment shader gets early fragment tests only in a pipeline that has alpha to
|
|
// coverage off (spirv_pool_mark), so the pipeline has to know.
|
|
const char *da = cfg_get("NX2NAVI_VK_DYN_A2C"); // no fixed default: it depends on the driver, see above
|
|
int marks = vk_pool_marks_enabled();
|
|
int da_on = !marks && (da && *da ? strcmp(da, "0") != 0 : !vk_caps.amd_sc);
|
|
if (!da_on && P.dyn_a2c) {
|
|
P.dyn_a2c = 0;
|
|
rt_log("vk_pipeline: dynamic alpha to coverage off (%s)", marks ? "the texture pool marks, NX2NAVI_TEX_DISUSE"
|
|
: da && *da ? "NX2NAVI_VK_DYN_A2C=0"
|
|
: "AMD's driver for RDNA 2 and older");
|
|
}
|
|
#undef PROC
|
|
static const VkDynamicState core[] = {
|
|
VK_DYNAMIC_STATE_VIEWPORT_WITH_COUNT, VK_DYNAMIC_STATE_SCISSOR_WITH_COUNT, VK_DYNAMIC_STATE_DEPTH_BIAS,
|
|
VK_DYNAMIC_STATE_BLEND_CONSTANTS, VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK, VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
|
|
VK_DYNAMIC_STATE_STENCIL_REFERENCE, VK_DYNAMIC_STATE_CULL_MODE, VK_DYNAMIC_STATE_FRONT_FACE,
|
|
VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY, VK_DYNAMIC_STATE_DEPTH_TEST_ENABLE, VK_DYNAMIC_STATE_DEPTH_WRITE_ENABLE,
|
|
VK_DYNAMIC_STATE_DEPTH_COMPARE_OP, VK_DYNAMIC_STATE_STENCIL_TEST_ENABLE, VK_DYNAMIC_STATE_STENCIL_OP,
|
|
VK_DYNAMIC_STATE_DEPTH_BIAS_ENABLE, VK_DYNAMIC_STATE_RASTERIZER_DISCARD_ENABLE,
|
|
VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE};
|
|
P.n_dyn = 0;
|
|
for (size_t i = 0; i < sizeof core / sizeof *core; i++) P.dyn[P.n_dyn++] = core[i];
|
|
P.dyn[P.n_dyn++] = P.dyn_vi ? VK_DYNAMIC_STATE_VERTEX_INPUT_EXT : VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE;
|
|
if (P.dyn_clamp) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_DEPTH_CLAMP_ENABLE_EXT;
|
|
if (P.dyn_poly) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_POLYGON_MODE_EXT;
|
|
if (P.dyn_blend_en) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_BLEND_ENABLE_EXT;
|
|
if (P.dyn_blend_eq) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_BLEND_EQUATION_EXT;
|
|
if (P.dyn_wmask) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_COLOR_WRITE_MASK_EXT;
|
|
if (P.dyn_a2c) P.dyn[P.n_dyn++] = VK_DYNAMIC_STATE_ALPHA_TO_COVERAGE_ENABLE_EXT;
|
|
vk_mem_stage_log("device creation");
|
|
cache_load(c);
|
|
vk_mem_stage_log("the pipeline cache load");
|
|
rt_log("vk_pipeline: max push descriptors %u; dynamic vertex input %d, depth clamp %d, polygon mode %d, blend enable %d, "
|
|
"blend equation %d, write mask %d, alpha to coverage %d; cache %s",
|
|
P.max_push, P.dyn_vi, P.dyn_clamp, P.dyn_poly, P.dyn_blend_en, P.dyn_blend_eq, P.dyn_wmask, P.dyn_a2c,
|
|
P.cache_path[0] ? P.cache_path : "<memory only>");
|
|
keys_open();
|
|
cu_open();
|
|
plat_xchg32(&P.ready, 1);
|
|
plat_mutex_unlock(&P.lock);
|
|
prebuild_start();
|
|
return 1;
|
|
}
|
|
|
|
// ---- NVN -> Vulkan mappings
|
|
|
|
static VkBlendFactor blend_factor(unsigned f) {
|
|
switch (f) {
|
|
case 1: return VK_BLEND_FACTOR_ZERO;
|
|
case 2: return VK_BLEND_FACTOR_ONE;
|
|
case 3: return VK_BLEND_FACTOR_SRC_COLOR;
|
|
case 4: return VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR;
|
|
case 5: return VK_BLEND_FACTOR_SRC_ALPHA;
|
|
case 6: return VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
|
|
case 7: return VK_BLEND_FACTOR_DST_ALPHA;
|
|
case 8: return VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA;
|
|
case 9: return VK_BLEND_FACTOR_DST_COLOR;
|
|
case 10: return VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR;
|
|
case 11: return VK_BLEND_FACTOR_SRC_ALPHA_SATURATE;
|
|
case 0x10: return VK_BLEND_FACTOR_SRC1_COLOR;
|
|
case 0x11: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR;
|
|
case 0x12: return VK_BLEND_FACTOR_SRC1_ALPHA;
|
|
case 0x13: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA;
|
|
case 0x61: return VK_BLEND_FACTOR_CONSTANT_COLOR;
|
|
case 0x62: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR;
|
|
case 0x63: return VK_BLEND_FACTOR_CONSTANT_ALPHA;
|
|
case 0x64: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA;
|
|
default: LOG_ONCE("vk_pipeline: unknown NVN blend factor %#x, using ONE", f); return VK_BLEND_FACTOR_ONE;
|
|
}
|
|
}
|
|
|
|
static VkBlendOp blend_op(unsigned e) {
|
|
switch (e) {
|
|
case 2: return VK_BLEND_OP_SUBTRACT;
|
|
case 3: return VK_BLEND_OP_REVERSE_SUBTRACT;
|
|
case 4: return VK_BLEND_OP_MIN;
|
|
case 5: return VK_BLEND_OP_MAX;
|
|
default: return VK_BLEND_OP_ADD;
|
|
}
|
|
}
|
|
|
|
// NVN_DEPTH_FUNC_* / NVN_STENCIL_FUNC_* are NEVER = 1 .. ALWAYS = 8 in VkCompareOp order.
|
|
static VkCompareOp compare_op(unsigned f) { return f >= 1 && f <= 8 ? (VkCompareOp)(f - 1) : VK_COMPARE_OP_ALWAYS; }
|
|
// NVN_STENCIL_OP_* are KEEP = 1 .. DECR_WRAP = 8 in VkStencilOp order.
|
|
static VkStencilOp stencil_op(unsigned o) { return o >= 1 && o <= 8 ? (VkStencilOp)(o - 1) : VK_STENCIL_OP_KEEP; }
|
|
// NVN_POLYGON_MODE_POINT/LINE/FILL = 0/1/2, VkPolygonMode FILL/LINE/POINT = 0/1/2.
|
|
static VkPolygonMode polygon_mode(unsigned m) {
|
|
if (!P.f_fill_non_solid || m > 2) return VK_POLYGON_MODE_FILL;
|
|
return (VkPolygonMode)(2 - m);
|
|
}
|
|
|
|
VkPrimitiveTopology vk_pipeline_topology(int prim) {
|
|
switch (prim) {
|
|
case NVN_DRAW_PRIMITIVE_POINTS: return VK_PRIMITIVE_TOPOLOGY_POINT_LIST;
|
|
case NVN_DRAW_PRIMITIVE_LINES: return VK_PRIMITIVE_TOPOLOGY_LINE_LIST;
|
|
case NVN_DRAW_PRIMITIVE_LINE_LOOP:
|
|
case NVN_DRAW_PRIMITIVE_LINE_STRIP: return VK_PRIMITIVE_TOPOLOGY_LINE_STRIP;
|
|
case NVN_DRAW_PRIMITIVE_TRIANGLE_STRIP:
|
|
case NVN_DRAW_PRIMITIVE_QUAD_STRIP: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; // same triangles, same coverage
|
|
case NVN_DRAW_PRIMITIVE_TRIANGLE_FAN:
|
|
case NVN_DRAW_PRIMITIVE_POLYGON: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN;
|
|
case NVN_DRAW_PRIMITIVE_LINES_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY;
|
|
case NVN_DRAW_PRIMITIVE_LINE_STRIP_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY;
|
|
case NVN_DRAW_PRIMITIVE_TRIANGLES_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY;
|
|
case NVN_DRAW_PRIMITIVE_TRIANGLE_STRIP_ADJACENCY: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY;
|
|
case NVN_DRAW_PRIMITIVE_PATCHES: return VK_PRIMITIVE_TOPOLOGY_PATCH_LIST;
|
|
default: return VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; // TRIANGLES; QUADS after the replay's index rewrite
|
|
}
|
|
}
|
|
|
|
// Topology class (a pipeline's static topology only fixes the class): 0 point, 1 line, 2 triangle, 3 patch.
|
|
static int topology_class(VkPrimitiveTopology t) {
|
|
switch (t) {
|
|
case VK_PRIMITIVE_TOPOLOGY_POINT_LIST: return 0;
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_LIST:
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP:
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY:
|
|
case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY: return 1;
|
|
case VK_PRIMITIVE_TOPOLOGY_PATCH_LIST: return 3;
|
|
default: return 2;
|
|
}
|
|
}
|
|
|
|
int vk_pipeline_tex_type(int target) {
|
|
switch (target) {
|
|
case NVN_TEXTURE_TARGET_1D: return GFX_TEX_1D;
|
|
case NVN_TEXTURE_TARGET_3D: return GFX_TEX_3D;
|
|
case NVN_TEXTURE_TARGET_1D_ARRAY: return GFX_TEX_1D_ARRAY;
|
|
case NVN_TEXTURE_TARGET_2D_ARRAY: return GFX_TEX_2D_ARRAY;
|
|
case NVN_TEXTURE_TARGET_2D_MULTISAMPLE: return GFX_TEX_2D_MS;
|
|
case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY: return GFX_TEX_2D_MS_ARRAY;
|
|
case NVN_TEXTURE_TARGET_CUBEMAP: return GFX_TEX_CUBE;
|
|
case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: return GFX_TEX_CUBE_ARRAY;
|
|
case NVN_TEXTURE_TARGET_BUFFER: return GFX_TEX_BUFFER;
|
|
default: return GFX_TEX_2D; // 2D, RECTANGLE
|
|
}
|
|
}
|
|
|
|
// Numeric type of a colour format as a shader sees it: GFX_TYPE_*.
|
|
static int format_type(VkFormat f) {
|
|
switch (f) {
|
|
case VK_FORMAT_R8_UINT: case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8B8_UINT: case VK_FORMAT_R8G8B8A8_UINT:
|
|
case VK_FORMAT_R16_UINT: case VK_FORMAT_R16G16_UINT: case VK_FORMAT_R16G16B16_UINT: case VK_FORMAT_R16G16B16A16_UINT:
|
|
case VK_FORMAT_R32_UINT: case VK_FORMAT_R32G32_UINT: case VK_FORMAT_R32G32B32_UINT: case VK_FORMAT_R32G32B32A32_UINT:
|
|
case VK_FORMAT_A2B10G10R10_UINT_PACK32: case VK_FORMAT_B8G8R8A8_UINT: case VK_FORMAT_S8_UINT:
|
|
return GFX_TYPE_UINT;
|
|
case VK_FORMAT_R8_SINT: case VK_FORMAT_R8G8_SINT: case VK_FORMAT_R8G8B8_SINT: case VK_FORMAT_R8G8B8A8_SINT:
|
|
case VK_FORMAT_R16_SINT: case VK_FORMAT_R16G16_SINT: case VK_FORMAT_R16G16B16_SINT: case VK_FORMAT_R16G16B16A16_SINT:
|
|
case VK_FORMAT_R32_SINT: case VK_FORMAT_R32G32_SINT: case VK_FORMAT_R32G32B32_SINT: case VK_FORMAT_R32G32B32A32_SINT:
|
|
case VK_FORMAT_A2B10G10R10_SINT_PACK32: case VK_FORMAT_B8G8R8A8_SINT:
|
|
return GFX_TYPE_SINT;
|
|
default: return GFX_TYPE_FLOAT;
|
|
}
|
|
}
|
|
|
|
// Optimal-tiling (o = 1) or buffer (o = 0) features of a core format, cached.
|
|
static VkFormatFeatureFlags format_features(VkFormat f, int optimal) {
|
|
static VkFormatFeatureFlags known[2][256];
|
|
static uint8_t have[2][256];
|
|
VkCtx *c = vk_ctx();
|
|
if (!c) return 0;
|
|
if ((unsigned)f < 256 && have[optimal][f]) return known[optimal][f];
|
|
VkFormatProperties p;
|
|
vkGetPhysicalDeviceFormatProperties(c->phys, f, &p);
|
|
VkFormatFeatureFlags r = optimal ? p.optimalTilingFeatures : p.bufferFeatures;
|
|
if ((unsigned)f < 256) known[optimal][f] = r, have[optimal][f] = 1;
|
|
return r;
|
|
}
|
|
|
|
// Vertex attribute formats by NVNformat.
|
|
typedef struct { VkFormat vk; uint8_t type; } VtxFmt;
|
|
#define F(v) {VK_FORMAT_##v, GFX_TYPE_FLOAT}
|
|
#define U(v) {VK_FORMAT_##v, GFX_TYPE_UINT}
|
|
#define I(v) {VK_FORMAT_##v, GFX_TYPE_SINT}
|
|
static const VtxFmt vtx_formats[0x9B] = {
|
|
[0x01] = F(R8_UNORM), [0x02] = F(R8_SNORM), [0x03] = U(R8_UINT), [0x04] = I(R8_SINT), [0x05] = F(R16_SFLOAT),
|
|
[0x06] = F(R16_UNORM), [0x07] = F(R16_SNORM), [0x08] = U(R16_UINT), [0x09] = I(R16_SINT), [0x0A] = F(R32_SFLOAT),
|
|
[0x0B] = U(R32_UINT), [0x0C] = I(R32_SINT), [0x0D] = F(R8G8_UNORM), [0x0E] = F(R8G8_SNORM), [0x0F] = U(R8G8_UINT),
|
|
[0x10] = I(R8G8_SINT), [0x11] = F(R16G16_SFLOAT), [0x12] = F(R16G16_UNORM), [0x13] = F(R16G16_SNORM),
|
|
[0x14] = U(R16G16_UINT), [0x15] = I(R16G16_SINT), [0x16] = F(R32G32_SFLOAT), [0x17] = U(R32G32_UINT),
|
|
[0x18] = I(R32G32_SINT), [0x19] = F(R8G8B8_UNORM), [0x1A] = F(R8G8B8_SNORM), [0x1B] = U(R8G8B8_UINT),
|
|
[0x1C] = I(R8G8B8_SINT), [0x1D] = F(R16G16B16_SFLOAT), [0x1E] = F(R16G16B16_UNORM), [0x1F] = F(R16G16B16_SNORM),
|
|
[0x20] = U(R16G16B16_UINT), [0x21] = I(R16G16B16_SINT), [0x22] = F(R32G32B32_SFLOAT), [0x23] = U(R32G32B32_UINT),
|
|
[0x24] = I(R32G32B32_SINT), [0x25] = F(R8G8B8A8_UNORM), [0x26] = F(R8G8B8A8_SNORM), [0x27] = U(R8G8B8A8_UINT),
|
|
[0x28] = I(R8G8B8A8_SINT), [0x29] = F(R16G16B16A16_SFLOAT), [0x2A] = F(R16G16B16A16_UNORM),
|
|
[0x2B] = F(R16G16B16A16_SNORM), [0x2C] = U(R16G16B16A16_UINT), [0x2D] = I(R16G16B16A16_SINT),
|
|
[0x2E] = F(R32G32B32A32_SFLOAT), [0x2F] = U(R32G32B32A32_UINT), [0x30] = I(R32G32B32A32_SINT),
|
|
[0x3D] = F(A2B10G10R10_UNORM_PACK32), [0x3E] = U(A2B10G10R10_UINT_PACK32), [0x3F] = F(B10G11R11_UFLOAT_PACK32),
|
|
// *_UI2F / *_I2F: integer data read as float; the 8- and 16-bit ones are Vulkan's SCALED formats
|
|
[0x51] = F(R8_USCALED), [0x52] = F(R8_SSCALED), [0x53] = F(R16_USCALED), [0x54] = F(R16_SSCALED),
|
|
[0x57] = F(R8G8_USCALED), [0x58] = F(R8G8_SSCALED), [0x59] = F(R16G16_USCALED), [0x5A] = F(R16G16_SSCALED),
|
|
[0x5D] = F(R8G8B8_USCALED), [0x5E] = F(R8G8B8_SSCALED), [0x5F] = F(R16G16B16_USCALED), [0x60] = F(R16G16B16_SSCALED),
|
|
[0x63] = F(R8G8B8A8_USCALED), [0x64] = F(R8G8B8A8_SSCALED), [0x65] = F(R16G16B16A16_USCALED),
|
|
[0x66] = F(R16G16B16A16_SSCALED),
|
|
// 32-bit ones have no SCALED format: read as integers (values are not converted; logged when used)
|
|
[0x55] = U(R32_UINT), [0x56] = I(R32_SINT), [0x5B] = U(R32G32_UINT), [0x5C] = I(R32G32_SINT),
|
|
[0x61] = U(R32G32B32_UINT), [0x62] = I(R32G32B32_SINT), [0x67] = U(R32G32B32A32_UINT), [0x68] = I(R32G32B32A32_SINT),
|
|
[0x69] = F(A2B10G10R10_SNORM_PACK32), [0x6A] = I(A2B10G10R10_SINT_PACK32), [0x6B] = F(A2B10G10R10_USCALED_PACK32),
|
|
[0x6C] = F(A2B10G10R10_SSCALED_PACK32), [0x9A] = F(B8G8R8A8_UNORM),
|
|
};
|
|
#undef F
|
|
#undef U
|
|
#undef I
|
|
|
|
// 3-component 8/16-bit formats are not vertex formats on every GPU: read 4 components (LAS vertex_format_supported).
|
|
static VkFormat widen(VkFormat f) {
|
|
switch (f) {
|
|
case VK_FORMAT_R8G8B8_UNORM: return VK_FORMAT_R8G8B8A8_UNORM;
|
|
case VK_FORMAT_R8G8B8_SNORM: return VK_FORMAT_R8G8B8A8_SNORM;
|
|
case VK_FORMAT_R8G8B8_UINT: return VK_FORMAT_R8G8B8A8_UINT;
|
|
case VK_FORMAT_R8G8B8_SINT: return VK_FORMAT_R8G8B8A8_SINT;
|
|
case VK_FORMAT_R8G8B8_USCALED: return VK_FORMAT_R8G8B8A8_USCALED;
|
|
case VK_FORMAT_R8G8B8_SSCALED: return VK_FORMAT_R8G8B8A8_SSCALED;
|
|
case VK_FORMAT_R16G16B16_SFLOAT: return VK_FORMAT_R16G16B16A16_SFLOAT;
|
|
case VK_FORMAT_R16G16B16_UNORM: return VK_FORMAT_R16G16B16A16_UNORM;
|
|
case VK_FORMAT_R16G16B16_SNORM: return VK_FORMAT_R16G16B16A16_SNORM;
|
|
case VK_FORMAT_R16G16B16_UINT: return VK_FORMAT_R16G16B16A16_UINT;
|
|
case VK_FORMAT_R16G16B16_SINT: return VK_FORMAT_R16G16B16A16_SINT;
|
|
case VK_FORMAT_R16G16B16_USCALED: return VK_FORMAT_R16G16B16A16_USCALED;
|
|
case VK_FORMAT_R16G16B16_SSCALED: return VK_FORMAT_R16G16B16A16_SSCALED;
|
|
default: return VK_FORMAT_UNDEFINED;
|
|
}
|
|
}
|
|
|
|
VkFormat vk_vertex_format(int nvn_format, int *type) {
|
|
if (type) *type = GFX_TYPE_FLOAT;
|
|
if (nvn_format <= 0 || nvn_format >= (int)(sizeof vtx_formats / sizeof *vtx_formats)) return VK_FORMAT_UNDEFINED;
|
|
const VtxFmt *v = &vtx_formats[nvn_format];
|
|
if (v->vk == VK_FORMAT_UNDEFINED) return VK_FORMAT_UNDEFINED;
|
|
if (type) *type = v->type;
|
|
if ((nvn_format >= 0x55 && nvn_format <= 0x68) && v->type != GFX_TYPE_FLOAT)
|
|
LOG_ONCE("vk_pipeline: 32-bit *_UI2F/*_I2F vertex formats are read as integers (first: %#x)", nvn_format);
|
|
VkFormat f = v->vk;
|
|
if (vk_ctx() && !(format_features(f, 0) & VK_FORMAT_FEATURE_VERTEX_BUFFER_BIT)) {
|
|
VkFormat w = widen(f);
|
|
if (w == VK_FORMAT_UNDEFINED || !(format_features(w, 0) & VK_FORMAT_FEATURE_VERTEX_BUFFER_BIT)) {
|
|
LOG_ONCE("vk_pipeline: vertex format %d (NVN %#x) unsupported", (int)f, nvn_format);
|
|
return VK_FORMAT_UNDEFINED;
|
|
}
|
|
LOG_ONCE("vk_pipeline: vertex format %d unsupported, read as 4 components (first: NVN %#x)", (int)f, nvn_format);
|
|
f = w;
|
|
}
|
|
return f;
|
|
}
|
|
|
|
// Attachment format of an NVN format (tex_vk_format: the format the images are created with).
|
|
static VkFormat target_format(int nvn_format, int *depth, int *stencil) {
|
|
TexVkFormat t;
|
|
if (depth) *depth = 0;
|
|
if (stencil) *stencil = 0;
|
|
if (!nvn_format || !tex_vk_format(nvn_format, &t)) return VK_FORMAT_UNDEFINED;
|
|
if (depth) *depth = t.depth;
|
|
if (stencil) *stencil = t.stencil;
|
|
return t.vk;
|
|
}
|
|
|
|
void vk_pipe_state_defaults(VkPipeState *s) {
|
|
memset(s, 0, sizeof *s);
|
|
s->samples = 1;
|
|
s->prim = NVN_DRAW_PRIMITIVE_TRIANGLES;
|
|
for (int i = 0; i < 8; i++)
|
|
s->blend[i] = (blend_st){(uint64_t)i, NVN_BLEND_FUNC_ONE, NVN_BLEND_FUNC_ZERO, NVN_BLEND_FUNC_ONE,
|
|
NVN_BLEND_FUNC_ZERO, NVN_BLEND_EQUATION_ADD, NVN_BLEND_EQUATION_ADD};
|
|
s->color = (color_st){0, NVN_LOGIC_OP_COPY, NVN_ALPHA_FUNC_ALWAYS};
|
|
s->chmask.mask = 0xffffffffu;
|
|
s->ms.enable = 1;
|
|
s->poly = (poly_st){NVN_FACE_NONE, NVN_FRONT_FACE_CCW, NVN_POLYGON_MODE_FILL, 0};
|
|
s->ds = (ds_st){0, 1, NVN_DEPTH_FUNC_LESS, 0, NVN_STENCIL_FUNC_ALWAYS, NVN_STENCIL_FUNC_ALWAYS,
|
|
NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP,
|
|
NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP, NVN_STENCIL_OP_KEEP};
|
|
}
|
|
|
|
void vk_pipeline_answers(const VkPipeState *s, GfxShaderAnswers *vs, GfxShaderAnswers *fs) {
|
|
if (vs)
|
|
for (int i = 0; i < 16; i++) {
|
|
va_st a;
|
|
memcpy(&a, &s->vattr[i], sizeof a);
|
|
int type = GFX_TYPE_FLOAT;
|
|
if (a.fmt) vk_vertex_format((int)a.fmt, &type);
|
|
vs->attr_type[i] = (uint8_t)type;
|
|
}
|
|
if (fs)
|
|
for (int i = 0; i < 8; i++) {
|
|
VkFormat f = target_format(s->color_fmt[i], NULL, NULL);
|
|
fs->frag_out_type[i] = (uint8_t)(f == VK_FORMAT_UNDEFINED ? GFX_TYPE_FLOAT : format_type(f));
|
|
}
|
|
}
|
|
|
|
// ---- vertex input (pipeline state, or vkCmdSetVertexInputEXT)
|
|
|
|
typedef VkVtxInput VtxInput;
|
|
|
|
// An attribute the game left without a format is a fixed one on Maxwell, a single 32-bit component of 0 (an unused
|
|
// attribute is bound as IsFixed 1x32 float): it reads (0, 0, 0, 1), as Vulkan expands a
|
|
// one-component format. NX2NAVI's foliage shaders scale wind sway by 1 - colour.w; shapes without a vertex colour
|
|
// (Brightbloom stem and leaves) must not sway, or they bend away from the parts that have one (the bulb).
|
|
static VkFormat zero_format(int type) {
|
|
return type == GFX_TYPE_UINT ? VK_FORMAT_R32_UINT : type == GFX_TYPE_SINT ? VK_FORMAT_R32_SINT : VK_FORMAT_R32_SFLOAT;
|
|
}
|
|
|
|
// Attributes for the Locations the vertex shader reads; one without a usable NVN attribute reads VK_PIPE_ZERO_BINDING.
|
|
static void vertex_input(const VkPipeState *s, VtxInput *v) {
|
|
GfxShaderIo io[32];
|
|
uint32_t used = gfx_shader_inputs(s->vs, io), streams = 0;
|
|
int zero = 0;
|
|
memset(v, 0, sizeof *v);
|
|
for (int l = 0; l < 32; l++) {
|
|
if (!(used >> l & 1)) continue;
|
|
VkVertexInputAttributeDescription2EXT *a = &v->a[v->n_attr++];
|
|
*a = (VkVertexInputAttributeDescription2EXT){.sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_ATTRIBUTE_DESCRIPTION_2_EXT,
|
|
.location = (uint32_t)l};
|
|
va_st va = {0};
|
|
if (l < 16) memcpy(&va, &s->vattr[l], sizeof va);
|
|
VkFormat f = va.fmt ? vk_vertex_format((int)va.fmt, NULL) : VK_FORMAT_UNDEFINED;
|
|
if (f != VK_FORMAT_UNDEFINED && va.stream < 16) {
|
|
a->binding = va.stream, a->format = f, a->offset = va.offset;
|
|
streams |= 1u << va.stream;
|
|
} else {
|
|
a->binding = VK_PIPE_ZERO_BINDING, a->format = zero_format(io[l].type);
|
|
zero = 1;
|
|
}
|
|
}
|
|
for (uint32_t b = 0; b < 16; b++) {
|
|
if (!(streams >> b & 1)) continue;
|
|
uint32_t div = s->vstream[b].divisor;
|
|
if (div > 1) LOG_ONCE("vk_pipeline: instance divisor %u treated as 1 (no VK_EXT_vertex_attribute_divisor)", div);
|
|
v->b[v->n_bind++] = (VkVertexInputBindingDescription2EXT){
|
|
.sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT, .binding = b, .stride = s->vstream[b].stride,
|
|
.inputRate = div ? VK_VERTEX_INPUT_RATE_INSTANCE : VK_VERTEX_INPUT_RATE_VERTEX, .divisor = 1};
|
|
}
|
|
if (zero)
|
|
v->b[v->n_bind++] = (VkVertexInputBindingDescription2EXT){
|
|
.sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT, .binding = VK_PIPE_ZERO_BINDING,
|
|
.stride = 0, .inputRate = VK_VERTEX_INPUT_RATE_VERTEX, .divisor = 1};
|
|
}
|
|
|
|
int vk_pipeline_vertex_input_dynamic(void) { return ensure_init() && P.dyn_vi; }
|
|
|
|
PFN_vkCmdPushDescriptorSetKHR vk_pipeline_push_fn(void) { return ensure_init() ? P.push : NULL; }
|
|
PFN_vkCmdPushDescriptorSetWithTemplateKHR vk_pipeline_push_tpl_fn(void) { return ensure_init() && P.tpl ? P.tpl_push : NULL; }
|
|
PFN_vkUpdateDescriptorSetWithTemplate vk_pipeline_update_tpl_fn(void) { return ensure_init() && P.tpl ? P.tpl_update : NULL; }
|
|
|
|
int vk_pipeline_set_indep(int on) {
|
|
if (!ensure_init()) return 0;
|
|
plat_mutex_lock(&P.lock);
|
|
int was = P.indep;
|
|
P.indep = on && vk_caps.gpl;
|
|
plat_mutex_unlock(&P.lock);
|
|
return was;
|
|
}
|
|
|
|
// ---- SPIR-V rewrites
|
|
|
|
// Binding decorations b >= 1 become b + off (the fragment stage's share of set 0).
|
|
static uint32_t *spirv_rebind(const uint32_t *w, uint64_t n, uint32_t off) {
|
|
uint32_t *o = malloc(n * 4);
|
|
if (!o) return NULL;
|
|
memcpy(o, w, n * 4);
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = o[i] & 0xffff, len = o[i] >> 16;
|
|
if (!len || i + len > n || op == SPV_OP_FUNCTION) break;
|
|
if (op == SPV_OP_DECORATE && len >= 4 && o[i + 2] == SPV_DEC_BINDING && o[i + 3] >= 1) o[i + 3] += off;
|
|
i += len;
|
|
}
|
|
return o;
|
|
}
|
|
|
|
// DescriptorSet decorations become set (the fragment stage's resources move to set 1 with independent layouts).
|
|
static uint32_t *spirv_set(const uint32_t *w, uint64_t n, uint32_t set) {
|
|
uint32_t *o = malloc(n * 4);
|
|
if (!o) return NULL;
|
|
memcpy(o, w, n * 4);
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = o[i] & 0xffff, len = o[i] >> 16;
|
|
if (!len || i + len > n || op == SPV_OP_FUNCTION) break;
|
|
if (op == SPV_OP_DECORATE && len >= 4 && o[i + 2] == SPV_DEC_DESCRIPTOR_SET) o[i + 3] = set;
|
|
i += len;
|
|
}
|
|
return o;
|
|
}
|
|
|
|
// Dual-source blending (docs/maxwell.md 5.6): SRC1 blend factors read the fragment shader's colour output 1,
|
|
// which Vulkan takes from Location 0, Index 1. The Output variable at Location 1 moves there, as the translator declares it
|
|
// with DualSourceBlendEnable (CodeGen/Spirv/Declarations.cs); the pipeline is keyed by GKey.dsb, so only pipelines
|
|
// that blend with SRC1 get this module. NULL when the module has no Output at Location 1.
|
|
static uint32_t *spirv_dual_source(const uint32_t *w, uint64_t n, uint64_t *out_n) {
|
|
uint32_t outs[64];
|
|
int nout = 0;
|
|
uint64_t at = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n) return NULL;
|
|
if (op == SPV_OP_VARIABLE && len >= 4 && w[i + 3] == SPV_SC_OUTPUT && nout < 64) outs[nout++] = w[i + 2];
|
|
i += len;
|
|
}
|
|
for (uint64_t i = 5; i < n && !at;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (op == SPV_OP_DECORATE && len == 4 && w[i + 2] == SPV_DEC_LOCATION && w[i + 3] == 1)
|
|
for (int k = 0; k < nout; k++)
|
|
if (outs[k] == w[i + 1]) at = i;
|
|
i += len;
|
|
}
|
|
if (!at) return NULL;
|
|
uint32_t *o = malloc((n + 4) * 4);
|
|
if (!o) return NULL;
|
|
memcpy(o, w, (at + 4) * 4);
|
|
o[at + 3] = 0;
|
|
o[at + 4] = 4u << 16 | SPV_OP_DECORATE, o[at + 5] = w[at + 1], o[at + 6] = SPV_DEC_INDEX, o[at + 7] = 1;
|
|
memcpy(o + at + 8, w + at + 4, (n - at - 4) * 4);
|
|
*out_n = n + 4;
|
|
return o;
|
|
}
|
|
|
|
// Adds an Output variable for each Location of `missing` (type and width from io) to a module, declared in the entry
|
|
// point interface and initialised as Maxwell reads an unwritten output: (0, 0, 0, 1), the w as the bits of 1.0f whatever
|
|
// the type (attribute memory is untyped); fewer than 4 components are all zero. Existing scalar, vector and pointer types
|
|
// are reused. NULL when the module does not have the expected layout.
|
|
static uint32_t *spirv_add_outputs(const uint32_t *w, uint64_t n, uint32_t missing, const GfxShaderIo io[32],
|
|
uint64_t *out_n) {
|
|
uint64_t ep_at = 0, ann_end = 0, fn_at = 0;
|
|
uint32_t bound = w[3], scalar[3] = {0, 0, 0}; // GFX_TYPE_FLOAT, SINT, UINT
|
|
struct { uint32_t id, comp, count; } vec[64];
|
|
struct { uint32_t id, type; } ptr[64];
|
|
int nvec = 0, nptr = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n) return NULL;
|
|
const uint32_t *a = w + i + 1;
|
|
if (op == SPV_OP_ENTRY_POINT && !ep_at) ep_at = i;
|
|
else if (op == SPV_OP_DECORATE || op == 72 || op == 73 || op == 74 || op == 75 || op == 332 || op == 5632 || op == 5633)
|
|
ann_end = i + len;
|
|
else if (op == SPV_OP_TYPE_FLOAT && len >= 3 && a[1] == 32) scalar[GFX_TYPE_FLOAT] = a[0];
|
|
else if (op == SPV_OP_TYPE_INT && len >= 4 && a[1] == 32) scalar[a[2] ? GFX_TYPE_SINT : GFX_TYPE_UINT] = a[0];
|
|
else if (op == SPV_OP_TYPE_VECTOR && len >= 4 && nvec < 64) vec[nvec].id = a[0], vec[nvec].comp = a[1], vec[nvec++].count = a[2];
|
|
else if (op == SPV_OP_TYPE_POINTER && len >= 4 && a[1] == SPV_SC_OUTPUT && nptr < 64) ptr[nptr].id = a[0], ptr[nptr++].type = a[2];
|
|
else if (op == SPV_OP_FUNCTION) { fn_at = i; break; }
|
|
i += len;
|
|
}
|
|
if (!ep_at || !ann_end || !fn_at || ann_end < ep_at || fn_at < ann_end) return NULL;
|
|
uint32_t types[32 * 40], decs[32 * 4], vars[32], zero[3] = {0, 0, 0}, one[3] = {0, 0, 0};
|
|
int nt = 0, nd = 0, nv = 0;
|
|
for (int l = 0; l < 32; l++) {
|
|
if (!(missing >> l & 1)) continue;
|
|
int t = io[l].type <= GFX_TYPE_UINT ? io[l].type : GFX_TYPE_FLOAT;
|
|
uint32_t comps = io[l].comps ? io[l].comps : 4;
|
|
if (!scalar[t]) {
|
|
scalar[t] = bound++;
|
|
if (t == GFX_TYPE_FLOAT) types[nt++] = 3u << 16 | SPV_OP_TYPE_FLOAT, types[nt++] = scalar[t], types[nt++] = 32;
|
|
else types[nt++] = 4u << 16 | SPV_OP_TYPE_INT, types[nt++] = scalar[t], types[nt++] = 32, types[nt++] = t == GFX_TYPE_SINT;
|
|
}
|
|
uint32_t type = scalar[t];
|
|
if (comps > 1) {
|
|
int k = 0;
|
|
while (k < nvec && !(vec[k].comp == type && vec[k].count == comps)) k++;
|
|
if (k == nvec) {
|
|
if (nvec == 64) return NULL;
|
|
vec[nvec].id = bound++, vec[nvec].comp = type, vec[nvec++].count = comps;
|
|
types[nt++] = 4u << 16 | SPV_OP_TYPE_VECTOR, types[nt++] = vec[k].id, types[nt++] = type, types[nt++] = comps;
|
|
}
|
|
type = vec[k].id;
|
|
}
|
|
int k = 0;
|
|
while (k < nptr && ptr[k].type != type) k++;
|
|
if (k == nptr) {
|
|
if (nptr == 64) return NULL;
|
|
ptr[nptr].id = bound++, ptr[nptr++].type = type;
|
|
types[nt++] = 4u << 16 | SPV_OP_TYPE_POINTER, types[nt++] = ptr[k].id, types[nt++] = SPV_SC_OUTPUT, types[nt++] = type;
|
|
}
|
|
uint32_t init = bound++, var = bound++;
|
|
if (comps == 4) {
|
|
if (!zero[t]) {
|
|
zero[t] = bound++, one[t] = bound++;
|
|
types[nt++] = 4u << 16 | SPV_OP_CONSTANT, types[nt++] = scalar[t], types[nt++] = zero[t], types[nt++] = 0;
|
|
types[nt++] = 4u << 16 | SPV_OP_CONSTANT, types[nt++] = scalar[t], types[nt++] = one[t], types[nt++] = 0x3f800000;
|
|
}
|
|
types[nt++] = 7u << 16 | SPV_OP_CONSTANT_COMPOSITE, types[nt++] = type, types[nt++] = init;
|
|
types[nt++] = zero[t], types[nt++] = zero[t], types[nt++] = zero[t], types[nt++] = one[t];
|
|
} else {
|
|
types[nt++] = 3u << 16 | SPV_OP_CONSTANT_NULL, types[nt++] = type, types[nt++] = init;
|
|
}
|
|
types[nt++] = 5u << 16 | SPV_OP_VARIABLE, types[nt++] = ptr[k].id, types[nt++] = var, types[nt++] = SPV_SC_OUTPUT,
|
|
types[nt++] = init;
|
|
decs[nd++] = 4u << 16 | SPV_OP_DECORATE, decs[nd++] = var, decs[nd++] = SPV_DEC_LOCATION, decs[nd++] = (uint32_t)l;
|
|
vars[nv++] = var;
|
|
}
|
|
uint64_t total = n + (uint64_t)(nt + nd + nv);
|
|
uint32_t *o = malloc(total * 4), *p = o;
|
|
if (!o) return NULL;
|
|
uint32_t ep_len = w[ep_at] >> 16;
|
|
memcpy(p, w, 5 * 4), p[3] = bound, p += 5;
|
|
memcpy(p, w + 5, (ep_at - 5) * 4), p += ep_at - 5;
|
|
*p++ = (ep_len + (uint32_t)nv) << 16 | SPV_OP_ENTRY_POINT;
|
|
memcpy(p, w + ep_at + 1, (ep_len - 1) * 4), p += ep_len - 1;
|
|
memcpy(p, vars, (size_t)nv * 4), p += nv;
|
|
memcpy(p, w + ep_at + ep_len, (ann_end - ep_at - ep_len) * 4), p += ann_end - ep_at - ep_len;
|
|
memcpy(p, decs, (size_t)nd * 4), p += nd;
|
|
memcpy(p, w + ann_end, (fn_at - ann_end) * 4), p += fn_at - ann_end;
|
|
memcpy(p, types, (size_t)nt * 4), p += nt;
|
|
memcpy(p, w + fn_at, (n - fn_at) * 4), p += n - fn_at;
|
|
*out_n = (uint64_t)(p - o);
|
|
return o;
|
|
}
|
|
|
|
// The locations a mesh module writes (its Output variables with a Location), and whether it draws points (execution
|
|
// mode OutputPoints). 0 when the module cannot be read.
|
|
static uint32_t spirv_mesh_outputs(const uint32_t *w, uint64_t n, int *points) {
|
|
enum { OP_EXECUTION_MODE = 16, MODE_OUTPUT_POINTS = 27 };
|
|
uint32_t bound = n >= 5 && w[3] < (1u << 22) ? w[3] : 0, mask = 0;
|
|
uint8_t *out = bound ? calloc(bound, 1) : NULL;
|
|
int8_t *loc = bound ? malloc(bound) : NULL;
|
|
*points = 0;
|
|
if (!out || !loc) return free(out), free(loc), 0;
|
|
memset(loc, -1, bound);
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n) break;
|
|
const uint32_t *a = w + i + 1;
|
|
if (op == OP_EXECUTION_MODE && len >= 3 && a[1] == MODE_OUTPUT_POINTS) *points = 1;
|
|
else if (op == SPV_OP_DECORATE && len >= 4 && a[1] == SPV_DEC_LOCATION && a[0] < bound && a[2] < 32) loc[a[0]] = (int8_t)a[2];
|
|
else if (op == SPV_OP_VARIABLE && len >= 4 && a[2] == SPV_SC_OUTPUT && a[1] < bound) out[a[1]] = 1;
|
|
else if (op == SPV_OP_FUNCTION) break;
|
|
i += len;
|
|
}
|
|
for (uint32_t id = 0; id < bound; id++)
|
|
if (out[id] && loc[id] >= 0) mask |= 1u << loc[id];
|
|
free(out), free(loc);
|
|
return mask;
|
|
}
|
|
|
|
// A fragment module whose inputs at the locations of `missing` read what Maxwell gives a fragment input no earlier
|
|
// stage writes: (0, 0, 0, 1), or with `points` the point sprite's coordinate (s, t, 0, 1). Each such Input variable
|
|
// (a float or a float vector) becomes a Private one, the access chains through it Private pointers, its decorations
|
|
// go, and the entry function stores the value into it before anything else. NULL when nothing is to be done or the
|
|
// module does not have the expected layout.
|
|
static uint32_t *spirv_fs_fill_inputs(const uint32_t *w, uint64_t n, uint32_t missing, int points, uint64_t *out_n) {
|
|
enum { SC_INPUT = 1, SC_PRIVATE = 6, OP_EXECUTION_MODE = 16, OP_TYPE_POINTER = 32, OP_LABEL = 248, OP_LOAD = 61,
|
|
OP_STORE = 62, OP_ACCESS_CHAIN = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66, OP_COMPOSITE_CONSTRUCT = 80,
|
|
OP_COMPOSITE_EXTRACT = 81, OP_DECORATE_ID = 332, BUILTIN_POINT_COORD = 16, MAX_FILL = 32, MAX_PRIV = 64 };
|
|
if (!missing || n < 5 || w[3] >= (1u << 22)) return NULL;
|
|
uint32_t old_bound = w[3], bound = old_bound;
|
|
uint32_t *pt_type = calloc(old_bound, 4), *vec_comp = calloc(old_bound, 4), *vec_n = calloc(old_bound, 4);
|
|
int8_t *loc = malloc(old_bound);
|
|
uint8_t *fill = calloc(old_bound, 1); // 1: a filled variable, 2: an access chain through one
|
|
uint32_t *o = NULL;
|
|
if (!pt_type || !vec_comp || !vec_n || !loc || !fill) goto done;
|
|
memset(loc, -1, old_bound);
|
|
uint32_t f32 = 0, v2 = 0, entry = 0;
|
|
uint64_t ep_at = 0, ann_end = 0, fn_at = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n) goto done;
|
|
const uint32_t *a = w + i + 1;
|
|
if (op == SPV_OP_ENTRY_POINT && !ep_at) ep_at = i, entry = a[1];
|
|
else if (op == SPV_OP_DECORATE || op == 72 || op == OP_DECORATE_ID) {
|
|
ann_end = i + len;
|
|
if (op == SPV_OP_DECORATE && len >= 4 && a[1] == SPV_DEC_LOCATION && a[0] < old_bound && a[2] < 32)
|
|
loc[a[0]] = (int8_t)a[2];
|
|
} else if (op == SPV_OP_TYPE_FLOAT && len >= 3 && a[1] == 32) f32 = a[0];
|
|
else if (op == SPV_OP_TYPE_VECTOR && len >= 4 && a[0] < old_bound) {
|
|
vec_comp[a[0]] = a[1], vec_n[a[0]] = a[2];
|
|
if (a[1] == f32 && a[2] == 2) v2 = a[0];
|
|
} else if (op == OP_TYPE_POINTER && len >= 4 && a[0] < old_bound) pt_type[a[0]] = a[2];
|
|
else if (op == SPV_OP_FUNCTION) { fn_at = i; break; }
|
|
i += len;
|
|
}
|
|
if (!ep_at || !ann_end || !fn_at || !f32 || !entry || ann_end > fn_at) goto done;
|
|
// the variables to fill, and the Private pointer types they and the chains through them need (pass 1)
|
|
uint32_t vars[MAX_FILL], vtype[MAX_FILL], nvars = 0, priv[MAX_PRIV][2], npriv = 0; // priv: Input pointer type -> Private
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
const uint32_t *a = w + i + 1;
|
|
uint32_t ptr = 0;
|
|
if (op == SPV_OP_VARIABLE && len >= 4 && a[2] == SC_INPUT && a[1] < old_bound && a[0] < old_bound && i < fn_at) {
|
|
int l = loc[a[1]];
|
|
uint32_t t = pt_type[a[0]];
|
|
int floaty = t == f32 || (t < old_bound && vec_comp[t] == f32 && vec_n[t] >= 2 && vec_n[t] <= 4);
|
|
if (l >= 0 && (missing >> l & 1) && floaty && nvars < MAX_FILL) {
|
|
fill[a[1]] = 1, vars[nvars] = a[1], vtype[nvars++] = t, ptr = a[0];
|
|
}
|
|
} else if ((op == OP_ACCESS_CHAIN || op == OP_IN_BOUNDS_ACCESS_CHAIN) && len >= 4 && a[2] < old_bound && fill[a[2]] &&
|
|
a[1] < old_bound && a[0] < old_bound) {
|
|
fill[a[1]] = 2, ptr = a[0];
|
|
}
|
|
if (ptr) {
|
|
uint32_t k = 0;
|
|
while (k < npriv && priv[k][0] != ptr) k++;
|
|
if (k == npriv) {
|
|
if (npriv == MAX_PRIV) goto done;
|
|
priv[npriv][0] = ptr, priv[npriv++][1] = bound++;
|
|
}
|
|
}
|
|
i += len;
|
|
}
|
|
if (!nvars) goto done;
|
|
// the globals added before the first function: two constants, gl_PointCoord, and the Private pointer types not yet
|
|
// declared (a filled variable's goes right before the variable, below; an access chain's is only used in a function)
|
|
uint32_t extra[MAX_PRIV * 4 + 32], ne = 0, c0 = bound++, c1 = bound++, pc_ptr = 0, pc_var = 0;
|
|
uint8_t declared[MAX_PRIV] = {0};
|
|
extra[ne++] = 4u << 16 | SPV_OP_CONSTANT, extra[ne++] = f32, extra[ne++] = c0, extra[ne++] = 0;
|
|
extra[ne++] = 4u << 16 | SPV_OP_CONSTANT, extra[ne++] = f32, extra[ne++] = c1, extra[ne++] = 0x3f800000;
|
|
if (points) {
|
|
if (!v2) v2 = bound++, extra[ne++] = 4u << 16 | SPV_OP_TYPE_VECTOR, extra[ne++] = v2, extra[ne++] = f32, extra[ne++] = 2;
|
|
pc_ptr = bound++, pc_var = bound++;
|
|
extra[ne++] = 4u << 16 | OP_TYPE_POINTER, extra[ne++] = pc_ptr, extra[ne++] = SC_INPUT, extra[ne++] = v2;
|
|
extra[ne++] = 4u << 16 | SPV_OP_VARIABLE, extra[ne++] = pc_ptr, extra[ne++] = pc_var, extra[ne++] = SC_INPUT;
|
|
}
|
|
o = malloc((n + ne + (uint64_t)npriv * 4 + 16 + (uint64_t)nvars * 12 + 32) * 4);
|
|
if (!o) goto done;
|
|
uint32_t *p = o;
|
|
memcpy(p, w, 5 * 4), p += 5;
|
|
int in_entry = 0, stored = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
const uint32_t *a = w + i + 1;
|
|
if (i == fn_at) {
|
|
for (uint32_t k = 0; k < npriv; k++)
|
|
if (!declared[k]) *p++ = 4u << 16 | OP_TYPE_POINTER, *p++ = priv[k][1], *p++ = SC_PRIVATE, *p++ = pt_type[priv[k][0]];
|
|
memcpy(p, extra, ne * 4), p += ne;
|
|
}
|
|
if (i == ep_at) { // the interface: filled variables out before SPIR-V 1.4 (Private is not listed), gl_PointCoord in
|
|
uint32_t *head = p, name_words = (uint32_t)(strnlen((const char *)(a + 2), (len - 3) * 4) / 4 + 1);
|
|
for (uint32_t k = 0; k < len; k++) {
|
|
uint32_t v = w[i + k];
|
|
if (k >= 3 + name_words && v < old_bound && fill[v] == 1 && w[1] < 0x00010400) continue;
|
|
*p++ = v;
|
|
}
|
|
if (pc_var) *p++ = pc_var;
|
|
*head = (uint32_t)(p - head) << 16 | SPV_OP_ENTRY_POINT;
|
|
i += len;
|
|
continue;
|
|
}
|
|
if ((op == SPV_OP_DECORATE || op == OP_DECORATE_ID) && len >= 3 && a[0] < old_bound && fill[a[0]] == 1) {
|
|
i += len; // Location, Flat and the like belong to Input variables
|
|
if (i == ann_end && pc_var)
|
|
*p++ = 4u << 16 | SPV_OP_DECORATE, *p++ = pc_var, *p++ = SPV_DEC_BUILTIN, *p++ = BUILTIN_POINT_COORD;
|
|
continue;
|
|
}
|
|
if (op == SPV_OP_VARIABLE && len >= 4 && a[1] < old_bound && fill[a[1]] == 1) // its pointer type first
|
|
for (uint32_t k = 0; k < npriv; k++)
|
|
if (priv[k][0] == a[0] && !declared[k])
|
|
declared[k] = 1, *p++ = 4u << 16 | OP_TYPE_POINTER, *p++ = priv[k][1], *p++ = SC_PRIVATE, *p++ = pt_type[priv[k][0]];
|
|
memcpy(p, w + i, len * 4);
|
|
if ((op == SPV_OP_VARIABLE || op == OP_ACCESS_CHAIN || op == OP_IN_BOUNDS_ACCESS_CHAIN) && len >= 4 &&
|
|
a[1] < old_bound && fill[a[1]]) {
|
|
for (uint32_t k = 0; k < npriv; k++)
|
|
if (priv[k][0] == a[0]) p[1] = priv[k][1];
|
|
if (op == SPV_OP_VARIABLE) p[3] = SC_PRIVATE;
|
|
}
|
|
p += len;
|
|
if (i + len == ann_end && pc_var) // gl_PointCoord's BuiltIn after the last decoration
|
|
*p++ = 4u << 16 | SPV_OP_DECORATE, *p++ = pc_var, *p++ = SPV_DEC_BUILTIN, *p++ = BUILTIN_POINT_COORD;
|
|
if (op == SPV_OP_FUNCTION && len >= 5) in_entry = a[1] == entry;
|
|
i += len;
|
|
if (op == OP_LABEL && in_entry && !stored) { // the entry's first block: its OpVariables, then the stores
|
|
while (i < n && (w[i] & 0xffff) == SPV_OP_VARIABLE) memcpy(p, w + i, (w[i] >> 16) * 4), p += w[i] >> 16, i += w[i] >> 16;
|
|
uint32_t sx = c0, sy = c0;
|
|
if (pc_var) {
|
|
uint32_t pc = bound++;
|
|
sx = bound++, sy = bound++;
|
|
*p++ = 4u << 16 | OP_LOAD, *p++ = v2, *p++ = pc, *p++ = pc_var;
|
|
*p++ = 5u << 16 | OP_COMPOSITE_EXTRACT, *p++ = f32, *p++ = sx, *p++ = pc, *p++ = 0;
|
|
*p++ = 5u << 16 | OP_COMPOSITE_EXTRACT, *p++ = f32, *p++ = sy, *p++ = pc, *p++ = 1;
|
|
}
|
|
for (uint32_t k = 0; k < nvars; k++) {
|
|
uint32_t t = vtype[k], c = t == f32 ? 1 : vec_n[t], comp[4] = {sx, sy, c0, c1}, val = comp[0];
|
|
if (c > 1) {
|
|
val = bound++;
|
|
*p++ = (3 + c) << 16 | OP_COMPOSITE_CONSTRUCT, *p++ = t, *p++ = val;
|
|
for (uint32_t q = 0; q < c; q++) *p++ = comp[q]; // x y z w in order: (s, t, 0, 1) or (0, 0, 0, 1)
|
|
}
|
|
*p++ = 3u << 16 | OP_STORE, *p++ = vars[k], *p++ = val;
|
|
}
|
|
stored = 1;
|
|
}
|
|
}
|
|
if (!stored) { free(o), o = NULL; goto done; }
|
|
o[3] = bound;
|
|
*out_n = (uint64_t)(p - o);
|
|
done:
|
|
free(pt_type), free(vec_comp), free(vec_n), free(loc), free(fill);
|
|
return o;
|
|
}
|
|
|
|
// Two fixes for what the translator declares but a shader does not use (NULL: nothing to change):
|
|
// - the fragment shader interlock capability and extension, which the translator adds to every fragment shader when the host
|
|
// reports interlock support (SpirvGenerator.cs) and vk_device.c does not enable; kept when the shader interlocks;
|
|
// - the CullDistance[1] member of gl_PerVertex next to ClipDistance[8] (Declarations.cs CreatePerVertexStructType):
|
|
// 9 distances exceed maxCombinedClipAndCullDistances (8 on the RX 9070). The translator never accesses CullDistance; the
|
|
// member goes when it is the struct's last and no access chain selects it.
|
|
static uint32_t *spirv_fixup(const uint32_t *w, uint64_t n, uint64_t *out_n) {
|
|
enum { OP_EXTENSION = 10, OP_CAPABILITY = 17, OP_MEMBER_NAME = 6, OP_EXECUTION_MODE = 16, OP_TYPE_ARRAY = 28,
|
|
OP_TYPE_STRUCT = 30, OP_CONSTANT = 43, OP_MEMBER_DECORATE = 72, OP_ACCESS_CHAIN = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66,
|
|
OP_BEGIN_INTERLOCK = 5364, OP_END_INTERLOCK = 5365, CAP_SAMPLE_INTERLOCK = 5363, CAP_PIXEL_INTERLOCK = 5378,
|
|
CAP_RATE_INTERLOCK = 5372, BUILTIN_CLIP = 3, BUILTIN_CULL = 4 };
|
|
if (n < 5) return NULL;
|
|
uint32_t bound = w[3] < (1u << 22) ? w[3] : (1u << 22);
|
|
uint32_t *val = calloc(bound, sizeof *val), *elem = calloc(bound, sizeof *elem), *len = calloc(bound, sizeof *len),
|
|
*pointee = calloc(bound, sizeof *pointee), *var_type = calloc(bound, sizeof *var_type);
|
|
uint32_t *o = NULL;
|
|
int interlock_used = 0, interlock_declared = 0, cull_ok = 0;
|
|
uint32_t cull_struct = 0, cull_member = 0, clip_member = ~0u, members = 0, member_types[8] = {0};
|
|
if (!val || !elem || !len || !pointee || !var_type) goto out;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) goto out;
|
|
const uint32_t *a = w + i + 1;
|
|
if (op == OP_BEGIN_INTERLOCK || op == OP_END_INTERLOCK) interlock_used = 1;
|
|
else if (op == OP_EXECUTION_MODE && l >= 3 && a[1] >= 5366 && a[1] <= 5371) interlock_used = 1;
|
|
else if (op == OP_CAPABILITY && l >= 2 && (a[0] == CAP_PIXEL_INTERLOCK || a[0] == CAP_SAMPLE_INTERLOCK || a[0] == CAP_RATE_INTERLOCK))
|
|
interlock_declared = 1;
|
|
else if (op == OP_CONSTANT && l == 4 && a[1] < bound) val[a[1]] = a[2];
|
|
else if (op == OP_TYPE_ARRAY && l >= 4 && a[0] < bound) elem[a[0]] = a[1], len[a[0]] = a[2] < bound ? val[a[2]] : 0;
|
|
else if (op == SPV_OP_TYPE_POINTER && l >= 4 && a[0] < bound) pointee[a[0]] = a[2];
|
|
else if (op == SPV_OP_VARIABLE && l >= 4 && a[1] < bound) var_type[a[1]] = a[0] < bound ? pointee[a[0]] : 0;
|
|
else if (op == OP_MEMBER_DECORATE && l >= 5 && a[2] == SPV_DEC_BUILTIN && a[3] == BUILTIN_CULL) cull_struct = a[0], cull_member = a[1];
|
|
else if (op == OP_MEMBER_DECORATE && l >= 5 && a[2] == SPV_DEC_BUILTIN && a[3] == BUILTIN_CLIP) clip_member = a[1];
|
|
else if (op == OP_TYPE_STRUCT && cull_struct && a[0] == cull_struct && l >= 2) {
|
|
members = l - 2;
|
|
for (uint32_t m = 0; m < members && m < 8; m++) member_types[m] = a[1 + m];
|
|
} else if ((op == OP_ACCESS_CHAIN || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && cull_struct && a[2] < bound) {
|
|
uint32_t t = var_type[a[2]], k = 3;
|
|
if (t < bound && t && elem[t] == cull_struct) k = 4; // arrayed per-vertex block
|
|
if ((t == cull_struct || k == 4) && k + 1 < l && a[k] < bound && val[a[k]] == cull_member) cull_struct = 0;
|
|
}
|
|
i += l;
|
|
}
|
|
VkCtx *c = vk_ctx();
|
|
uint32_t limit = c ? c->props.limits.maxCombinedClipAndCullDistances : 8;
|
|
if (cull_struct && members && members <= 8 && cull_member == members - 1 && clip_member < members) {
|
|
uint32_t clip_t = member_types[clip_member], cull_t = member_types[cull_member];
|
|
uint32_t clip_n = clip_t < bound ? len[clip_t] : 0, cull_n = cull_t < bound ? len[cull_t] : 0;
|
|
cull_ok = clip_n + cull_n > limit && clip_n <= limit;
|
|
}
|
|
int strip = interlock_declared && !interlock_used;
|
|
if (!strip && !cull_ok) goto out;
|
|
o = malloc(n * 4);
|
|
if (!o) goto out;
|
|
uint64_t on = 5;
|
|
memcpy(o, w, 5 * 4);
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
const uint32_t *a = w + i + 1;
|
|
int drop = 0;
|
|
if (strip && op == OP_CAPABILITY && (a[0] == CAP_PIXEL_INTERLOCK || a[0] == CAP_SAMPLE_INTERLOCK || a[0] == CAP_RATE_INTERLOCK)) drop = 1;
|
|
if (strip && op == OP_EXTENSION && l >= 2 && !strncmp((const char *)a, "SPV_EXT_fragment_shader_interlock", (l - 1) * 4)) drop = 1;
|
|
if (cull_ok && (op == OP_MEMBER_DECORATE || op == OP_MEMBER_NAME) && l >= 3 && a[0] == cull_struct && a[1] == cull_member) drop = 1;
|
|
if (!drop) {
|
|
memcpy(o + on, w + i, l * 4);
|
|
if (cull_ok && op == OP_TYPE_STRUCT && a[0] == cull_struct) o[on] = (l - 1) << 16 | op; // last member gone
|
|
on += op == OP_TYPE_STRUCT && cull_ok && a[0] == cull_struct ? l - 1 : l;
|
|
}
|
|
i += l;
|
|
}
|
|
*out_n = on;
|
|
out:
|
|
free(val), free(elem), free(len), free(pointee), free(var_type);
|
|
return o;
|
|
}
|
|
|
|
// True when the module declares a GroupNonUniform* or SubgroupBallot-style capability (lane masks assume 32 lanes).
|
|
static int uses_subgroup_ops(const uint32_t *w, uint64_t words) {
|
|
for (uint64_t i = 5; i < words;) {
|
|
uint32_t op = w[i] & 0xffff, n = w[i] >> 16;
|
|
if (!n || i + n > words) return 0;
|
|
if (op != 17) return 0; // OpCapability; capabilities come first, stop at the first other instruction
|
|
uint32_t cap = w[i + 1];
|
|
if ((cap >= 61 && cap <= 68) || cap == 4423 || cap == 4431) return 1; // NonUniform*, SubgroupBallotKHR, SubgroupVoteKHR
|
|
i += n;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Guest code runs warp-wide operations (ballot, shuffle, vote) where the guest compiler had the warp reconverged, for
|
|
// example right after a loop that each lane leaves at a different iteration. Vulkan only promises the same lanes at the
|
|
// same point with the MaximallyReconvergesKHR execution mode. Without it a driver may run the lanes that left the loop
|
|
// early through the operations on their own, with a ballot or scan that sees only some of the lanes: a mesh program
|
|
// that hands out output slots from one could give two primitives the same slot. This adds the extension and the mode
|
|
// for each entry point of a module that declares subgroup capabilities. NULL: nothing to change (or no memory).
|
|
static uint32_t *spirv_reconverge(const uint32_t *w, uint64_t n, uint64_t *out_n) {
|
|
enum { OP_EXTENSION = 10, OP_EXECUTION_MODE = 16, OP_CAPABILITY = 17, MODE_MAXIMALLY_RECONVERGES = 6023 };
|
|
static const char ext[] = "SPV_KHR_maximal_reconvergence";
|
|
const uint32_t ext_words = (uint32_t)(sizeof ext + 3) / 4; // the string with its terminating zero, padded
|
|
if (n < 5 || !uses_subgroup_ops(w, n)) return NULL;
|
|
uint64_t caps_end = 0, eps_end = 0;
|
|
uint32_t eps[16], neps = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) return NULL;
|
|
if (op == OP_CAPABILITY) caps_end = i + l;
|
|
else if (op == SPV_OP_ENTRY_POINT && l >= 3) {
|
|
if (neps == 16) return NULL;
|
|
eps[neps++] = w[i + 2], eps_end = i + l;
|
|
} else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == MODE_MAXIMALLY_RECONVERGES) return NULL;
|
|
else if (op == SPV_OP_FUNCTION) break;
|
|
i += l;
|
|
}
|
|
if (!caps_end || !neps || eps_end < caps_end) return NULL;
|
|
uint64_t on = n + 1 + ext_words + 3ull * neps;
|
|
uint32_t *o = malloc(on * 4);
|
|
if (!o) return NULL;
|
|
uint64_t p = 0;
|
|
memcpy(o, w, caps_end * 4), p = caps_end;
|
|
o[p++] = (1 + ext_words) << 16 | OP_EXTENSION;
|
|
memset(o + p, 0, ext_words * 4);
|
|
memcpy(o + p, ext, sizeof ext), p += ext_words;
|
|
memcpy(o + p, w + caps_end, (eps_end - caps_end) * 4), p += eps_end - caps_end;
|
|
for (uint32_t k = 0; k < neps; k++) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = eps[k], o[p++] = MODE_MAXIMALLY_RECONVERGES;
|
|
memcpy(o + p, w + eps_end, (n - eps_end) * 4), p += n - eps_end;
|
|
*out_n = p;
|
|
return o;
|
|
}
|
|
|
|
// Maxwell computes in IEEE 754, and the game's shaders rely on it: its compiler writes an unordered comparison as
|
|
// `x > c || isnan(x)`, a normalisation of a zero-length vector gives an infinity that a later comparison sorts out, and
|
|
// integer masks such as 0xffffffff travel through float moves and selects. The translator's own guards are isnan
|
|
// tests as well (fragment outputs, min, max and clamp). Without the SignedZeroInfNanPreserve execution mode, SPIR-V
|
|
// lets the driver assume that no float is NaN or infinite, and AMD's compilers act on it: on an RX 9070 XT a select
|
|
// between 1 and the bits 0xffffffff, converted to float as a signed integer, gave 2^32 instead of -1 and blew the
|
|
// shadow casters up into huge triangles (issue #156); RX 6000 cards drew Navi, and Link's skin next to her, with black
|
|
// and white blocks (#41, #110); and AMD GPUs drew Navi's sparkles but not her orb, whose light sits at her own centre.
|
|
// NVIDIA's driver keeps the IEEE results without the mode. This adds the capability, the extension (the translator
|
|
// writes SPIR-V 1.3) and the mode for each entry point: 32-bit floats always, 16-bit and 64-bit ones where the module
|
|
// declares Float16 or Float64 and `widths` (NP_*) allows them. NULL: nothing to change (the module has the mode, or no
|
|
// 32-bit width is allowed, or no memory).
|
|
static uint32_t *spirv_nan_preserve(const uint32_t *w, uint64_t n, uint32_t widths, uint64_t *out_n) {
|
|
enum { OP_EXTENSION = 10, OP_EXECUTION_MODE = 16, OP_CAPABILITY = 17, CAP_FLOAT16 = 9, CAP_FLOAT64 = 10,
|
|
CAP_PRESERVE = 4466, MODE_PRESERVE = 4461 };
|
|
static const char ext[] = "SPV_KHR_float_controls";
|
|
const uint32_t ext_words = (uint32_t)(sizeof ext + 3) / 4; // the string with its terminating zero, padded
|
|
if (n < 5 || !(widths & NP_32)) return NULL;
|
|
uint64_t caps_end = 0, eps_end = 0;
|
|
uint32_t eps[16], neps = 0, use = NP_32;
|
|
int has_cap = 0, has_ext = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) return NULL;
|
|
if (op == OP_CAPABILITY && l >= 2) {
|
|
caps_end = i + l;
|
|
if (w[i + 1] == CAP_FLOAT16) use |= NP_16;
|
|
else if (w[i + 1] == CAP_FLOAT64) use |= NP_64;
|
|
else if (w[i + 1] == CAP_PRESERVE) has_cap = 1;
|
|
} else if (op == OP_EXTENSION && (l - 1) * 4 >= sizeof ext && !memcmp(w + i + 1, ext, sizeof ext)) {
|
|
has_ext = 1;
|
|
} else if (op == SPV_OP_ENTRY_POINT && l >= 3) {
|
|
if (neps == 16) return NULL;
|
|
eps[neps++] = w[i + 2], eps_end = i + l;
|
|
} else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == MODE_PRESERVE) {
|
|
return NULL;
|
|
} else if (op == SPV_OP_FUNCTION) {
|
|
break;
|
|
}
|
|
i += l;
|
|
}
|
|
if (!caps_end || !neps || eps_end < caps_end) return NULL;
|
|
use &= widths;
|
|
uint32_t nw = 0, width[3];
|
|
if (use & NP_16) width[nw++] = 16;
|
|
width[nw++] = 32;
|
|
if (use & NP_64) width[nw++] = 64;
|
|
uint64_t on = n + (has_cap ? 0 : 2) + (has_ext ? 0 : 1 + ext_words) + 4ull * nw * neps;
|
|
uint32_t *o = malloc(on * 4);
|
|
if (!o) return NULL;
|
|
uint64_t p = 0;
|
|
memcpy(o, w, caps_end * 4), p = caps_end;
|
|
if (!has_cap) o[p++] = 2u << 16 | OP_CAPABILITY, o[p++] = CAP_PRESERVE;
|
|
if (!has_ext) {
|
|
o[p++] = (1 + ext_words) << 16 | OP_EXTENSION;
|
|
memset(o + p, 0, ext_words * 4);
|
|
memcpy(o + p, ext, sizeof ext), p += ext_words;
|
|
}
|
|
memcpy(o + p, w + caps_end, (eps_end - caps_end) * 4), p += eps_end - caps_end;
|
|
for (uint32_t k = 0; k < neps; k++)
|
|
for (uint32_t j = 0; j < nw; j++)
|
|
o[p++] = 4u << 16 | OP_EXECUTION_MODE, o[p++] = eps[k], o[p++] = MODE_PRESERVE, o[p++] = width[j];
|
|
memcpy(o + p, w + eps_end, (n - eps_end) * 4), p += n - eps_end;
|
|
*out_n = p;
|
|
return o;
|
|
}
|
|
|
|
// NaN tests that no compiler can drop (P.nan_bits). The game's shaders and the translator's own guards test for NaN with
|
|
// OpIsNan: 48,717 of an install's 54,959 translated modules, 26 tests each on average (the game's unordered
|
|
// comparisons, and the guard that keeps NaN out of every float colour output, patch 0027, so that one NaN texel does not
|
|
// spread through bloom and blur into blocks of black and white). SignedZeroInfNanPreserve asks the driver to keep NaN,
|
|
// but whether a compiler that reasons about floats keeps an isnan through every optimisation is up to it. The reports
|
|
// of black and white blocks, on Navi, on Link next to her and on bright surfaces seen from inside a house, come from
|
|
// RX 5000 and 6000 cards (#41, #110, #117), whose Windows driver compiles with "AMD proprietary shader compiler"
|
|
// (vk_caps.amd_sc); RDNA 3 and 4 (LLPC) draw the same scenes clean. It is not known whether that compiler keeps the
|
|
// mode: on it each float32 test becomes integer arithmetic on the bits, which means the same under any float
|
|
// assumption:
|
|
// isnan(x) -> (bitcast<uint>(x) & 0x7fffffff) > 0x7f800000
|
|
// for scalars and vectors of 32-bit floats; tests of 16- and 64-bit floats stay. A float32 select with a constant
|
|
// operand whose bits are a NaN (an integer mask such as 0xffffffff that the translator types as float) becomes a
|
|
// select of the bits (issue #156: on an RX 9070 XT without the mode such a select between 1 and -1 gave 2^32).
|
|
// NULL: nothing to change (or no memory); *tests and *selects count what changed.
|
|
static uint32_t *spirv_nan_bits(const uint32_t *w, uint64_t n, uint64_t *out_n, uint32_t *tests, uint32_t *selects) {
|
|
enum { OP_UNDEF = 1, OP_TYPE_BOOL = 20, OP_SPEC_CONSTANT = 50, OP_SPEC_CONSTANT_COMPOSITE = 51, OP_STORE = 62,
|
|
OP_BITCAST = 124, OP_IS_NAN = 156, OP_SELECT = 169, OP_UGREATER = 172, OP_BITWISE_AND = 199 };
|
|
*tests = *selects = 0;
|
|
if (n < 5 || w[3] >= (1u << 22)) return NULL;
|
|
uint32_t bound = w[3];
|
|
uint32_t *rtype = calloc(bound, sizeof *rtype), *vec_comp = calloc(bound, sizeof *vec_comp);
|
|
uint8_t *f32 = calloc(bound, 1), *vec_n = calloc(bound, 1), *nan_const = calloc(bound, 1);
|
|
uint32_t *o = NULL;
|
|
if (!rtype || !vec_comp || !f32 || !vec_n || !nan_const) goto out;
|
|
uint32_t tu = 0, uvec[5] = {0}; // uint32 and its vectors by component count
|
|
uint64_t fn_at = 0;
|
|
int vec_used[5] = {0};
|
|
// Ids that are not results of the instruction's second word, inside functions: first assignment wins anyway, as
|
|
// a definition comes before its uses (OpPhi operands aside, which this never reads).
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) goto out;
|
|
const uint32_t *a = w + i + 1;
|
|
if (!fn_at) {
|
|
if (op == SPV_OP_FUNCTION) fn_at = i;
|
|
else if (op == SPV_OP_TYPE_FLOAT && l >= 3 && a[0] < bound && a[1] == 32) f32[a[0]] = 1;
|
|
else if (op == SPV_OP_TYPE_INT && l >= 4 && a[1] == 32 && a[2] == 0 && !tu) tu = a[0];
|
|
else if (op == SPV_OP_TYPE_VECTOR && l >= 4 && a[0] < bound && a[2] >= 2 && a[2] <= 4) {
|
|
vec_comp[a[0]] = a[1], vec_n[a[0]] = (uint8_t)a[2];
|
|
if (tu && a[1] == tu && !uvec[a[2]]) uvec[a[2]] = a[0];
|
|
} else if ((op == SPV_OP_CONSTANT || op == SPV_OP_CONSTANT_COMPOSITE || op == SPV_OP_CONSTANT_NULL ||
|
|
op == OP_UNDEF || op == SPV_OP_VARIABLE || op == OP_SPEC_CONSTANT ||
|
|
op == OP_SPEC_CONSTANT_COMPOSITE) && l >= 3 && a[1] < bound) {
|
|
rtype[a[1]] = a[0];
|
|
if (op == SPV_OP_CONSTANT && l == 4 && a[0] < bound && f32[a[0]] && (a[2] & 0x7fffffffu) > 0x7f800000u)
|
|
nan_const[a[1]] = 1;
|
|
}
|
|
}
|
|
if (fn_at) {
|
|
int no_result = op == OP_STORE || op == 63 || op == 64 || op == 8 || op == 317 || (op >= 246 && op <= 255) ||
|
|
op == 99 || op == 224 || op == 225 || op == 228 || op == 56 || op == 218 || op == 219 ||
|
|
op == 5294 || op == 5295 || op == 4416 || op == 5380 || op == 5364 || op == 5365;
|
|
if (!no_result && op != SPV_OP_FUNCTION && l >= 3 && a[1] < bound && !rtype[a[1]]) rtype[a[1]] = a[0];
|
|
if (op == OP_IS_NAN && l == 4 && a[2] < bound) {
|
|
uint32_t t = rtype[a[2]];
|
|
if (t < bound && f32[t]) ++*tests;
|
|
else if (t < bound && vec_n[t] && vec_comp[t] < bound && f32[vec_comp[t]]) ++*tests, vec_used[vec_n[t]] = 1;
|
|
} else if (op == OP_SELECT && l == 6 && a[0] < bound && f32[a[0]] && a[3] < bound && a[4] < bound &&
|
|
(nan_const[a[3]] || nan_const[a[4]])) {
|
|
++*selects;
|
|
}
|
|
}
|
|
i += l;
|
|
}
|
|
if (!fn_at || (!*tests && !*selects)) goto out;
|
|
// the declarations go right before the first function: uint32 and its vectors where missing, then the constants
|
|
uint32_t next = bound, decl[64], nd = 0;
|
|
if (!tu) tu = next++, decl[nd++] = 4u << 16 | SPV_OP_TYPE_INT, decl[nd++] = tu, decl[nd++] = 32, decl[nd++] = 0;
|
|
uint32_t c_mag = next++, c_inf = next++, cv_mag[5] = {0}, cv_inf[5] = {0};
|
|
for (int k = 2; k <= 4; k++)
|
|
if (vec_used[k] && !uvec[k])
|
|
uvec[k] = next++, decl[nd++] = 4u << 16 | SPV_OP_TYPE_VECTOR, decl[nd++] = uvec[k], decl[nd++] = tu, decl[nd++] = (uint32_t)k;
|
|
decl[nd++] = 4u << 16 | SPV_OP_CONSTANT, decl[nd++] = tu, decl[nd++] = c_mag, decl[nd++] = 0x7fffffffu;
|
|
decl[nd++] = 4u << 16 | SPV_OP_CONSTANT, decl[nd++] = tu, decl[nd++] = c_inf, decl[nd++] = 0x7f800000u;
|
|
for (int k = 2; k <= 4; k++) {
|
|
if (!vec_used[k]) continue;
|
|
cv_mag[k] = next++, cv_inf[k] = next++;
|
|
decl[nd++] = (uint32_t)(3 + k) << 16 | SPV_OP_CONSTANT_COMPOSITE, decl[nd++] = uvec[k], decl[nd++] = cv_mag[k];
|
|
for (int j = 0; j < k; j++) decl[nd++] = c_mag;
|
|
decl[nd++] = (uint32_t)(3 + k) << 16 | SPV_OP_CONSTANT_COMPOSITE, decl[nd++] = uvec[k], decl[nd++] = cv_inf[k];
|
|
for (int j = 0; j < k; j++) decl[nd++] = c_inf;
|
|
}
|
|
uint64_t cap = n + nd + 10ull * *tests + 12ull * *selects;
|
|
o = malloc(cap * 4);
|
|
if (!o) goto out;
|
|
memcpy(o, w, fn_at * 4);
|
|
uint64_t p = fn_at;
|
|
memcpy(o + p, decl, nd * 4), p += nd;
|
|
for (uint64_t i = fn_at; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
const uint32_t *a = w + i + 1;
|
|
uint32_t t = op == OP_IS_NAN && l == 4 && a[2] < bound ? rtype[a[2]] : 0;
|
|
int k = t && t < bound ? (f32[t] ? 1 : vec_n[t] && vec_comp[t] < bound && f32[vec_comp[t]] ? vec_n[t] : 0) : 0;
|
|
if (k) {
|
|
uint32_t ut = k == 1 ? tu : uvec[k], mag = k == 1 ? c_mag : cv_mag[k], inf = k == 1 ? c_inf : cv_inf[k];
|
|
uint32_t bits = next++, low = next++;
|
|
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = ut, o[p++] = bits, o[p++] = a[2];
|
|
o[p++] = 5u << 16 | OP_BITWISE_AND, o[p++] = ut, o[p++] = low, o[p++] = bits, o[p++] = mag;
|
|
o[p++] = 5u << 16 | OP_UGREATER, o[p++] = a[0], o[p++] = a[1], o[p++] = low, o[p++] = inf;
|
|
} else if (op == OP_SELECT && l == 6 && a[0] < bound && f32[a[0]] && a[3] < bound && a[4] < bound &&
|
|
(nan_const[a[3]] || nan_const[a[4]])) {
|
|
uint32_t x = next++, y = next++, s = next++;
|
|
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = tu, o[p++] = x, o[p++] = a[3];
|
|
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = tu, o[p++] = y, o[p++] = a[4];
|
|
o[p++] = 6u << 16 | OP_SELECT, o[p++] = tu, o[p++] = s, o[p++] = a[2], o[p++] = x, o[p++] = y;
|
|
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = a[0], o[p++] = a[1], o[p++] = s;
|
|
} else {
|
|
memcpy(o + p, w + i, l * 4), p += l;
|
|
}
|
|
i += l;
|
|
}
|
|
o[3] = next;
|
|
*out_n = p;
|
|
out:
|
|
free(rtype), free(vec_comp), free(f32), free(vec_n), free(nan_const);
|
|
return o;
|
|
}
|
|
|
|
static int spirv_mark_list_test(const char *list); // texture pool marks, below
|
|
|
|
// --selftest (main.c): spirv_nan_preserve on a small fragment module, without a device. Returns the number of failures.
|
|
int vk_pipeline_spirv_selftest(void) {
|
|
enum { CAP = 2u << 16 | 17, MODE = 4u << 16 | 16 };
|
|
static const uint32_t head[] = {0x07230203, 0x00010300, 0, 5, 0, CAP, 1, CAP, 10}; // Shader, Float64
|
|
static const uint32_t body[] = {
|
|
3u << 16 | 14, 0, 1, // OpMemoryModel Logical GLSL450
|
|
5u << 16 | 15, 4, 1, 0x6e69616d, 0, // OpEntryPoint Fragment %1 "main"
|
|
3u << 16 | 16, 1, 7, // OpExecutionMode %1 OriginUpperLeft
|
|
2u << 16 | 19, 2, // %2 = OpTypeVoid
|
|
3u << 16 | 33, 3, 2, // %3 = OpTypeFunction %2
|
|
5u << 16 | 54, 2, 1, 0, 3, // %1 = OpFunction %2 None %3
|
|
2u << 16 | 248, 4, 1u << 16 | 253, 1u << 16 | 56}; // OpLabel, OpReturn, OpFunctionEnd
|
|
enum { NH = sizeof head / 4, NB = sizeof body / 4, EP_END = 8 }; // EP_END: body words up to the entry point's end
|
|
uint32_t in[NH + NB], want[NH + NB + 32];
|
|
memcpy(in, head, sizeof head), memcpy(in + NH, body, sizeof body);
|
|
int failures = 0;
|
|
// widths: what the device allows; f64: the module declares Float64; w64: a 64-bit mode is expected
|
|
for (int t = 0; t < 3; t++) {
|
|
uint32_t widths = t == 0 ? NP_16 | NP_32 | NP_64 : NP_32, n = NH + NB, p = 0;
|
|
int f64 = t != 2, w64 = t == 0;
|
|
if (!f64) in[NH - 1] = 0; // Matrix in place of Float64
|
|
memcpy(want, in, NH * 4), p = NH;
|
|
want[p++] = CAP, want[p++] = 4466;
|
|
want[p++] = 7u << 16 | 10;
|
|
memset(want + p, 0, 24), memcpy(want + p, "SPV_KHR_float_controls", 23), p += 6;
|
|
memcpy(want + p, body, EP_END * 4), p += EP_END;
|
|
want[p++] = MODE, want[p++] = 1, want[p++] = 4461, want[p++] = 32;
|
|
if (w64) want[p++] = MODE, want[p++] = 1, want[p++] = 4461, want[p++] = 64;
|
|
memcpy(want + p, body + EP_END, (NB - EP_END) * 4), p += NB - EP_END;
|
|
uint64_t on = 0, again_n = 0;
|
|
uint32_t *o = spirv_nan_preserve(in, n, widths, &on);
|
|
uint32_t *again = o ? spirv_nan_preserve(o, on, widths, &again_n) : NULL;
|
|
if (!o || on != p || memcmp(o, want, p * 4) || again) {
|
|
printf("spirv: SignedZeroInfNanPreserve rewrite, case %d: wrong output\n", t);
|
|
failures++;
|
|
}
|
|
free(o), free(again);
|
|
}
|
|
if (spirv_nan_preserve(in, NH + NB, NP_16 | NP_64, &(uint64_t){0})) {
|
|
printf("spirv: SignedZeroInfNanPreserve rewrite: a module changed without the 32-bit width\n");
|
|
failures++;
|
|
}
|
|
printf("spirv: SignedZeroInfNanPreserve rewrite: %s\n", failures ? "FAIL" : "ok");
|
|
{ // spirv_nan_bits: a scalar and a vec2 isnan, a select on a NaN constant; no uint32 type in the module
|
|
static const uint32_t m[] = {
|
|
0x07230203, 0x00010300, 0, 20, 0,
|
|
2u << 16 | 17, 1, // OpCapability Shader
|
|
3u << 16 | 14, 0, 1, // OpMemoryModel Logical GLSL450
|
|
5u << 16 | 15, 5, 1, 0x6e69616d, 0, // OpEntryPoint GLCompute %1 "main"
|
|
2u << 16 | 19, 2, // %2 void
|
|
3u << 16 | 33, 3, 2, // %3 fn
|
|
3u << 16 | 22, 4, 32, // %4 float
|
|
2u << 16 | 20, 5, // %5 bool
|
|
4u << 16 | 23, 6, 4, 2, // %6 vec2
|
|
4u << 16 | 23, 7, 5, 2, // %7 bvec2
|
|
4u << 16 | 43, 4, 8, 0xffffffffu, // %8 = NaN bits
|
|
4u << 16 | 43, 4, 9, 0x3f800000u, // %9 = 1.0
|
|
5u << 16 | 44, 6, 10, 8, 9, // %10 = vec2(%8, %9)
|
|
3u << 16 | 41, 5, 11, // %11 = OpConstantTrue
|
|
5u << 16 | 54, 2, 1, 0, 3, // %1 = OpFunction
|
|
2u << 16 | 248, 12, // OpLabel
|
|
4u << 16 | 156, 5, 13, 9, // %13 = OpIsNan bool %9
|
|
4u << 16 | 156, 7, 14, 10, // %14 = OpIsNan bvec2 %10
|
|
6u << 16 | 169, 4, 15, 11, 9, 8, // %15 = OpSelect float %11 %9 %8
|
|
1u << 16 | 253, 1u << 16 | 56}; // OpReturn, OpFunctionEnd
|
|
uint64_t on = 0;
|
|
uint32_t tests = 0, sels = 0, isnan_left = 0, ugt = 0, int32 = 0, bitcasts = 0;
|
|
uint32_t *o = spirv_nan_bits(m, sizeof m / 4, &on, &tests, &sels);
|
|
for (uint64_t i = 5; o && i < on;) {
|
|
uint32_t op = o[i] & 0xffff, l = o[i] >> 16;
|
|
if (!l) break;
|
|
isnan_left += op == 156, ugt += op == 172, int32 += op == 21 && o[i + 2] == 32, bitcasts += op == 124;
|
|
i += l;
|
|
}
|
|
int bad = !o || tests != 2 || sels != 1 || isnan_left || ugt != 2 || int32 != 1 || bitcasts != 5 || o[3] <= 20;
|
|
printf("spirv: integer NaN tests rewrite: %s\n", bad ? "FAIL" : "ok");
|
|
failures += bad;
|
|
free(o);
|
|
}
|
|
const char *list = cfg_get("NX2NAVI_SPV_MARK_TEST");
|
|
if (list && *list) failures += spirv_mark_list_test(list);
|
|
return failures;
|
|
}
|
|
|
|
// Chain {size 32} into a stage create info when the stage uses subgroup ops and the driver allows the pin.
|
|
static void pin_subgroup32(VkPipelineShaderStageCreateInfo *st, VkPipelineShaderStageRequiredSubgroupSizeCreateInfo *rq,
|
|
const uint32_t *w, uint64_t words) {
|
|
if (!(P.sg32_stages & st->stage) || !w || !uses_subgroup_ops(w, words)) return;
|
|
*rq = (VkPipelineShaderStageRequiredSubgroupSizeCreateInfo){
|
|
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO, .requiredSubgroupSize = vk_caps.sg_size};
|
|
st->pNext = rq;
|
|
}
|
|
|
|
// The x of a compute module's LocalSize (OpExecutionMode 17), 0 when it has none.
|
|
static uint32_t local_size_x(const uint32_t *w, uint64_t words) {
|
|
for (uint64_t i = 5; w && i < words;) {
|
|
uint32_t op = w[i] & 0xffff, n = w[i] >> 16;
|
|
if (!n || i + n > words) return 0;
|
|
if (op == 16 && n >= 6 && w[i + 2] == 17) return w[i + 3];
|
|
if (op == 54) return 0; // OpFunction: the execution modes are all before it
|
|
i += n;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Pool arrays (the translator's bindless texture and sampler arrays of VK_POOL_TEX / VK_POOL_SAMP elements, found by
|
|
// their types) move to set 2, at the binding of their view kind (vk.h VK_POOL_KINDS) or VK_POOL_KINDS for samplers;
|
|
// the kinds used are noted in vk_pool_kinds_used so the replay fills them. NULL: nothing to move (or no memory).
|
|
static uint32_t *spirv_pool_rebind(const uint32_t *w, uint64_t n) {
|
|
enum { OP_TYPE_IMAGE = 25, OP_TYPE_SAMPLER = 26, OP_TYPE_ARRAY_ = 28 };
|
|
uint32_t bound = n >= 5 && w[3] < (1u << 22) ? w[3] : 0;
|
|
if (!bound) return NULL;
|
|
int32_t *kind = malloc(bound * sizeof *kind); // per id: image kind, 100 sampler, -1 other (types, then variables)
|
|
uint32_t *cval = calloc(bound, sizeof *cval);
|
|
if (!kind || !cval) return free(kind), free(cval), NULL;
|
|
memset(kind, 0xff, bound * sizeof *kind);
|
|
// Pool variables and their binding decorations are module declarations, all before the first function.
|
|
int any = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n || op == SPV_OP_FUNCTION) break;
|
|
if (op == OP_TYPE_IMAGE && len >= 9 && w[i + 1] < bound && w[i + 7] == 1) { // sampled image
|
|
uint32_t dim = w[i + 3], depth = w[i + 4], arrayed = w[i + 5], ms = w[i + 6];
|
|
if (dim <= 3 && !ms) kind[w[i + 1]] = (int32_t)((depth == 1 ? 8 : 0) + dim * 2 + (arrayed ? 1 : 0));
|
|
} else if (op == OP_TYPE_SAMPLER && len >= 2 && w[i + 1] < bound) {
|
|
kind[w[i + 1]] = 100;
|
|
} else if (op == SPV_OP_CONSTANT && len >= 4 && w[i + 2] < bound) {
|
|
cval[w[i + 2]] = w[i + 3];
|
|
} else if (op == OP_TYPE_ARRAY_ && len >= 4 && w[i + 1] < bound && w[i + 2] < bound && w[i + 3] < bound) {
|
|
int32_t k = kind[w[i + 2]];
|
|
uint32_t want = k == 100 ? VK_POOL_SAMP : VK_POOL_TEX;
|
|
kind[w[i + 1]] = k >= 0 && cval[w[i + 3]] == want ? k + 1000 : -1; // + 1000: a pool array
|
|
} else if (op == SPV_OP_TYPE_POINTER && len >= 4 && w[i + 1] < bound && w[i + 3] < bound) {
|
|
kind[w[i + 1]] = kind[w[i + 3]] >= 1000 ? kind[w[i + 3]] : -1;
|
|
} else if (op == SPV_OP_VARIABLE && len >= 4 && w[i + 1] < bound && w[i + 2] < bound) {
|
|
kind[w[i + 2]] = kind[w[i + 1]] >= 1000 ? kind[w[i + 1]] + 1000 : -1; // + 2000: a pool variable
|
|
if (kind[w[i + 2]] >= 2000) any = 1;
|
|
}
|
|
i += len;
|
|
}
|
|
uint32_t *o = NULL;
|
|
if (any && (o = malloc(n * 4))) {
|
|
memcpy(o, w, n * 4);
|
|
uint32_t used = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = o[i] & 0xffff, len = o[i] >> 16;
|
|
if (!len || i + len > n || op == SPV_OP_FUNCTION) break;
|
|
if (op == SPV_OP_DECORATE && len >= 4 && o[i + 1] < bound && kind[o[i + 1]] >= 2000) {
|
|
int32_t k = kind[o[i + 1]] - 2000;
|
|
if (o[i + 2] == SPV_DEC_DESCRIPTOR_SET) o[i + 3] = 2;
|
|
else if (o[i + 2] == SPV_DEC_BINDING) o[i + 3] = k == 100 ? VK_POOL_KINDS : (uint32_t)k;
|
|
if (k != 100) used |= 1u << k;
|
|
}
|
|
i += len;
|
|
}
|
|
__atomic_fetch_or(&vk_pool_kinds_used, used, __ATOMIC_RELAXED);
|
|
}
|
|
free(kind), free(cval);
|
|
return o;
|
|
}
|
|
|
|
// ---- texture pool marks (NX2NAVI_TEX_DISUSE, default on; vk_texture.c "disuse" uses them)
|
|
// Ocarina samples nearly every texture through the bindless pool: a shader reads a handle from memory and indexes the
|
|
// pool array with it, so the CPU never learns which of the registered textures a frame samples, and every registered
|
|
// texture counts as in use (vk_texture.c idle). The game keeps the textures of the areas it has loaded registered, so
|
|
// none of them could go, and their memory, which the driver charges to the system's commit, grew to 6 GiB in a session.
|
|
// This rewrite gives each module that indexes the texture pool a storage buffer at set 2, binding VK_POOL_MARK_BINDING
|
|
// (a uint per pool slot, vk_exec_draw.c "pool marks") and, after every access chain into a pool texture array, a store
|
|
// of 1 into the slot it indexes. The stores of one warp to one slot are one memory transaction. The replay copies the
|
|
// buffer out and clears it every few submissions; slots marked mean textures sampled. A module that indexes the pool
|
|
// and cannot be marked (SPIR-V before 1.3 or after 1.6, a stage whose stores the device does not allow, no memory) is
|
|
// counted in vk_pool_marks_missed, which keeps vk_texture.c from evicting by disuse: an unmarked shader could sample a
|
|
// texture the marks call unused. make_module logs the first few by stage, version, size, hash and reason.
|
|
volatile int32_t vk_pool_marks_missed;
|
|
static int marks_on = -1, marks_vs_ok, marks_fs_ok;
|
|
|
|
int vk_pool_marks_enabled(void) {
|
|
if (marks_on < 0) {
|
|
marks_on = rt_knob("TEX_DISUSE", "1") >= 1; // 2: marks and stand-ins, nothing freed (vk_texture.c "disuse")
|
|
VkCtx *c = vk_ctx();
|
|
if (c) {
|
|
VkPhysicalDeviceFeatures f;
|
|
vkGetPhysicalDeviceFeatures(c->phys, &f); // vk_device.c enables every feature the GPU reports
|
|
marks_vs_ok = f.vertexPipelineStoresAndAtomics, marks_fs_ok = f.fragmentStoresAndAtomics;
|
|
}
|
|
}
|
|
return marks_on;
|
|
}
|
|
|
|
// A fragment shader that stores to memory runs before the depth test unless it declares EarlyFragmentTests: the GPU
|
|
// must run it for fragments the test then rejects, as the stores are visible. The marks would so take the early depth
|
|
// rejection from every marked fragment shader and make overdraw cost full shading. With early_ok, a fragment shader
|
|
// whose result is the same under early tests gets EarlyFragmentTests along with its marks: one that discards, writes
|
|
// its depth, sample mask or stencil reference, or already writes memory does not, as early tests would change what it
|
|
// does. The caller passes early_ok only for a pipeline without alpha to coverage, which also drops samples after the
|
|
// shader (ensure_init keeps alpha to coverage out of the dynamic state while marks are on, so a pipeline knows).
|
|
// *status: 1 marked, 0 no pool texture access to mark, -1 one that could not be marked (*why says what stopped it).
|
|
// NULL unless marked.
|
|
static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, uint64_t *out_n, int *status,
|
|
const char **why) {
|
|
enum { OP_TYPE_RUNTIME_ARRAY = 29, OP_TYPE_STRUCT_ = 30, OP_ACCESS_CHAIN_ = 65, OP_IN_BOUNDS_ACCESS_CHAIN = 66,
|
|
OP_STORE_ = 62, OP_MEMBER_DECORATE_ = 72, OP_DECORATION_GROUP = 73, OP_GROUP_DECORATE = 74,
|
|
OP_GROUP_MEMBER_DECORATE = 75, OP_DECORATE_ID = 332, OP_DECORATE_STRING = 5632, OP_MEMBER_DECORATE_STRING = 5633,
|
|
OP_EXECUTION_MODE = 16, OP_EXECUTION_MODE_ID = 331, OP_IMAGE_WRITE = 99, OP_ATOMIC_FIRST = 227,
|
|
OP_ATOMIC_LAST = 242, OP_KILL = 252, OP_TERMINATE_INVOCATION = 4416, OP_DEMOTE_TO_HELPER = 5380,
|
|
EM_EARLY_FRAGMENT_TESTS = 9, BI_SAMPLE_MASK = 20, BI_FRAG_DEPTH = 22, BI_FRAG_STENCIL_REF = 5014,
|
|
SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35 };
|
|
*status = 0, *why = NULL;
|
|
if (n < 5) return NULL;
|
|
uint32_t bound = w[3];
|
|
if (!bound || bound > (1u << 22)) return NULL;
|
|
uint32_t *bind = calloc(bound, sizeof *bind); // per id: its Binding + 1 when its DescriptorSet is 2, else 0
|
|
uint8_t *set2 = calloc(bound, 1);
|
|
if (!bind || !set2) return free(bind), free(set2), *status = -1, *why = "no memory", NULL;
|
|
uint64_t ann_end = 0, fn_start = 0, em_end = 0; // em_end: past the entry points and execution modes
|
|
uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0;
|
|
int vertex_stage = 0, fragment_stage = 0, entries = 0, early = early_ok;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) return free(bind), free(set2), *status = -1, *why = "an instruction past the module's end", NULL;
|
|
if (op == SPV_OP_FUNCTION) { fn_start = i; break; }
|
|
if (op == SPV_OP_ENTRY_POINT || op == OP_EXECUTION_MODE || op == OP_EXECUTION_MODE_ID) em_end = i + l;
|
|
if (op == SPV_OP_ENTRY_POINT && l >= 3) {
|
|
entries++;
|
|
if (w[i + 1] <= 3) vertex_stage = 1; // Vertex, TessellationControl, TessellationEvaluation, Geometry
|
|
else if (w[i + 1] == 4) fragment_stage = 1, fs_entry = w[i + 2];
|
|
} else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == EM_EARLY_FRAGMENT_TESTS) {
|
|
early = 0; // has them already
|
|
} else if (op == SPV_OP_DECORATE && l >= 4 && w[i + 1] < bound) {
|
|
if (w[i + 2] == SPV_DEC_DESCRIPTOR_SET && w[i + 3] == 2) set2[w[i + 1]] = 1;
|
|
else if (w[i + 2] == SPV_DEC_BINDING) bind[w[i + 1]] = w[i + 3] + 1;
|
|
else if (w[i + 2] == SPV_DEC_BUILTIN &&
|
|
(w[i + 3] == BI_SAMPLE_MASK || w[i + 3] == BI_FRAG_DEPTH || w[i + 3] == BI_FRAG_STENCIL_REF))
|
|
early = 0;
|
|
} else if (op == SPV_OP_VARIABLE && l >= 4 && w[i + 3] == SC_STORAGE_BUFFER) {
|
|
early = 0; // the guest's own storage buffers: its stores would change under early tests
|
|
} else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 0) {
|
|
uint_t = w[i + 1];
|
|
} else if (op == SPV_OP_CONSTANT && l == 4 && uint_t && w[i + 1] == uint_t) {
|
|
if (w[i + 3] == 0 && !c0) c0 = w[i + 2];
|
|
if (w[i + 3] == 1 && !c1) c1 = w[i + 2];
|
|
} else if (op == SPV_OP_TYPE_POINTER && l == 4 && w[i + 2] == SC_STORAGE_BUFFER && uint_t && w[i + 3] == uint_t) {
|
|
ptr_u = w[i + 1];
|
|
}
|
|
if (op == SPV_OP_DECORATE || op == OP_MEMBER_DECORATE_ || op == OP_DECORATION_GROUP || op == OP_GROUP_DECORATE ||
|
|
op == OP_GROUP_MEMBER_DECORATE || op == OP_DECORATE_ID || op == OP_DECORATE_STRING || op == OP_MEMBER_DECORATE_STRING)
|
|
ann_end = i + l;
|
|
i += l;
|
|
}
|
|
// pool texture variables: set 2, a binding below VK_POOL_KINDS (spirv_pool_rebind); one already at the marks
|
|
// binding means the module was marked before
|
|
uint64_t sites = 0;
|
|
int marked = 0;
|
|
for (uint32_t id = 0; id < bound; id++)
|
|
if (set2[id] && bind[id] == VK_POOL_MARK_BINDING + 1) marked = 1;
|
|
for (uint64_t i = fn_start; fn_start && !marked && i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l || i + l > n) break;
|
|
if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] &&
|
|
bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS)
|
|
sites++;
|
|
else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE ||
|
|
(op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST))
|
|
early = 0;
|
|
i += l;
|
|
}
|
|
if (marked || !sites) return free(bind), free(set2), NULL;
|
|
*why = w[1] < 0x00010300 || w[1] > 0x00010600 ? "a SPIR-V version before 1.3 or after 1.6"
|
|
: !ann_end || !fn_start ? "no decorations or no function"
|
|
: vertex_stage && !marks_vs_ok ? "the GPU allows no stores in vertex-pipeline shaders"
|
|
: fragment_stage && !marks_fs_ok ? "the GPU allows no stores in fragment shaders"
|
|
: NULL;
|
|
if (*why) return free(bind), free(set2), *status = -1, NULL;
|
|
early = early && fragment_stage && entries == 1 && em_end && em_end < ann_end;
|
|
// From SPIR-V 1.4 on an entry point lists every global variable its functions use, not only the inputs and outputs
|
|
// (the translator's mesh shaders are 1.4): each entry point gets the marks variable, a word more.
|
|
int iface = w[1] >= 0x00010400;
|
|
// the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site
|
|
uint64_t on = n + 3 + (uint64_t)entries + 20 + 30 + 9 * sites;
|
|
uint32_t *o = malloc(on * 4);
|
|
if (!o) return free(bind), free(set2), *status = -1, *why = "no memory", NULL;
|
|
uint32_t next = bound;
|
|
uint32_t new_uint = uint_t ? 0 : (uint_t = next++), new_c0 = c0 ? 0 : (c0 = next++), new_c1 = c1 ? 0 : (c1 = next++);
|
|
uint32_t rta = next++, st = next++, ptr_st = next++, var = next++, new_ptr_u = ptr_u ? 0 : (ptr_u = next++);
|
|
uint64_t p = 5;
|
|
memcpy(o, w, 5 * 4);
|
|
for (uint64_t i = 5; i < ann_end;) { // the header up to the annotations' end: entry points widened, the mode added
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (early && i == em_end) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = fs_entry, o[p++] = EM_EARLY_FRAGMENT_TESTS;
|
|
memcpy(o + p, w + i, l * 4);
|
|
if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) o[p] = (l + 1) << 16 | op, o[p + l] = var, p++;
|
|
p += l;
|
|
i += l;
|
|
}
|
|
o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = rta, o[p++] = DEC_ARRAY_STRIDE, o[p++] = 4;
|
|
o[p++] = 5u << 16 | OP_MEMBER_DECORATE_, o[p++] = st, o[p++] = 0, o[p++] = DEC_OFFSET_, o[p++] = 0;
|
|
o[p++] = 3u << 16 | SPV_OP_DECORATE, o[p++] = st, o[p++] = DEC_BLOCK;
|
|
o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = var, o[p++] = SPV_DEC_DESCRIPTOR_SET, o[p++] = 2;
|
|
o[p++] = 4u << 16 | SPV_OP_DECORATE, o[p++] = var, o[p++] = SPV_DEC_BINDING, o[p++] = VK_POOL_MARK_BINDING;
|
|
memcpy(o + p, w + ann_end, (fn_start - ann_end) * 4), p += fn_start - ann_end;
|
|
if (new_uint) o[p++] = 4u << 16 | SPV_OP_TYPE_INT, o[p++] = uint_t, o[p++] = 32, o[p++] = 0;
|
|
if (new_c0) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c0, o[p++] = 0;
|
|
if (new_c1) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c1, o[p++] = 1;
|
|
o[p++] = 3u << 16 | OP_TYPE_RUNTIME_ARRAY, o[p++] = rta, o[p++] = uint_t;
|
|
o[p++] = 3u << 16 | OP_TYPE_STRUCT_, o[p++] = st, o[p++] = rta;
|
|
o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_st, o[p++] = SC_STORAGE_BUFFER, o[p++] = st;
|
|
if (new_ptr_u) o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_u, o[p++] = SC_STORAGE_BUFFER, o[p++] = uint_t;
|
|
o[p++] = 4u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_st, o[p++] = var, o[p++] = SC_STORAGE_BUFFER;
|
|
for (uint64_t i = fn_start; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
memcpy(o + p, w + i, l * 4), p += l;
|
|
if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] &&
|
|
bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) {
|
|
uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access
|
|
o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4];
|
|
o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1;
|
|
}
|
|
i += l;
|
|
}
|
|
o[3] = next;
|
|
free(bind), free(set2);
|
|
*out_n = p;
|
|
*status = 1;
|
|
return o;
|
|
}
|
|
|
|
// --selftest with NX2NAVI_SPV_MARK_TEST=<list>: each line "<in.spv> <out.spv>"; the module is moved to the pools
|
|
// (spirv_pool_rebind) and marked, and written to <out> when marked, for spirv-val. Prints the counts; 0 failures unless
|
|
// a file cannot be read or written.
|
|
static int spirv_mark_list_test(const char *list) {
|
|
marks_vs_ok = marks_fs_ok = 1;
|
|
FILE *f = fopen(list, "rb");
|
|
if (!f) return printf("spirv: pool marks test: %s cannot be read\n", list), 1;
|
|
char line[2048];
|
|
int marked = 0, none = 0, missed = 0, bad = 0;
|
|
while (fgets(line, sizeof line, f)) {
|
|
char in[1024], out[1024];
|
|
if (sscanf(line, "%1023s %1023s", in, out) != 2) continue;
|
|
FILE *g = fopen(in, "rb");
|
|
if (!g) { bad++; continue; }
|
|
fseek(g, 0, SEEK_END);
|
|
long sz = ftell(g);
|
|
fseek(g, 0, SEEK_SET);
|
|
uint32_t *w = sz > 20 ? malloc((size_t)sz) : NULL;
|
|
if (!w || fread(w, 1, (size_t)sz, g) != (size_t)sz) { fclose(g), free(w), bad++; continue; }
|
|
fclose(g);
|
|
uint64_t n = (uint64_t)sz / 4, mn = 0;
|
|
uint32_t *pooled = spirv_pool_rebind(w, n);
|
|
int st;
|
|
const char *why;
|
|
uint32_t *m = spirv_pool_mark(pooled ? pooled : w, n, 1, &mn, &st, &why);
|
|
if (st > 0) {
|
|
marked++;
|
|
if (!plat_file_write_atomic(out, m, (size_t)mn * 4)) {
|
|
if (!bad) printf("spirv: pool marks test: %s cannot be written\n", out);
|
|
bad++;
|
|
}
|
|
} else if (st < 0) {
|
|
missed++;
|
|
printf("spirv: pool marks test: %s not markable: %s\n", in, why);
|
|
} else {
|
|
none++;
|
|
}
|
|
free(m), free(pooled), free(w);
|
|
}
|
|
fclose(f);
|
|
printf("spirv: pool marks test: %d marked, %d without pool texture accesses, %d not markable, %d file errors\n", marked,
|
|
none, missed, bad);
|
|
return bad;
|
|
}
|
|
|
|
// early_ok: a fragment shader whose pipeline has no alpha to coverage (spirv_pool_mark)
|
|
static VkShaderModule make_module(const uint32_t *w, uint64_t words, int early_ok) {
|
|
const uint32_t *orig_w = w; // for the log of a module the marks cannot take
|
|
uint64_t orig_words = words, fixed_n = 0;
|
|
uint32_t *fixed = spirv_fixup(w, words, &fixed_n);
|
|
if (fixed) w = fixed, words = fixed_n;
|
|
uint32_t *pooled = spirv_pool_rebind(w, words);
|
|
if (pooled) {
|
|
free(fixed);
|
|
w = fixed = pooled;
|
|
if (vk_pool_marks_enabled()) {
|
|
uint64_t mk_n = 0;
|
|
int st;
|
|
const char *why;
|
|
uint32_t *mk = spirv_pool_mark(w, words, early_ok, &mk_n, &st, &why);
|
|
if (mk) {
|
|
free(fixed);
|
|
w = fixed = mk, words = mk_n;
|
|
} else if (st < 0) {
|
|
int nth = plat_inc32(&vk_pool_marks_missed);
|
|
// the hash is FNV-1a of the words make_module got, as vk_dump hashes a shader's SPIR-V; it differs
|
|
// from the translation's only where the pipeline patched the module first (added vertex outputs)
|
|
if (nth <= 8) {
|
|
uint32_t model = ~0u;
|
|
for (uint64_t i = 5; i < words;) {
|
|
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
|
if (!l) break;
|
|
if (op == SPV_OP_ENTRY_POINT && l >= 2) { model = w[i + 1]; break; }
|
|
i += l;
|
|
}
|
|
const char *stage = model == 0 ? "vertex" : model <= 3 ? "tessellation or geometry" : model == 4 ? "fragment"
|
|
: model == 5 ? "compute" : model == 5364 ? "task" : model == 5365 ? "mesh" : "other";
|
|
rt_log("vk_pipeline: a %s shader that indexes the texture pool could not be marked (SPIR-V %u.%u, %llu "
|
|
"words, hash %016llx): %s%s", stage, w[1] >> 16 & 0xff, w[1] >> 8 & 0xff,
|
|
(unsigned long long)words, (unsigned long long)fnv(FNV0, orig_w, (size_t)orig_words * 4), why,
|
|
nth == 1 ? "; textures are not evicted by disuse this run (NX2NAVI_TEX_DISUSE)" : "");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
uint64_t rc_n = 0;
|
|
uint32_t *rc = P.reconverge ? spirv_reconverge(w, words, &rc_n) : NULL;
|
|
if (rc) {
|
|
free(fixed);
|
|
w = fixed = rc, words = rc_n;
|
|
}
|
|
uint64_t np_n = 0;
|
|
uint32_t *np = P.nan_preserve ? spirv_nan_preserve(w, words, P.nan_preserve, &np_n) : NULL;
|
|
if (np) {
|
|
free(fixed);
|
|
w = fixed = np, words = np_n;
|
|
}
|
|
uint64_t nb_n = 0;
|
|
uint32_t nb_tests = 0, nb_sels = 0;
|
|
uint32_t *nb = P.nan_bits ? spirv_nan_bits(w, words, &nb_n, &nb_tests, &nb_sels) : NULL;
|
|
if (nb) {
|
|
free(fixed);
|
|
w = fixed = nb, words = nb_n;
|
|
__atomic_fetch_add(&P.nan_bits_tests, nb_tests, __ATOMIC_RELAXED);
|
|
__atomic_fetch_add(&P.nan_bits_selects, nb_sels, __ATOMIC_RELAXED);
|
|
if (__atomic_add_fetch(&P.nan_bits_modules, 1, __ATOMIC_RELAXED) == 1)
|
|
rt_log("vk_pipeline: first module with NaN tests made integer: %u tests, %u selects of NaN bits", nb_tests,
|
|
nb_sels);
|
|
}
|
|
if (P.feedback >= 2) mm_hash = fnv(FNV0, w, (size_t)words * 4);
|
|
VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)words * 4, .pCode = w};
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
VkResult r = vkCreateShaderModule(P.device, &ci, NULL, &m);
|
|
free(fixed);
|
|
return r == VK_SUCCESS ? m : VK_NULL_HANDLE;
|
|
}
|
|
|
|
// ---- descriptor layouts
|
|
|
|
typedef struct {
|
|
uint32_t n;
|
|
VkDescriptorSetLayoutBinding b[MAX_BINDINGS];
|
|
VkPipeBinding pb[MAX_BINDINGS];
|
|
int pool; // a stage indexes the texture or sampler pool (set 2, not in b / pb)
|
|
} Bindings;
|
|
|
|
static VkDescriptorType descriptor_type(int kind, const GfxShaderRes *r) {
|
|
int buffer = (r->type & 0xff) == 2;
|
|
switch (kind) {
|
|
case GFX_RES_UBO: return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
|
|
case GFX_RES_SSBO: return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
|
|
case GFX_RES_TEX:
|
|
if (buffer) return VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER;
|
|
if (r->separate) LOG_ONCE("vk_pipeline: a separate texture handle: bound as SAMPLED_IMAGE");
|
|
return r->separate ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
|
|
default: return buffer ? VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
}
|
|
}
|
|
|
|
static VkShaderStageFlags stage_bit(int stage) {
|
|
return stage == GFX_STAGE_VS ? VK_SHADER_STAGE_VERTEX_BIT : stage == GFX_STAGE_FS ? VK_SHADER_STAGE_FRAGMENT_BIT
|
|
: stage == GFX_STAGE_TS ? VK_SHADER_STAGE_TASK_BIT_EXT : stage == GFX_STAGE_MS ? VK_SHADER_STAGE_MESH_BIT_EXT
|
|
: VK_SHADER_STAGE_COMPUTE_BIT;
|
|
}
|
|
|
|
static int is_mesh(const VkPipeState *s) { return s->vs && gfx_shader_stage(s->vs) == GFX_STAGE_MS; }
|
|
|
|
// Where a shader's translation came from (gfx_shader_origin), for the log.
|
|
static const char *origin_name(const GfxShader *s) {
|
|
static const char *const name[3] = {"from the shader cache", "a previous translator's", "translated in this session"};
|
|
return s ? name[gfx_shader_origin(s)] : "none";
|
|
}
|
|
|
|
// The shaders of the pipeline this thread builds, for the crash notes (platform.h plat_crash_context): the first 8
|
|
// bytes of each cache key, as gfx_shader.c names its cache files.
|
|
static void crash_note(const char *what, const GfxShader *a, const GfxShader *b, const GfxShader *c) {
|
|
static __thread char buf[160];
|
|
char *o = buf;
|
|
o += snprintf(o, 40, "building %s", what);
|
|
const GfxShader *sh[3] = {a, b, c};
|
|
for (int i = 0; i < 3; i++)
|
|
if (sh[i]) {
|
|
const uint8_t *k = gfx_shader_key(sh[i]);
|
|
o += snprintf(o, 40, " stage %d %02x%02x%02x%02x%02x%02x%02x%02x", gfx_shader_stage(sh[i]), k[0], k[1], k[2], k[3],
|
|
k[4], k[5], k[6], k[7]);
|
|
}
|
|
plat_crash_context(buf);
|
|
}
|
|
|
|
// One past the highest binding a shader uses (1 with no resources: binding 0 is the support buffer).
|
|
static uint32_t binding_span(const GfxShader *s) {
|
|
uint32_t span = 1;
|
|
for (int k = 0; k < GFX_RES_KINDS; k++) {
|
|
const GfxShaderRes *r;
|
|
int n = gfx_shader_resources(s, k, &r);
|
|
for (int i = 0; i < n; i++)
|
|
if (r[i].binding + (uint32_t)(r[i].count ? r[i].count : 1) > span) span = r[i].binding + (r[i].count ? r[i].count : 1);
|
|
}
|
|
return span;
|
|
}
|
|
|
|
// Adds a stage's resources with bindings b >= 1 moved by off; binding 0 (support buffer) is shared.
|
|
static int add_stage(Bindings *B, const GfxShader *s, uint32_t off) {
|
|
int stage = gfx_shader_stage(s);
|
|
VkShaderStageFlags bit = stage_bit(stage);
|
|
if (!B->n) {
|
|
B->b[0] = (VkDescriptorSetLayoutBinding){0, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1, 0, NULL};
|
|
B->pb[0] = (VkPipeBinding){0, (uint8_t)stage, VK_PIPE_SUPPORT, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1, 0, NULL};
|
|
B->n = 1;
|
|
}
|
|
B->b[0].stageFlags |= bit;
|
|
B->pb[0].stages |= bit;
|
|
for (int k = 0; k < GFX_RES_KINDS; k++) {
|
|
const GfxShaderRes *r;
|
|
int n = gfx_shader_resources(s, k, &r);
|
|
for (int i = 0; i < n; i++) {
|
|
if (r[i].pool) { // set 2 (spirv_pool_rebind), bound by the replay from the pools
|
|
B->pool = 1;
|
|
continue;
|
|
}
|
|
if (B->n == MAX_BINDINGS || !r[i].binding) return 0;
|
|
VkDescriptorType t = descriptor_type(k, &r[i]);
|
|
uint32_t binding = r[i].binding + off, count = r[i].count ? r[i].count : 1;
|
|
B->b[B->n] = (VkDescriptorSetLayoutBinding){binding, t, count, bit, NULL};
|
|
B->pb[B->n] = (VkPipeBinding){(uint16_t)binding, (uint8_t)stage, (uint8_t)k, t, (uint16_t)count, bit, &r[i]};
|
|
B->n++;
|
|
}
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
// The first binding a module's resource variables use that the binding list b[0..n) lacks (module bindings >= 1 moved
|
|
// by off), or -1. A reflection that misses a resource would otherwise hand the driver an invalid layout.
|
|
static int module_binding_missing(const GfxShader *s, const VkDescriptorSetLayoutBinding *b, uint32_t n, uint32_t off) {
|
|
uint64_t words = 0;
|
|
const uint32_t *w = gfx_shader_spirv(s, &words);
|
|
const GfxShaderRes *tr;
|
|
int nt = gfx_shader_resources(s, GFX_RES_TEX, &tr);
|
|
for (uint64_t i = 5; w && i < words;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > words) break;
|
|
int pooled = 0; // a pool array: set 2, not in this layout (spirv_pool_rebind)
|
|
for (int t = 0; t < nt && op == SPV_OP_DECORATE && len >= 4; t++)
|
|
if (tr[t].pool && tr[t].binding == (w[i + 3] & 0xffff)) pooled = 1; // GfxShaderRes.binding is 16 bits
|
|
if (op == SPV_OP_DECORATE && len >= 4 && w[i + 2] == SPV_DEC_BINDING && !pooled) {
|
|
uint32_t want = w[i + 3] + (w[i + 3] >= 1 ? off : 0), j = 0;
|
|
while (j < n && b[j].binding != want) j++;
|
|
if (j == n) return (int)w[i + 3];
|
|
}
|
|
i += len;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
// Checks both stages of a graphics pipeline against its layout; 0 with why filled when one uses a missing binding.
|
|
static int layout_covers(const VkPipeState *s, const Layout *l, uint32_t fs_off, char *why, size_t cap) {
|
|
uint32_t nv = l->set1 ? l->n0 : l->n;
|
|
int m = module_binding_missing(s->vs, l->b, nv, 0);
|
|
const char *stage = "vertex";
|
|
if (m < 0 && s->fs) {
|
|
m = l->set1 ? module_binding_missing(s->fs, l->b + l->n0, l->n - l->n0, 0) : module_binding_missing(s->fs, l->b, l->n, fs_off);
|
|
stage = "fragment";
|
|
}
|
|
if (m >= 0) snprintf(why, cap, "the %s shader uses binding %d, which its reflection and layout lack", stage, m);
|
|
return m < 0;
|
|
}
|
|
|
|
// The update template of a set whose bindings are b[0..n): one entry per binding, in order, over the packed array the
|
|
// replay fills from the set's first descriptor, a VkDescSlot per descriptor. The bind point follows the stages (compute
|
|
// layouts hold the compute stage only). VK_NULL_HANDLE when templates are off, for an empty set, for a push set
|
|
// without vkCmdPushDescriptorSetWithTemplateKHR or when creation fails: the replay then writes the set with write
|
|
// structs.
|
|
static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl,
|
|
int push, VkPipelineLayout pl, uint32_t set) {
|
|
if (!P.tpl || !n || (push && !P.tpl_push)) return VK_NULL_HANDLE;
|
|
VkDescriptorUpdateTemplateEntry e[MAX_BINDINGS];
|
|
size_t off = 0;
|
|
VkShaderStageFlags stages = 0;
|
|
for (uint32_t i = 0; i < n; i++) {
|
|
e[i] = (VkDescriptorUpdateTemplateEntry){b[i].binding, 0, b[i].descriptorCount, b[i].descriptorType, off, sizeof(VkDescSlot)};
|
|
off += sizeof(VkDescSlot) * b[i].descriptorCount;
|
|
stages |= b[i].stageFlags;
|
|
}
|
|
VkDescriptorUpdateTemplateCreateInfo ci = {
|
|
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_UPDATE_TEMPLATE_CREATE_INFO, .descriptorUpdateEntryCount = n,
|
|
.pDescriptorUpdateEntries = e,
|
|
.templateType = push ? VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR : VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET,
|
|
.descriptorSetLayout = sl,
|
|
.pipelineBindPoint = stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS,
|
|
.pipelineLayout = pl, .set = set};
|
|
VkDescriptorUpdateTemplate t;
|
|
if (P.tpl_create(P.device, &ci, NULL, &t) != VK_SUCCESS) {
|
|
LOG_ONCE("vk_pipeline: descriptor update template creation failed: that set is written with write structs");
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
return t;
|
|
}
|
|
|
|
// Finds or creates the layout of a binding list (sorted here); NULL on failure. Caller holds P.lock.
|
|
// Sorts a binding list by binding and hashes it into *h; the descriptor total, or -1 when a binding repeats.
|
|
static int64_t bindings_sort(Bindings *B, uint64_t *h) {
|
|
for (uint32_t i = 1; i < B->n; i++) // insertion sort by binding; the lists are short
|
|
for (uint32_t j = i; j > 0 && B->b[j - 1].binding > B->b[j].binding; j--) {
|
|
VkDescriptorSetLayoutBinding t = B->b[j]; B->b[j] = B->b[j - 1]; B->b[j - 1] = t;
|
|
VkPipeBinding u = B->pb[j]; B->pb[j] = B->pb[j - 1]; B->pb[j - 1] = u;
|
|
}
|
|
int64_t total = 0;
|
|
for (uint32_t i = 0; i < B->n; i++) {
|
|
if (i && B->b[i].binding == B->b[i - 1].binding) return -1;
|
|
uint32_t k[4] = {B->b[i].binding, B->b[i].descriptorType, B->b[i].descriptorCount, B->b[i].stageFlags};
|
|
*h = fnv(*h, k, sizeof k);
|
|
total += B->b[i].descriptorCount;
|
|
}
|
|
return total;
|
|
}
|
|
|
|
static Layout *layout_get(Bindings *B) {
|
|
uint64_t h = FNV0;
|
|
int64_t total = bindings_sort(B, &h);
|
|
if (total < 0) return NULL;
|
|
Layout **head = &layouts[h % LAYOUT_BUCKETS];
|
|
for (Layout *l = *head; l; l = l->next)
|
|
if (l->hash == h && l->n == B->n && !memcmp(l->b, B->b, B->n * sizeof *B->b)) return l;
|
|
Layout *l = calloc(1, sizeof *l + B->n * sizeof *B->b);
|
|
if (!l) return NULL;
|
|
l->hash = h, l->n = B->n;
|
|
memcpy(l->b, B->b, B->n * sizeof *B->b);
|
|
l->push = total <= (int64_t)P.max_push;
|
|
VkDescriptorSetLayoutCreateInfo si = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
.flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0,
|
|
.bindingCount = l->n, .pBindings = l->b};
|
|
if (vkCreateDescriptorSetLayout(P.device, &si, NULL, &l->set) != VK_SUCCESS) {
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
VkDescriptorSetLayout sets3[3] = {l->set, vk_empty_set_layout(), vk_pool_set_layout()}; // set 2: the pools
|
|
VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 3,
|
|
.pSetLayouts = sets3};
|
|
if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) {
|
|
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
l->tpl[0] = make_tpl(l->b, l->n, l->set, l->push, l->layout, 0);
|
|
l->next = *head;
|
|
*head = l;
|
|
P.stats.layouts++;
|
|
P.stats.push_layouts += (uint64_t)l->push;
|
|
return l;
|
|
}
|
|
|
|
// Independent layouts (P.indep): the vertex stage's bindings in set 0, the fragment stage's in set 1 (empty without a
|
|
// fragment shader), each with its own support buffer at binding 0, the pipeline layout created with
|
|
// VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT so stage libraries can use it. Only the P.push_stage set is a
|
|
// push-descriptor set (a layout holds one), when it fits maxPushDescriptors; the other is allocated per draw from the
|
|
// command buffer's pools. On success *B holds both lists, set 0 first, VkPipeBinding.set filled in. Caller holds P.lock.
|
|
static Layout *layout_pair(Bindings *B, Bindings *F) {
|
|
uint64_t h = fnv(FNV0, "indep", 5);
|
|
int64_t tv = bindings_sort(B, &h);
|
|
h = fnv(h, "|", 1);
|
|
int64_t tf = bindings_sort(F, &h);
|
|
if (tv < 0 || tf < 0 || B->n + F->n > MAX_BINDINGS) return NULL;
|
|
uint32_t n0 = B->n, n = B->n + F->n;
|
|
memcpy(B->b + n0, F->b, F->n * sizeof *F->b);
|
|
memcpy(B->pb + n0, F->pb, F->n * sizeof *F->pb);
|
|
for (uint32_t i = n0; i < n; i++) B->pb[i].set = 1;
|
|
B->n = n;
|
|
Layout **head = &layouts[h % LAYOUT_BUCKETS];
|
|
for (Layout *l = *head; l; l = l->next)
|
|
if (l->hash == h && l->set1 && l->n == n && l->n0 == n0 && !memcmp(l->b, B->b, n * sizeof *B->b)) return l;
|
|
Layout *l = calloc(1, sizeof *l + n * sizeof *B->b);
|
|
if (!l) return NULL;
|
|
l->hash = h, l->n = n, l->n0 = n0;
|
|
memcpy(l->b, B->b, n * sizeof *B->b);
|
|
l->push = P.push_stage == GFX_STAGE_VS && tv <= (int64_t)P.max_push;
|
|
l->push1 = P.push_stage == GFX_STAGE_FS && F->n && tf <= (int64_t)P.max_push;
|
|
VkDescriptorSetLayoutCreateInfo s0 = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
.flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0,
|
|
.bindingCount = n0, .pBindings = l->b};
|
|
VkDescriptorSetLayoutCreateInfo s1 = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
.flags = l->push1 ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0,
|
|
.bindingCount = n - n0, .pBindings = l->b + n0};
|
|
if (vkCreateDescriptorSetLayout(P.device, &s0, NULL, &l->set) != VK_SUCCESS) {
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
if (vkCreateDescriptorSetLayout(P.device, &s1, NULL, &l->set1) != VK_SUCCESS) {
|
|
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
VkDescriptorSetLayout sets[3] = {l->set, l->set1, vk_pool_set_layout()};
|
|
VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
|
|
.flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 3,
|
|
.pSetLayouts = sets};
|
|
if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) {
|
|
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
|
|
vkDestroyDescriptorSetLayout(P.device, l->set1, NULL);
|
|
free(l);
|
|
return NULL;
|
|
}
|
|
l->tpl[0] = make_tpl(l->b, n0, l->set, l->push, l->layout, 0);
|
|
l->tpl[1] = make_tpl(l->b + n0, n - n0, l->set1, l->push1, l->layout, 1);
|
|
l->next = *head;
|
|
*head = l;
|
|
P.stats.layouts++;
|
|
P.stats.push_layouts += (uint64_t)(l->push | l->push1);
|
|
return l;
|
|
}
|
|
|
|
// Vertex outputs with independent layouts: each vertex shader declares the union of the inputs of every fragment
|
|
// shader met with it (pipeline-keys.bin at the prewarm, then each pipeline built), so its module does not depend on the
|
|
// fragment shader of one pipeline. Unwritten ones read (0, 0, 0, 1) as before. NX2NAVI_VS_OUTS=all declares all 32.
|
|
#define VS_OUTS_CAP 8192
|
|
static struct {
|
|
PlatMutex lock;
|
|
const GfxShader *vs[VS_OUTS_CAP];
|
|
uint32_t mask[VS_OUTS_CAP];
|
|
GfxShaderIo io[VS_OUTS_CAP][32];
|
|
uint32_t n, full;
|
|
} VO = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
// Adds fs's inputs (packed ones excepted) to vs's union; returns the union, types in io (the first type met wins).
|
|
static uint32_t vs_outs(const GfxShader *vs, const GfxShader *fs, GfxShaderIo io[32]) {
|
|
GfxShaderIo fin[32];
|
|
uint32_t need = fs ? gfx_shader_inputs(fs, fin) : 0, mask;
|
|
for (int i = 0; i < 32; i++)
|
|
if (need >> i & 1 && fin[i].packed) need &= ~(1u << i);
|
|
plat_mutex_lock(&VO.lock);
|
|
uint32_t j = (uint32_t)(((uintptr_t)vs >> 4) * 0x9e3779b1u) & (VS_OUTS_CAP - 1);
|
|
while (VO.vs[j] && VO.vs[j] != vs) j = (j + 1) & (VS_OUTS_CAP - 1);
|
|
if (!VO.vs[j] && VO.n + 1 >= VS_OUTS_CAP * 3 / 4) { // full: this pair alone
|
|
VO.full++;
|
|
plat_mutex_unlock(&VO.lock);
|
|
LOG_ONCE("vk_pipeline: vertex output union table full: outputs from the pair");
|
|
memset(io, 0, 32 * sizeof *io);
|
|
for (int i = 0; i < 32; i++)
|
|
if (need >> i & 1) io[i] = fin[i];
|
|
return need;
|
|
}
|
|
if (!VO.vs[j]) VO.vs[j] = vs, VO.n++;
|
|
for (int i = 0; i < 32; i++)
|
|
if (!(need >> i & 1)) continue;
|
|
else if (!(VO.mask[j] >> i & 1)) VO.io[j][i] = fin[i], VO.mask[j] |= 1u << i;
|
|
else if (VO.io[j][i].type == fin[i].type && VO.io[j][i].comps < fin[i].comps) VO.io[j][i].comps = fin[i].comps;
|
|
mask = VO.mask[j];
|
|
memcpy(io, VO.io[j], 32 * sizeof *io);
|
|
plat_mutex_unlock(&VO.lock);
|
|
return mask;
|
|
}
|
|
|
|
// ---- pipeline cache table
|
|
|
|
static Pipe *pipe_find(const void *key, size_t size, uint64_t h) {
|
|
for (Pipe *p = pipes[h % PIPE_BUCKETS]; p; p = p->next)
|
|
if (p->hash == h && p->key_size == size && !memcmp(p->key, key, size)) return p;
|
|
return NULL;
|
|
}
|
|
|
|
// Stores a result (a failed one too, so it is not retried every draw). Caller holds P.lock.
|
|
static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipeline, const Layout *l, const Bindings *B,
|
|
VkPipelineBindPoint bp, uint32_t vs_inputs) {
|
|
Pipe *p = calloc(1, sizeof *p + size);
|
|
if (!p) return NULL;
|
|
p->hash = h, p->key_size = size;
|
|
memcpy(p->key, key, size);
|
|
if (l && B) {
|
|
p->bindings = malloc(B->n * sizeof *p->bindings);
|
|
if (p->bindings) memcpy(p->bindings, B->pb, B->n * sizeof *p->bindings);
|
|
}
|
|
p->info = (VkPipeInfo){pipeline, l ? l->layout : VK_NULL_HANDLE, l ? l->set : VK_NULL_HANDLE, l ? l->push : 0, bp,
|
|
p->bindings ? B->n : 0, p->bindings, vs_inputs};
|
|
p->info.set_layout1 = l ? l->set1 : VK_NULL_HANDLE;
|
|
p->info.push1 = l ? l->push1 : 0;
|
|
p->info.n_sets = l && l->set1 ? 2 : 1;
|
|
p->info.pool = B ? B->pool : 0;
|
|
if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1];
|
|
p->lay = l;
|
|
Pipe **head = &pipes[h % PIPE_BUCKETS];
|
|
p->next = *head;
|
|
*head = p;
|
|
return p;
|
|
}
|
|
|
|
// ---- graphics pipelines
|
|
|
|
typedef struct { // normalized: zero where the state is dynamic or unused
|
|
const GfxShader *vs, *fs, *ts; // ts: the task stage of a mesh draw
|
|
uint32_t color[8], depth, stencil; // VkFormat
|
|
uint8_t n_color, samples, topo_class, minus_w;
|
|
uint8_t clamp, poly_mode, a2c, logic_op; // logic_op: VkLogicOp + 1 when enabled
|
|
uint8_t blend_en, dsb, fbl, pad; // dsb: an enabled target blends with a SRC1 factor; fbl: VkPipeState.feedback
|
|
uint32_t wmask;
|
|
uint8_t blend[8][6]; // src, dst, op, src alpha, dst alpha, alpha op (VkBlendFactor / VkBlendOp)
|
|
uint32_t attr_fmt[32], attr_off[32];
|
|
uint8_t attr_bind[32];
|
|
uint8_t rate[17]; // per binding: 0 unused, 1 vertex, 2 instance
|
|
uint8_t indep; // P.indep when built: the probe draws through both layouts
|
|
uint8_t pad2[2];
|
|
} GKey;
|
|
|
|
static int blendable(VkFormat f) {
|
|
return f != VK_FORMAT_UNDEFINED && (format_features(f, 1) & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT);
|
|
}
|
|
|
|
static int target_count(const VkPipeState *s) {
|
|
int n = 0;
|
|
for (int i = 0; i < 8; i++)
|
|
if (s->color_fmt[i]) n = i + 1;
|
|
return n;
|
|
}
|
|
|
|
// SRC1 blend factors read the fragment shader's second output (dual-source blending); the pipeline then needs a
|
|
// fragment module whose output 1 is Location 0, Index 1 (spirv_dual_source).
|
|
static int is_src1(unsigned f) { return f >= NVN_BLEND_FUNC_SRC1_COLOR && f <= NVN_BLEND_FUNC_ONE_MINUS_SRC1_ALPHA; }
|
|
// Runs per draw (gkey_fill, target_wmask): only the blend factors of blend-enabled targets are read until one is SRC1,
|
|
// which NX2navi never sets (0 of 876 pipelines in the field and fight saves).
|
|
// NX2NAVI_VK_NOOPT=1: pipelines built without the driver's optimiser, for a driver whose shader compiler crashes on
|
|
// one (the test switch for such a report); slower shaders, the same picture. mesh: only pipelines with task and mesh
|
|
// stages, the foliage's, whose optimiser is the suspect in device losses some minutes into a forest on AMD GPUs.
|
|
static VkPipelineCreateFlags noopt_flag_for(int mesh) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
const char *v = cfg_value("NX2NAVI_VK_NOOPT");
|
|
on = !strcmp(v, "mesh") ? 2 : strcmp(v, "0") != 0;
|
|
if (on) rt_log("vk_pipeline: %s built without the driver's optimiser (NX2NAVI_VK_NOOPT)",
|
|
on == 2 ? "pipelines with task and mesh stages" : "pipelines");
|
|
}
|
|
return on == 1 || (on == 2 && mesh) ? VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT : 0;
|
|
}
|
|
static VkPipelineCreateFlags noopt_flag(void) { return noopt_flag_for(0); }
|
|
|
|
// ---- pipeline creation feedback (NX2NAVI_PIPE_FEEDBACK, default 1; 0 off)
|
|
// Each pipeline, library and compute build asks the driver whether pipeline-cache.bin (P.cache) held it
|
|
// (VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) and how long the create took. The counts appear
|
|
// with the prewarm, the stage library prebuild and every cache save, so a log tells whether a slow boot compiled or
|
|
// read its pipelines: on NVIDIA the driver's own disk cache is kept per executable, so a renamed or new exe gets nothing
|
|
// from it, and only pipeline-cache.bin can make its first boot quick. =2 also logs every compute build with a hash of
|
|
// its final module, and every graphics pipeline build with the hashes of its final modules and of its fixed state, so
|
|
// two runs can be compared for a module or a state that changes between them.
|
|
enum { FB_GRAPHICS, FB_LIBRARY, FB_COMPUTE, FB_KINDS };
|
|
static struct {
|
|
PlatMutex lock;
|
|
uint64_t n[FB_KINDS], hit[FB_KINDS];
|
|
double hit_ms[FB_KINDS], miss_ms[FB_KINDS];
|
|
} FB = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
// Chains fb into a create info's pNext (next: its current pNext). Returns what the create info's pNext becomes.
|
|
static const void *fb_chain(VkPipelineCreationFeedbackCreateInfo *ci, VkPipelineCreationFeedback *fb, const void *next) {
|
|
if (!P.feedback) return next;
|
|
memset(fb, 0, sizeof *fb);
|
|
*ci = (VkPipelineCreationFeedbackCreateInfo){.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO,
|
|
.pNext = next, .pPipelineCreationFeedback = fb};
|
|
return ci;
|
|
}
|
|
|
|
static void fb_note(int kind, const VkPipelineCreationFeedback *fb) {
|
|
if (!P.feedback || !P.cache || !(fb->flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) return;
|
|
int hit = (fb->flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0;
|
|
double ms = (double)fb->duration / 1e6;
|
|
plat_mutex_lock(&FB.lock);
|
|
FB.n[kind]++, FB.hit[kind] += (uint64_t)hit;
|
|
if (hit) FB.hit_ms[kind] += ms;
|
|
else FB.miss_ms[kind] += ms;
|
|
plat_mutex_unlock(&FB.lock);
|
|
}
|
|
|
|
// One line: per kind, builds that came from pipeline-cache.bin of all builds so far, with the average time of each.
|
|
static void fb_log(const char *when) {
|
|
pb_log(when);
|
|
if (!P.feedback || !P.cache) return;
|
|
static const char *const kn[FB_KINDS] = {"pipelines", "libraries", "compute"};
|
|
char line[512];
|
|
int o = snprintf(line, sizeof line, "vk_pipeline: pipeline-cache.bin hits %s:", when);
|
|
plat_mutex_lock(&FB.lock);
|
|
for (int k = 0; k < FB_KINDS && o < (int)sizeof line - 96; k++) {
|
|
uint64_t miss = FB.n[k] - FB.hit[k];
|
|
o += snprintf(line + o, sizeof line - o, " %s %llu of %llu (%.0f%%; avg %.2f ms from the file, %.1f ms otherwise)%s", kn[k],
|
|
(unsigned long long)FB.hit[k], (unsigned long long)FB.n[k], FB.n[k] ? 100.0 * FB.hit[k] / FB.n[k] : 0.0,
|
|
FB.hit[k] ? FB.hit_ms[k] / FB.hit[k] : 0.0, miss ? FB.miss_ms[k] / miss : 0.0, k + 1 < FB_KINDS ? ";" : "");
|
|
}
|
|
plat_mutex_unlock(&FB.lock);
|
|
rt_log("%s", line);
|
|
}
|
|
|
|
static int uses_src1(const VkPipeState *s) {
|
|
for (uint32_t m = s->color.blend_en & 0xffu; m; m &= m - 1) {
|
|
int i = __builtin_ctz(m);
|
|
const blend_st *b = &s->blend[i];
|
|
if ((is_src1(b->src_rgb) || is_src1(b->dst_rgb) || is_src1(b->src_a) || is_src1(b->dst_a)) &&
|
|
i < target_count(s) && blendable(target_format(s->color_fmt[i], NULL, NULL)))
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// Colour write mask of target i: NVN's channel mask, zero for a target outside the fragment shader's output map (SPH
|
|
// omap). Maxwell leaves such a target untouched;
|
|
// Vulkan writes undefined values, and on AMD they lit grass in front of Link as flat lime. Zeros are wrong
|
|
// too: NX2NAVI's half-res transparency pass binds its linear depth (R32F) as target 7, samples it in the same draws and
|
|
// writes only targets 0-2, 4 and 5; zeroed, the depth that later passes rebuild from it was 0 in tiles, and the fog
|
|
// over distant geometry showed rectangles of clear sky.
|
|
// The same holds per component: a shader that writes a target's RGB and not its alpha leaves the alpha alone on
|
|
// Maxwell, while Vulkan writes undefined values, and AMD's driver writes whatever its export registers held (NaN and
|
|
// saturated pixels the bloom then spreads). Components outside the shader's omap are masked out, where it is known.
|
|
static uint32_t target_wmask(const VkPipeState *s, int i) {
|
|
uint32_t m = s->chmask.mask >> (4 * i) & 0xf;
|
|
if (s->fs && !(gfx_shader_outputs(s->fs, NULL) >> i & 1)) m = 0;
|
|
if (s->fs) {
|
|
uint32_t c = gfx_shader_omap(s->fs) >> (4 * i) & 0xf;
|
|
if (c) m &= c;
|
|
}
|
|
if (i == 1 && s->fs && P.f_dual_src && uses_src1(s)) m = 0; // output 1 is the second blend source (spirv_dual_source)
|
|
return m;
|
|
}
|
|
|
|
uint32_t vk_pipeline_target_wmask(const VkPipeState *s, int i) { return target_wmask(s, i); }
|
|
|
|
static void gkey_fill(const VkPipeState *s, GKey *k) {
|
|
memset(k, 0, sizeof *k);
|
|
k->vs = s->vs, k->fs = s->fs, k->ts = s->ts;
|
|
k->indep = (uint8_t)P.indep;
|
|
k->n_color = (uint8_t)target_count(s);
|
|
for (int i = 0; i < k->n_color; i++) k->color[i] = target_format(s->color_fmt[i], NULL, NULL);
|
|
int d, st;
|
|
VkFormat df = target_format(s->depth_fmt, &d, &st);
|
|
k->depth = d ? df : VK_FORMAT_UNDEFINED;
|
|
k->stencil = st ? df : VK_FORMAT_UNDEFINED;
|
|
k->samples = s->samples > 1 ? s->samples : 1;
|
|
k->topo_class = (uint8_t)topology_class(vk_pipeline_topology(s->prim));
|
|
k->minus_w = s->depth_minus_w != 0;
|
|
k->dsb = s->fs && P.f_dual_src && uses_src1(s);
|
|
k->fbl = s->fs ? s->feedback : 0;
|
|
if (!P.dyn_clamp) k->clamp = s->depth_clamp && P.f_depth_clamp;
|
|
if (!P.dyn_poly) k->poly_mode = (uint8_t)polygon_mode(s->poly.mode);
|
|
if (!P.dyn_a2c) k->a2c = s->ms.a2c != 0;
|
|
if (P.f_logic_op && s->color.logic_op != NVN_LOGIC_OP_COPY) k->logic_op = (uint8_t)(s->color.logic_op + 1);
|
|
for (int i = 0; i < k->n_color; i++) {
|
|
int on = (s->color.blend_en >> i & 1) && blendable(k->color[i]);
|
|
if (!P.dyn_blend_en && on) k->blend_en |= (uint8_t)(1u << i);
|
|
if (!P.dyn_wmask) k->wmask |= target_wmask(s, i) << (4 * i);
|
|
if (!P.dyn_blend_eq && on) {
|
|
const blend_st *b = &s->blend[i];
|
|
uint8_t *o = k->blend[i];
|
|
o[0] = (uint8_t)blend_factor(b->src_rgb), o[1] = (uint8_t)blend_factor(b->dst_rgb), o[2] = (uint8_t)blend_op(b->eq_rgb);
|
|
o[3] = (uint8_t)blend_factor(b->src_a), o[4] = (uint8_t)blend_factor(b->dst_a), o[5] = (uint8_t)blend_op(b->eq_a);
|
|
}
|
|
}
|
|
if (is_mesh(s)) k->topo_class = 2; // mesh draws: no vertex input, the mesh stage declares its primitives
|
|
if (!P.dyn_vi && !is_mesh(s)) {
|
|
VtxInput v;
|
|
vertex_input(s, &v);
|
|
for (uint32_t i = 0; i < v.n_attr; i++) {
|
|
uint32_t l = v.a[i].location;
|
|
k->attr_fmt[l] = v.a[i].format, k->attr_off[l] = v.a[i].offset, k->attr_bind[l] = (uint8_t)v.a[i].binding;
|
|
}
|
|
for (uint32_t i = 0; i < v.n_bind; i++)
|
|
k->rate[v.b[i].binding] = v.b[i].inputRate == VK_VERTEX_INPUT_RATE_INSTANCE ? 2 : 1;
|
|
}
|
|
}
|
|
|
|
// Independent layouts: the outputs the vertex shader lacks from its union with s->fs's inputs added (vs_outs), types in
|
|
// fin. The vertex module (and its pre-rasterization library) depends on the vertex shader and this mask only.
|
|
static uint32_t vs_union_missing(const VkPipeState *s, GfxShaderIo fin[32]) {
|
|
GfxShaderIo vout[32];
|
|
uint32_t need = vs_outs(s->vs, s->fs, fin), have = gfx_shader_outputs(s->vs, vout), missing = 0;
|
|
if (s->fs) { // a location another fragment shader reads as another type: this pair's type (rare)
|
|
GfxShaderIo fi[32];
|
|
uint32_t fneed = gfx_shader_inputs(s->fs, fi);
|
|
for (int i = 0; i < 32; i++)
|
|
if (fneed >> i & 1 && !fi[i].packed &&
|
|
(!(need >> i & 1) || fin[i].type != fi[i].type || fin[i].comps < fi[i].comps)) {
|
|
if (need >> i & 1 && fin[i].type != fi[i].type)
|
|
LOG_ONCE("vk_pipeline: varying %d read as type %d and %d by fragment shaders of one vertex shader", i, fin[i].type, fi[i].type);
|
|
fin[i] = fi[i], need |= 1u << i;
|
|
}
|
|
}
|
|
if (P.vs_outs_all)
|
|
for (int i = 0; i < 32; i++)
|
|
if (!(need >> i & 1)) fin[i] = (GfxShaderIo){GFX_TYPE_FLOAT, 4, 0}, need |= 1u << i;
|
|
for (int i = 0; i < 32; i++)
|
|
if (need >> i & 1 && !(have >> i & 1)) missing |= 1u << i;
|
|
return missing;
|
|
}
|
|
|
|
static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Layout *l, uint32_t fs_off) {
|
|
VkShaderModule vm = VK_NULL_HANDLE, fm = VK_NULL_HANDLE;
|
|
uint64_t vw, fw, vm_hash = 0, fm_hash = 0; // the hashes: of the final modules, for the PIPE_FEEDBACK 2 line
|
|
const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw);
|
|
uint32_t *vpatched = NULL, *fpatched = NULL;
|
|
int mesh = is_mesh(s);
|
|
VkShaderModule tm = VK_NULL_HANDLE;
|
|
if (mesh && s->ts) { // the task stage's bindings follow the fragment stage's (pipe_miss)
|
|
uint64_t tw;
|
|
const uint32_t *tcode = gfx_shader_spirv(s->ts, &tw);
|
|
uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0);
|
|
uint32_t *t = tcode && toff ? spirv_rebind(tcode, tw, toff) : NULL;
|
|
tm = make_module(t ? t : tcode, tw, 0);
|
|
free(t);
|
|
}
|
|
if (l->set1 && !mesh) { // independent layouts: outputs from the vertex shader's union, not from this fragment shader
|
|
GfxShaderIo fin[32];
|
|
uint32_t missing = vs_union_missing(s, fin);
|
|
if (missing) {
|
|
uint64_t n2 = 0;
|
|
vpatched = spirv_add_outputs(vcode, vw, missing, fin, &n2);
|
|
if (!vpatched) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", missing);
|
|
else vcode = vpatched, vw = n2;
|
|
}
|
|
}
|
|
if (s->fs) {
|
|
GfxShaderIo fin[32], vout[32];
|
|
// a mesh stage's outputs from its module (spirv_mesh_outputs): the translation reflects none
|
|
int points = 0;
|
|
uint32_t need = gfx_shader_inputs(s->fs, fin), missing = 0,
|
|
have = mesh ? (vcode ? spirv_mesh_outputs(vcode, vw, &points) : ~0u) : gfx_shader_outputs(s->vs, vout);
|
|
for (int i = 0; i < 32; i++) {
|
|
if (!(need >> i & 1) || fin[i].packed) continue;
|
|
if (!(have >> i & 1)) missing |= 1u << i;
|
|
else if (!mesh && vout[i].type != fin[i].type)
|
|
LOG_ONCE("vk_pipeline: varying %d is type %d in the vertex stage, %d in the fragment stage", i, vout[i].type, fin[i].type);
|
|
}
|
|
if (missing && !l->set1 && !mesh) {
|
|
uint64_t n2 = 0;
|
|
vpatched = spirv_add_outputs(vcode, vw, missing, fin, &n2);
|
|
if (!vpatched) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", missing);
|
|
else vcode = vpatched, vw = n2;
|
|
}
|
|
const uint32_t *fcode = gfx_shader_spirv(s->fs, &fw);
|
|
if (missing && mesh && fcode) { // inputs no mesh output feeds read as Maxwell gives them (spirv_fs_fill_inputs)
|
|
uint64_t n2 = 0;
|
|
uint32_t *f = spirv_fs_fill_inputs(fcode, fw, missing, points, &n2);
|
|
if (f) fcode = fpatched = f, fw = n2;
|
|
else LOG_ONCE("vk_pipeline: cannot fill fragment inputs the mesh stage lacks (mask %#x)", missing);
|
|
}
|
|
if (k->dsb) {
|
|
uint64_t n2 = 0;
|
|
uint32_t *d = spirv_dual_source(fcode, fw, &n2);
|
|
if (d) free(fpatched), fcode = fpatched = d, fw = n2;
|
|
else LOG_ONCE("vk_pipeline: SRC1 blend factors but the fragment shader writes no colour output 1");
|
|
}
|
|
if (l->set1 || fs_off) {
|
|
uint32_t *r = !fcode ? NULL : l->set1 ? spirv_set(fcode, fw, 1) : spirv_rebind(fcode, fw, fs_off);
|
|
free(fpatched);
|
|
fcode = fpatched = r;
|
|
}
|
|
// A bound colour target outside the shader's output map gets write mask 0 (target_wmask)
|
|
if (fcode) fm = make_module(fcode, fw, !k->a2c), fm_hash = mm_hash;
|
|
}
|
|
vm = make_module(vcode, vw, 0), vm_hash = mm_hash;
|
|
free(vpatched), free(fpatched);
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (!vm || (s->fs && !fm) || (mesh && s->ts && !tm)) goto out;
|
|
|
|
VkPipelineShaderStageCreateInfo stages[3] = {
|
|
{.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = mesh ? VK_SHADER_STAGE_MESH_BIT_EXT : VK_SHADER_STAGE_VERTEX_BIT,
|
|
.module = vm, .pName = "main"},
|
|
{.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_FRAGMENT_BIT, .module = fm, .pName = "main"},
|
|
{.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_TASK_BIT_EXT, .module = tm, .pName = "main"}};
|
|
uint32_t n_stages = 1;
|
|
if (s->fs) n_stages = 2;
|
|
if (tm) {
|
|
if (!s->fs) stages[1] = stages[2];
|
|
n_stages++;
|
|
}
|
|
VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqv, rqf;
|
|
uint64_t sw;
|
|
const uint32_t *scode = gfx_shader_spirv(s->vs, &sw); // vcode may be a patched copy that is freed by now
|
|
pin_subgroup32(&stages[0], &rqv, scode, sw);
|
|
if (s->fs) pin_subgroup32(&stages[1], &rqf, gfx_shader_spirv(s->fs, &sw), sw);
|
|
VkVertexInputAttributeDescription va[32];
|
|
VkVertexInputBindingDescription vb[17];
|
|
uint32_t na = 0, nb = 0;
|
|
if (!P.dyn_vi) {
|
|
VtxInput v;
|
|
vertex_input(s, &v);
|
|
for (uint32_t i = 0; i < v.n_attr; i++) va[na++] = (VkVertexInputAttributeDescription){v.a[i].location, v.a[i].binding, v.a[i].format, v.a[i].offset};
|
|
for (uint32_t i = 0; i < v.n_bind; i++) vb[nb++] = (VkVertexInputBindingDescription){v.b[i].binding, v.b[i].stride, v.b[i].inputRate};
|
|
}
|
|
VkPipelineVertexInputStateCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
|
|
.vertexBindingDescriptionCount = nb, .pVertexBindingDescriptions = vb,
|
|
.vertexAttributeDescriptionCount = na, .pVertexAttributeDescriptions = va};
|
|
static const VkPrimitiveTopology class_base[4] = {VK_PRIMITIVE_TOPOLOGY_POINT_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST,
|
|
VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_PATCH_LIST};
|
|
VkPipelineInputAssemblyStateCreateInfo ia = {.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
|
|
.topology = class_base[k->topo_class]};
|
|
VkPipelineViewportDepthClipControlCreateInfoEXT clip = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT,
|
|
.negativeOneToOne = k->minus_w};
|
|
VkPipelineViewportStateCreateInfo vp = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pNext = &clip};
|
|
// NVN's default is the last vertex (SetProvokingVertexMode, nvn_ext.c); Vulkan's is the first.
|
|
VkPipelineRasterizationProvokingVertexStateCreateInfoEXT pvs = {
|
|
.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_PROVOKING_VERTEX_STATE_CREATE_INFO_EXT,
|
|
.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT};
|
|
VkPipelineRasterizationStateCreateInfo rs = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
|
|
.pNext = P.pv_last ? &pvs : NULL,
|
|
.depthClampEnable = k->clamp, .polygonMode = (VkPolygonMode)k->poly_mode,
|
|
.lineWidth = 1.0f};
|
|
VkPipelineMultisampleStateCreateInfo ms = {.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
|
|
.rasterizationSamples = (VkSampleCountFlagBits)k->samples,
|
|
.alphaToCoverageEnable = k->a2c};
|
|
VkPipelineDepthStencilStateCreateInfo ds = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
|
|
.maxDepthBounds = 1.0f};
|
|
VkPipelineColorBlendAttachmentState cba[8];
|
|
memset(cba, 0, sizeof cba);
|
|
for (int i = 0; i < k->n_color; i++) {
|
|
const uint8_t *b = k->blend[i];
|
|
cba[i].blendEnable = k->blend_en >> i & 1;
|
|
cba[i].srcColorBlendFactor = (VkBlendFactor)b[0], cba[i].dstColorBlendFactor = (VkBlendFactor)b[1];
|
|
cba[i].colorBlendOp = (VkBlendOp)b[2], cba[i].srcAlphaBlendFactor = (VkBlendFactor)b[3];
|
|
cba[i].dstAlphaBlendFactor = (VkBlendFactor)b[4], cba[i].alphaBlendOp = (VkBlendOp)b[5];
|
|
cba[i].colorWriteMask = k->wmask >> (4 * i) & 0xf;
|
|
}
|
|
VkPipelineColorBlendStateCreateInfo cb = {.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
|
|
.logicOpEnable = k->logic_op != 0,
|
|
.logicOp = k->logic_op ? (VkLogicOp)(k->logic_op - 1) : VK_LOGIC_OP_COPY,
|
|
.attachmentCount = k->n_color, .pAttachments = cba};
|
|
VkPipelineDynamicStateCreateInfo dy = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
|
|
.dynamicStateCount = P.n_dyn, .pDynamicStates = P.dyn};
|
|
VkDynamicState mdyn[sizeof P.dyn / sizeof P.dyn[0] + 1];
|
|
if (mesh) { // a mesh pipeline must not list vertex input, topology or primitive restart as dynamic
|
|
uint32_t n = 0;
|
|
for (uint32_t i = 0; i < P.n_dyn; i++)
|
|
if (P.dyn[i] != VK_DYNAMIC_STATE_VERTEX_INPUT_EXT && P.dyn[i] != VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE &&
|
|
P.dyn[i] != VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY && P.dyn[i] != VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE)
|
|
mdyn[n++] = P.dyn[i];
|
|
if (vk_ctx()->has_vrs) mdyn[n++] = VK_DYNAMIC_STATE_FRAGMENT_SHADING_RATE_KHR; // vk_exec_draw.c mesh_shading_rate
|
|
dy.dynamicStateCount = n, dy.pDynamicStates = mdyn;
|
|
}
|
|
VkPipelineRenderingCreateInfo ri = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, .colorAttachmentCount = k->n_color,
|
|
.pColorAttachmentFormats = (const VkFormat *)k->color,
|
|
.depthAttachmentFormat = (VkFormat)k->depth, .stencilAttachmentFormat = (VkFormat)k->stencil};
|
|
VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &ri,
|
|
.stageCount = n_stages, .pStages = stages, .pVertexInputState = mesh ? NULL : &vi,
|
|
.pInputAssemblyState = mesh ? NULL : &ia, .pViewportState = &vp, .pRasterizationState = &rs,
|
|
.pMultisampleState = &ms, .pDepthStencilState = &ds, .pColorBlendState = &cb,
|
|
.pDynamicState = &dy, .layout = l->layout, .flags = noopt_flag_for(mesh)};
|
|
if (k->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
|
if (k->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
|
VkPipelineCreationFeedbackCreateInfo fci;
|
|
VkPipelineCreationFeedback fb;
|
|
pi.pNext = fb_chain(&fci, &fb, pi.pNext);
|
|
uint64_t p0 = perf_begin();
|
|
crash_note("a graphics pipeline", s->vs, s->fs, s->ts);
|
|
VkResult r = pipe_create(0, &pi, &pipe);
|
|
if (r == VK_ERROR_UNKNOWN && !(pi.flags & VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT)) {
|
|
// A compiler that gives up on a pipeline (AMD's Windows driver answers VK_ERROR_UNKNOWN) may still build it
|
|
// without its optimiser: one more try that way, slower shaders for that pipeline instead of a draw left out.
|
|
static int told;
|
|
if (!told++) rt_log("vk_pipeline: the driver could not build a graphics pipeline (VK_ERROR_UNKNOWN): built again "
|
|
"without its optimiser (logged once)");
|
|
pi.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT;
|
|
r = pipe_create(0, &pi, &pipe);
|
|
}
|
|
plat_crash_context(NULL);
|
|
perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it
|
|
if (r != VK_SUCCESS) {
|
|
rt_log("vk_pipeline: vkCreateGraphicsPipelines failed (%d)", (int)r);
|
|
pipe = VK_NULL_HANDLE;
|
|
} else {
|
|
perf_count(PERF_CNT_PIPE_BUILT);
|
|
fb_note(FB_GRAPHICS, &fb);
|
|
if (P.feedback >= 2) { // the state hash: what the create info takes from the key besides the modules
|
|
uint64_t st = fnv(fnv(FNV0, cba, sizeof cba), k->color, sizeof k->color);
|
|
if (!P.dyn_vi) // the vertex layout is pipeline state then (zero in a mesh pipeline's key)
|
|
st = fnv(fnv(fnv(fnv(st, k->attr_fmt, sizeof k->attr_fmt), k->attr_off, sizeof k->attr_off),
|
|
k->attr_bind, sizeof k->attr_bind), k->rate, sizeof k->rate);
|
|
uint32_t bits[] = {(uint32_t)mesh, k->topo_class, k->minus_w, k->clamp, k->poly_mode, k->samples, k->a2c,
|
|
k->logic_op, k->n_color, k->fbl, k->dsb, l->set1 != 0, fs_off, pi.flags, k->depth, k->stencil};
|
|
rt_log("vk_pipeline: graphics: modules %016llx %016llx, state %016llx (mesh %d, topology class %u, samples %u, "
|
|
"colors %u, feedback loop %u, depth format %u, stage-independent layout %d, flags 0x%x), %s, %.1f ms",
|
|
(unsigned long long)vm_hash, (unsigned long long)fm_hash, (unsigned long long)fnv(st, bits, sizeof bits),
|
|
mesh, (unsigned)k->topo_class, (unsigned)k->samples, (unsigned)k->n_color, (unsigned)k->fbl,
|
|
(unsigned)k->depth, l->set1 != 0, (unsigned)pi.flags,
|
|
pb_last ? "from pipeline-binaries.bin" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled",
|
|
(double)fb.duration / 1e6);
|
|
}
|
|
}
|
|
out:
|
|
if (vm) vkDestroyShaderModule(P.device, vm, NULL);
|
|
if (fm) vkDestroyShaderModule(P.device, fm, NULL);
|
|
if (tm) vkDestroyShaderModule(P.device, tm, NULL);
|
|
return pipe;
|
|
}
|
|
|
|
// ---- graphics pipeline library (vk_caps.gpl)
|
|
// A pipeline is linked from four libraries, each cached by the part of the pipeline key it depends on: vertex input
|
|
// (topology class, vertex attributes without dynamic vertex input), pre-rasterization (vertex shader, the outputs its
|
|
// union adds, static rasterization state), fragment shader (fragment shader, dual-source rewrite, samples) and fragment
|
|
// output (targets, blend, samples). With independent layouts each stage's set depends on its shader alone, so the
|
|
// pre-rasterization library uses {set 0, null} and the fragment shader library {null, set 1}. The fast link runs on the
|
|
// thread of the draw that needs it (no link-time optimisation); the optimised pipeline is the job pipe_miss queued
|
|
// (a monolithic build with the same modules), which replaces the fast one in the Pipe when done. Libraries and fast
|
|
// pipelines live until vk_pipeline_shutdown.
|
|
// Background build signalling, both with P.lock: work_cv wakes the pipeline workers, done_cv wakes threads waiting for a
|
|
// pipeline or a stage library to be published.
|
|
static PlatCond work_cv = PLAT_COND_INIT, done_cv = PLAT_COND_INIT;
|
|
static PlatCond lib_cv = PLAT_COND_INIT; // wakes the library workers (lib_worker): work_cv's signal must reach a pool worker
|
|
#define LIB_BUCKETS 1024
|
|
enum { LIB_VI, LIB_PR, LIB_FS, LIB_FO };
|
|
typedef struct Lib { struct Lib *next; uint64_t hash; VkPipeline pipe; uint32_t size; uint8_t pre, used, building, pad;
|
|
struct LibJob *job; uint8_t key[]; } Lib;
|
|
// pre: built by the boot prebuild ("stage library prebuild" below); used: a draw's link found it; building: a thread
|
|
// builds it or job holds it queued (pipe not set yet); entries live until libs_destroy
|
|
static Lib *libs[LIB_BUCKETS];
|
|
|
|
typedef struct { uint32_t kind, topo_class, attr_fmt[32], attr_off[32]; uint8_t attr_bind[32], rate[17], pad[3]; } ViKey;
|
|
typedef struct { uint32_t kind, missing; const GfxShader *vs; uint8_t clamp, poly_mode, minus_w, pad; GfxShaderIo fin[32]; } PrKey;
|
|
typedef struct { uint32_t kind; uint8_t dsb, samples, a2c, pad; const GfxShader *fs; } FsKey;
|
|
typedef struct { uint32_t kind; uint8_t n_color, samples, a2c, logic_op, blend_en, fbl, pad[2]; uint32_t color[8], depth, stencil, wmask;
|
|
uint8_t blend[8][6]; } FoKey; // fbl: GKey.fbl, the feedback loop flags of the library (gpl_feedback)
|
|
|
|
static uint64_t qpc(void) { return plat_now_ns(); }
|
|
static double ms_since(uint64_t t0) { return (double)(qpc() - t0) / 1e6; }
|
|
|
|
static VkPipelineLayout half_layout(Layout *l, int h) {
|
|
if (l->half[h]) return l->half[h];
|
|
VkDescriptorSetLayout sets[3] = {h ? VK_NULL_HANDLE : l->set, h ? l->set1 : VK_NULL_HANDLE, vk_pool_set_layout()};
|
|
VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
|
|
.flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 3,
|
|
.pSetLayouts = sets};
|
|
if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->half[h]) != VK_SUCCESS) l->half[h] = VK_NULL_HANDLE;
|
|
return l->half[h];
|
|
}
|
|
|
|
// Builds one library from the pipeline's state; key is the library's key (PrKey: fin holds the added outputs).
|
|
static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPipelineLayout lay, const void *key) {
|
|
static const VkGraphicsPipelineLibraryFlagsEXT part[4] = {
|
|
VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT, VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT,
|
|
VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT, VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT};
|
|
VkGraphicsPipelineLibraryCreateInfoEXT gl = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, .flags = part[kind]};
|
|
VkPipelineRenderingCreateInfo ri = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, .pNext = &gl};
|
|
VkPipelineDynamicStateCreateInfo dy = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
|
|
.dynamicStateCount = P.n_dyn, .pDynamicStates = P.dyn};
|
|
VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &ri,
|
|
.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | noopt_flag(), .pDynamicState = &dy};
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
VkPipelineShaderStageCreateInfo st = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pName = "main"};
|
|
VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rq;
|
|
VkVertexInputAttributeDescription va[32];
|
|
VkVertexInputBindingDescription vb[17];
|
|
VkPipelineVertexInputStateCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO};
|
|
static const VkPrimitiveTopology class_base[4] = {VK_PRIMITIVE_TOPOLOGY_POINT_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST,
|
|
VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_PATCH_LIST};
|
|
VkPipelineInputAssemblyStateCreateInfo ia = {.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
|
|
.topology = class_base[k->topo_class]};
|
|
VkPipelineViewportDepthClipControlCreateInfoEXT clip = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_DEPTH_CLIP_CONTROL_CREATE_INFO_EXT,
|
|
.negativeOneToOne = k->minus_w};
|
|
VkPipelineViewportStateCreateInfo vp = {.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pNext = &clip};
|
|
VkPipelineRasterizationProvokingVertexStateCreateInfoEXT pvs = {
|
|
.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_PROVOKING_VERTEX_STATE_CREATE_INFO_EXT,
|
|
.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT};
|
|
VkPipelineRasterizationStateCreateInfo rs = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
|
|
.pNext = P.pv_last ? &pvs : NULL, .depthClampEnable = k->clamp,
|
|
.polygonMode = (VkPolygonMode)k->poly_mode, .lineWidth = 1.0f};
|
|
VkPipelineMultisampleStateCreateInfo ms = {.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
|
|
.rasterizationSamples = (VkSampleCountFlagBits)k->samples,
|
|
.alphaToCoverageEnable = k->a2c};
|
|
VkPipelineDepthStencilStateCreateInfo ds = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
|
|
.maxDepthBounds = 1.0f};
|
|
VkPipelineColorBlendAttachmentState cba[8];
|
|
VkPipelineColorBlendStateCreateInfo cb = {.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO};
|
|
uint32_t *patched = NULL;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (kind == LIB_VI) {
|
|
if (!P.dyn_vi) {
|
|
VtxInput v;
|
|
vertex_input(s, &v);
|
|
for (uint32_t i = 0; i < v.n_attr; i++)
|
|
va[i] = (VkVertexInputAttributeDescription){v.a[i].location, v.a[i].binding, v.a[i].format, v.a[i].offset};
|
|
for (uint32_t i = 0; i < v.n_bind; i++)
|
|
vb[i] = (VkVertexInputBindingDescription){v.b[i].binding, v.b[i].stride, v.b[i].inputRate};
|
|
vi.vertexAttributeDescriptionCount = v.n_attr, vi.pVertexAttributeDescriptions = va;
|
|
vi.vertexBindingDescriptionCount = v.n_bind, vi.pVertexBindingDescriptions = vb;
|
|
}
|
|
pi.pVertexInputState = &vi, pi.pInputAssemblyState = &ia;
|
|
} else if (kind == LIB_PR) {
|
|
const PrKey *pk = key;
|
|
uint64_t vw, sw, n2 = 0;
|
|
const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw), *scode = vcode;
|
|
sw = vw;
|
|
if (pk->missing && (patched = spirv_add_outputs(vcode, vw, pk->missing, pk->fin, &n2))) vcode = patched, vw = n2;
|
|
else if (pk->missing) LOG_ONCE("vk_pipeline: cannot add missing vertex outputs (mask %#x)", pk->missing);
|
|
if (!(m = make_module(vcode, vw, 0))) goto out;
|
|
st.stage = VK_SHADER_STAGE_VERTEX_BIT, st.module = m;
|
|
pin_subgroup32(&st, &rq, scode, sw);
|
|
pi.stageCount = 1, pi.pStages = &st, pi.pViewportState = &vp, pi.pRasterizationState = &rs;
|
|
pi.layout = lay;
|
|
} else if (kind == LIB_FS) {
|
|
if (s->fs) {
|
|
uint64_t fw, sw, n2 = 0;
|
|
const uint32_t *fcode = gfx_shader_spirv(s->fs, &fw), *scode = fcode;
|
|
sw = fw;
|
|
uint32_t *d = NULL;
|
|
if (k->dsb && fcode) {
|
|
if ((d = spirv_dual_source(fcode, fw, &n2))) fcode = d, fw = n2;
|
|
else LOG_ONCE("vk_pipeline: SRC1 blend factors but the fragment shader writes no colour output 1");
|
|
}
|
|
patched = fcode ? spirv_set(fcode, fw, 1) : NULL;
|
|
free(d);
|
|
if (!patched || !(m = make_module(patched, fw, !k->a2c))) goto out;
|
|
st.stage = VK_SHADER_STAGE_FRAGMENT_BIT, st.module = m;
|
|
pin_subgroup32(&st, &rq, scode, sw);
|
|
pi.stageCount = 1, pi.pStages = &st;
|
|
}
|
|
pi.pMultisampleState = &ms, pi.pDepthStencilState = &ds;
|
|
pi.layout = lay;
|
|
} else {
|
|
memset(cba, 0, sizeof cba);
|
|
for (int i = 0; i < k->n_color; i++) {
|
|
const uint8_t *b = k->blend[i];
|
|
cba[i].blendEnable = k->blend_en >> i & 1;
|
|
cba[i].srcColorBlendFactor = (VkBlendFactor)b[0], cba[i].dstColorBlendFactor = (VkBlendFactor)b[1];
|
|
cba[i].colorBlendOp = (VkBlendOp)b[2], cba[i].srcAlphaBlendFactor = (VkBlendFactor)b[3];
|
|
cba[i].dstAlphaBlendFactor = (VkBlendFactor)b[4], cba[i].alphaBlendOp = (VkBlendOp)b[5];
|
|
cba[i].colorWriteMask = k->wmask >> (4 * i) & 0xf;
|
|
}
|
|
cb.logicOpEnable = k->logic_op != 0, cb.logicOp = k->logic_op ? (VkLogicOp)(k->logic_op - 1) : VK_LOGIC_OP_COPY;
|
|
cb.attachmentCount = k->n_color, cb.pAttachments = cba;
|
|
ri.colorAttachmentCount = k->n_color, ri.pColorAttachmentFormats = (const VkFormat *)k->color;
|
|
ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil;
|
|
pi.pColorBlendState = &cb, pi.pMultisampleState = &ms;
|
|
// Whether the pipeline may sample an attachment it renders to is part of the fragment output state: the flags
|
|
// go on this library alone, and a pipeline linked from it has them (gpl_feedback).
|
|
const FoKey *ok = key;
|
|
if (ok->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
|
if (ok->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
|
|
}
|
|
VkPipelineCreationFeedbackCreateInfo fci;
|
|
VkPipelineCreationFeedback fb;
|
|
pi.pNext = fb_chain(&fci, &fb, pi.pNext);
|
|
crash_note("a pipeline library", s->vs, s->fs, NULL);
|
|
if (pipe_create(1, &pi, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
|
plat_crash_context(NULL);
|
|
if (pipe && (kind == LIB_PR || kind == LIB_FS)) fb_note(FB_LIBRARY, &fb); // the shader libraries: the others compile nothing
|
|
out:
|
|
if (m) vkDestroyShaderModule(P.device, m, NULL);
|
|
free(patched);
|
|
return pipe;
|
|
}
|
|
|
|
// The cached library for key. Caller holds P.lock. build 0: lookup only (VK_NULL_HANDLE and *miss when absent);
|
|
// build 1: a missing one is built with P.lock released (a failed build is cached too).
|
|
// The library entry for key, NULL when absent. Caller holds P.lock.
|
|
static Lib *lib_find(uint64_t h, const void *key, uint32_t size) {
|
|
for (Lib *b = libs[h % LIB_BUCKETS]; b; b = b->next)
|
|
if (b->hash == h && b->size == size && !memcmp(b->key, key, size)) return b;
|
|
return NULL;
|
|
}
|
|
|
|
// Enters a library built with P.lock released; another thread's entry made meanwhile wins and pipe is destroyed.
|
|
// Returns the entered handle. Caller holds P.lock.
|
|
static VkPipeline lib_put(uint64_t h, const void *key, uint32_t size, VkPipeline pipe) {
|
|
Lib *b = lib_find(h, key, size);
|
|
if (b && b->building) { // another thread's build publishes the entry
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
if (b) {
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
return b->pipe;
|
|
}
|
|
if (!(b = calloc(1, sizeof *b + size))) return pipe;
|
|
b->hash = h, b->size = size, b->pipe = pipe;
|
|
memcpy(b->key, key, size);
|
|
b->next = libs[h % LIB_BUCKETS];
|
|
libs[h % LIB_BUCKETS] = b;
|
|
return pipe;
|
|
}
|
|
|
|
// Shader libraries a waiting draw needs (lib_wait, link_queue): queued here ahead of every pipeline job, so an idle
|
|
// worker builds each at once and the vertex and fragment libraries of one draw build in parallel. A job no worker took
|
|
// is built by the first thread that needs its library (lib_get; lib_wait only without library workers). A job with p
|
|
// is a waiting draw's fast link of p (after its library jobs), run on a worker so the drawing thread only waits.
|
|
// Under P.lock.
|
|
typedef struct LibJob { struct LibJob *next; Lib *b; int kind; Layout *l; VkPipeState s; GKey k; Pipe *p; } LibJob;
|
|
static struct {
|
|
LibJob *head, *tail;
|
|
int idle; // workers asleep on work_cv
|
|
int running; // jobs workers are building
|
|
int on; // NX2NAVI_PIPE_WAIT: 1 workers (default), 0 the drawing thread builds them (draw);
|
|
// set by vk_pipeline_async_enable, before the replay runs
|
|
int lib_workers; // workers that take only these jobs (lib_worker), started with the pool
|
|
uint64_t waits, queued, by_worker, stolen, timeouts, links;
|
|
} U;
|
|
|
|
// A building entry for key, NULL when out of memory. Caller holds P.lock and found none.
|
|
static Lib *lib_new(uint64_t h, const void *key, uint32_t size) {
|
|
Lib *b = calloc(1, sizeof *b + size);
|
|
if (!b) return NULL;
|
|
b->hash = h, b->size = size, b->building = 1;
|
|
memcpy(b->key, key, size);
|
|
b->next = libs[h % LIB_BUCKETS];
|
|
libs[h % LIB_BUCKETS] = b;
|
|
return b;
|
|
}
|
|
|
|
// Builds b (building, not queued) with P.lock released and publishes it; a failed build is entered as VK_NULL_HANDLE.
|
|
static VkPipeline lib_build(Lib *b, int kind, const VkPipeState *s, const GKey *k, Layout *l) {
|
|
VkPipelineLayout lay = kind == LIB_PR ? half_layout(l, 0) : kind == LIB_FS ? half_layout(l, 1) : VK_NULL_HANDLE;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (lay || kind == LIB_VI || kind == LIB_FO) {
|
|
plat_mutex_unlock(&P.lock);
|
|
uint64_t t0 = qpc();
|
|
pipe = lib_create(kind, s, k, lay, b->key);
|
|
double ms = ms_since(t0);
|
|
plat_mutex_lock(&P.lock);
|
|
P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms;
|
|
if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms;
|
|
int added = kind == LIB_PR && ((const PrKey *)(const void *)b->key)->missing;
|
|
if (added) P.g.pr_added++, P.g.pr_added_ms += ms;
|
|
if (ms >= 25) P.g.lib_slow[kind]++, P.g.lib_slow_ms[kind] += ms;
|
|
// A shader library that took this long was not in the driver's cache; why not, for the first 32 of a run
|
|
if (ms >= 25 && (kind == LIB_PR || kind == LIB_FS) && P.g.lib_logged < 32) {
|
|
const GfxShader *sh = kind == LIB_PR ? s->vs : s->fs;
|
|
const uint8_t *id = sh ? gfx_shader_input_id(sh) : NULL;
|
|
P.g.lib_logged++;
|
|
rt_log("vk_pipeline: gpl: a %s library took %.0f ms: shader %02x%02x%02x%02x, %s%s",
|
|
kind == LIB_PR ? "pre-rasterization" : "fragment shader", ms, id ? id[0] : 0, id ? id[1] : 0,
|
|
id ? id[2] : 0, id ? id[3] : 0, origin_name(sh),
|
|
added ? "; outputs were added for its fragment shader, a module the preparation did not build"
|
|
: sh && gfx_shader_origin(sh) == 2 ? "; nothing was prepared from a shader first translated now"
|
|
: "; the driver's cache did not have it, or the build waited behind others in the driver");
|
|
}
|
|
}
|
|
if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind);
|
|
b->pipe = pipe, b->building = 0;
|
|
plat_cond_broadcast(&done_cv);
|
|
return pipe;
|
|
}
|
|
|
|
// Takes b's queued job out of the queue and builds it on this thread. Caller holds P.lock.
|
|
static void lib_steal(Lib *b) {
|
|
LibJob *j = b->job, **pp = &U.head, *prev = NULL;
|
|
while (*pp && *pp != j) prev = *pp, pp = &(*pp)->next;
|
|
if (*pp) *pp = j->next;
|
|
if (U.tail == j) U.tail = prev;
|
|
b->job = NULL;
|
|
lib_build(b, j->kind, &j->s, &j->k, j->l);
|
|
free(j);
|
|
}
|
|
|
|
static VkPipeline lib_get(int kind, const VkPipeState *s, const GKey *k, Layout *l, const void *key, uint32_t size,
|
|
int build, int *miss) {
|
|
uint64_t h = key_hash(key, size);
|
|
Lib *have = lib_find(h, key, size);
|
|
while (have && have->building) { // being built (wait) or queued (build it here)
|
|
if (!build) {
|
|
*miss = 1;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
if (have->job) U.stolen++, lib_steal(have);
|
|
else plat_cond_wait(&done_cv, &P.lock);
|
|
}
|
|
if (have) {
|
|
if (have->pre && !have->used) have->used = 1, P.g.pre_used[kind]++;
|
|
return have->pipe;
|
|
}
|
|
VkPipelineLayout lay = kind == LIB_PR ? half_layout(l, 0) : kind == LIB_FS ? half_layout(l, 1) : VK_NULL_HANDLE;
|
|
if (!build || ((kind == LIB_PR || kind == LIB_FS) && !lay)) {
|
|
*miss = 1;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
Lib *b = lib_new(h, key, size); // entered first, so other threads wait for this build instead of repeating it
|
|
if (b) return lib_build(b, kind, s, k, l);
|
|
plat_mutex_unlock(&P.lock);
|
|
uint64_t t0 = qpc();
|
|
VkPipeline pipe = lib_create(kind, s, k, lay, key);
|
|
double ms = ms_since(t0);
|
|
plat_mutex_lock(&P.lock);
|
|
if ((have = lib_find(h, key, size)) && !have->building) { // another thread built it meanwhile
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
return have->pipe;
|
|
}
|
|
P.g.lib_n[kind]++, P.g.lib_ms[kind] += ms;
|
|
if (ms > P.g.lib_max[kind]) P.g.lib_max[kind] = ms;
|
|
if (!pipe && ++P.g.lib_fail <= 8) rt_log("vk_pipeline: gpl: stage library %d not built", kind);
|
|
return lib_put(h, key, size, pipe);
|
|
}
|
|
|
|
static void libs_destroy(void) {
|
|
for (int i = 0; i < LIB_BUCKETS; i++) {
|
|
for (Lib *b = libs[i], *n; b; b = n) {
|
|
n = b->next;
|
|
if (b->pipe) vkDestroyPipeline(P.device, b->pipe, NULL);
|
|
free(b);
|
|
}
|
|
libs[i] = NULL;
|
|
}
|
|
}
|
|
|
|
// The four library keys of pipeline s (key k).
|
|
static void lib_keys(const VkPipeState *s, const GKey *k, ViKey *v, PrKey *pk, FsKey *fk, FoKey *ok) {
|
|
memset(v, 0, sizeof *v);
|
|
v->kind = LIB_VI, v->topo_class = k->topo_class;
|
|
memcpy(v->attr_fmt, k->attr_fmt, sizeof v->attr_fmt), memcpy(v->attr_off, k->attr_off, sizeof v->attr_off);
|
|
memcpy(v->attr_bind, k->attr_bind, sizeof v->attr_bind), memcpy(v->rate, k->rate, sizeof v->rate);
|
|
memset(pk, 0, sizeof *pk);
|
|
GfxShaderIo fin[32];
|
|
pk->kind = LIB_PR, pk->vs = s->vs, pk->clamp = k->clamp, pk->poly_mode = k->poly_mode, pk->minus_w = k->minus_w;
|
|
pk->missing = vs_union_missing(s, fin);
|
|
for (int i = 0; i < 32; i++)
|
|
if (pk->missing >> i & 1) pk->fin[i] = fin[i];
|
|
memset(fk, 0, sizeof *fk);
|
|
fk->kind = LIB_FS, fk->fs = s->fs, fk->dsb = k->dsb, fk->samples = k->samples, fk->a2c = k->a2c;
|
|
memset(ok, 0, sizeof *ok);
|
|
ok->kind = LIB_FO, ok->n_color = k->n_color, ok->samples = k->samples, ok->a2c = k->a2c, ok->logic_op = k->logic_op;
|
|
ok->blend_en = k->blend_en, ok->fbl = k->fbl, ok->depth = k->depth, ok->stencil = k->stencil, ok->wmask = k->wmask;
|
|
memcpy(ok->color, k->color, sizeof ok->color), memcpy(ok->blend, k->blend, sizeof ok->blend);
|
|
}
|
|
|
|
// NX2NAVI_GPL_FEEDBACK (default 1; 0 = as before): a pipeline whose fragment shader samples an attachment of its own pass
|
|
// (VkPipeState.feedback: soft particles and water reading the depth buffer, decals, full-screen passes that read the
|
|
// target they write) gets a fast link like any other. The feedback loop flags belong to the fragment output state, so
|
|
// they are part of that library's key (FoKey.fbl) and of nothing else; the shader libraries are the ones the same
|
|
// shaders use without a loop. Before, such a pipeline had no fast link at all and its draws waited for, or were
|
|
// skipped during, the whole optimised build. On a first visit to Kokiri Forest (RTX 5090, every stage library in the
|
|
// driver's cache, DLSS Frame Generation on, so no frame is held back) full-screen draws of such pipelines waited
|
|
// 267 ms at the area's first frame and 410 ms in the middle of a walk; linked, 3 ms.
|
|
static int gpl_feedback(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
on = cfg_bool("NX2NAVI_GPL_FEEDBACK");
|
|
if (!on) rt_log("vk_pipeline: gpl: pipelines with a feedback loop get no fast link (NX2NAVI_GPL_FEEDBACK=0)");
|
|
}
|
|
return on;
|
|
}
|
|
|
|
// A fast-linked pipeline for s from its stage libraries. Caller holds P.lock.
|
|
// build 0 (drawing thread): links only when the shader libraries exist (vertex input and fragment output libraries,
|
|
// which hold no shader, are built here); *final 0 = not yet, a worker builds them. build 1 (worker): builds the
|
|
// missing libraries and links with P.lock released. VK_NULL_HANDLE with *final 1 when GPL cannot express the pipeline
|
|
// (shared layout; a feedback loop with NX2NAVI_GPL_FEEDBACK=0) or a build failed.
|
|
static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc, int build, int *final) {
|
|
Layout *l = (Layout *)lc;
|
|
*final = 1;
|
|
if (!l->set1 || (k->fbl && !gpl_feedback())) return VK_NULL_HANDLE;
|
|
uint64_t t0 = qpc();
|
|
ViKey vk_;
|
|
PrKey pk;
|
|
FsKey fk;
|
|
FoKey ok;
|
|
lib_keys(s, k, &vk_, &pk, &fk, &ok);
|
|
int miss = 0;
|
|
VkPipeline lib[4];
|
|
lib[1] = lib_get(LIB_PR, s, k, l, &pk, sizeof pk, build, &miss); // shader libraries first: no work when absent
|
|
lib[2] = miss ? VK_NULL_HANDLE : lib_get(LIB_FS, s, k, l, &fk, sizeof fk, build, &miss);
|
|
if (miss) {
|
|
if (!build) *final = 0, P.g.defer++;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
lib[0] = lib_get(LIB_VI, s, k, l, &vk_, sizeof vk_, 1, &miss);
|
|
lib[3] = lib_get(LIB_FO, s, k, l, &ok, sizeof ok, 1, &miss);
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (lib[0] && lib[1] && lib[2] && lib[3]) {
|
|
VkPipelineLibraryCreateInfoKHR li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR, .libraryCount = 4,
|
|
.pLibraries = lib};
|
|
VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &li,
|
|
.layout = l->layout};
|
|
if (build) plat_mutex_unlock(&P.lock);
|
|
uint64_t t1 = qpc();
|
|
crash_note("a library link", s->vs, s->fs, NULL);
|
|
if (vkCreateGraphicsPipelines(P.device, VK_NULL_HANDLE, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
|
plat_crash_context(NULL);
|
|
double ms = ms_since(t1); // the link alone: library builds are counted apart
|
|
if (build) plat_mutex_lock(&P.lock);
|
|
P.g.link_ms += ms;
|
|
if (ms > P.g.link_max) P.g.link_max = ms;
|
|
}
|
|
if (pipe) P.g.links++, P.g.fb_links += k->fbl != 0;
|
|
else if (!P.g.link_fail++) rt_log("vk_pipeline: gpl: fast link failed; such draws wait or skip as without GPL");
|
|
double total = ms_since(t0);
|
|
if (!build && total > 5.0 && P.g.slow++ < 16)
|
|
rt_log("vk_pipeline: gpl: fast link on the drawing thread took %.1f ms", total);
|
|
return pipe;
|
|
}
|
|
|
|
// ---- background graphics pipeline builds (vk_pipeline_async_enable)
|
|
// A miss in vk_pipeline_graphics_async enters the Pipe as building and queues a job; the replay skips draws with it
|
|
// until a worker publishes the handle. The job holds copies of the state and the key. What they point to outlives it:
|
|
// shaders (gfx_shader_get results) live for the process (gfx_shader_drop_memory is the shader probe's, which builds no
|
|
// pipelines), layouts until vk_pipeline_shutdown, which drains the queue first. Workers call vkCreateGraphicsPipelines
|
|
// without P.lock: P.cache is created without VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT, so the driver
|
|
// synchronises builds into it; only the copy of its data in cache_save is kept apart from them (cache_rw). Compute
|
|
// pipelines stay synchronous: a skipped
|
|
// dispatch can change what the game computes.
|
|
//
|
|
// Two queues: draws' misses (A.head) and the boot prebuild (A.low, see "pipeline keys" below). Workers take the low
|
|
// queue only when the other is empty and run its jobs at PLAT_PRIO_BELOW_NORMAL, so guest threads win. A draw
|
|
// that finds a prebuild job still queued moves it to the draw queue; one that finds it running raises its worker.
|
|
typedef struct Job {
|
|
struct Job *next, *prev; // prev: low queue only
|
|
Pipe *p;
|
|
const Layout *l;
|
|
uint32_t fs_off;
|
|
uint8_t low; // in the low queue, or running at below-normal priority
|
|
uint8_t prebuild, demanded; // from the key file; a draw asked for it before it was ready
|
|
int worker; // running on A.thread[worker]; -1 queued; JOB_ON_DRAW on the thread that waits for it
|
|
VkPipeState s;
|
|
GKey k;
|
|
} Job;
|
|
|
|
#define MAX_WORKERS 4
|
|
#define JOB_ON_DRAW (-2)
|
|
|
|
static struct {
|
|
int on; // set once by vk_pipeline_async_enable, before the replay runs
|
|
int threads;
|
|
PlatThread *thread[MAX_WORKERS];
|
|
uint8_t prio_low[MAX_WORKERS]; // worker runs below normal (under P.lock)
|
|
Job *head, *tail; // draws' misses
|
|
Job *low, *low_tail; // prebuild
|
|
uint32_t inflight; // queued or running, both queues
|
|
int low_running; // pool workers running a job taken from the low queue
|
|
int gate_shut; // pool workers the memory gate has refused a low job since it last admitted one
|
|
uint64_t first, taken; // waited-for jobs moved to the head of the draw queue; built by the waiting thread
|
|
PlatCond idle; // with P.lock: no job queued or running
|
|
} A = {.idle = PLAT_COND_INIT};
|
|
|
|
// ---- pipeline keys (pipeline-keys.bin next to the pipeline cache)
|
|
// Every graphics pipeline built in a game run appends a KeyRec: the VkPipeState it was built from with the shader
|
|
// pointers zeroed and the shaders named by their gfx_shader input ids instead, which do not depend on the translator,
|
|
// so the record rebuilds the pipeline without guest memory. At the next boot vk_pipeline_prewarm (rt_boot, before the
|
|
// guest runs; NX2NAVI_PIPE_PREBUILD=0 skips it) loads each record's shaders, translating the ones not cached under the
|
|
// current translator from their stored inputs (gfx_shader_prewarm; a record whose shader has neither is dropped),
|
|
// enters each Pipe as building under the key a draw would compute, queues it to the low queue and builds the queue on
|
|
// every core. Records are deduplicated by a hash of their bytes; they are collected under P.lock by the worker that
|
|
// built the pipeline and appended by the present thread (vk_pipeline_autosave) or at exit (vk_pipeline_save). A file
|
|
// with another magic, version or record size is ignored and rewritten; a version 3 file (no task stage) is converted
|
|
// and rewritten. Bump KEYS_VERSION when VkPipeState or the record changes meaning without changing size.
|
|
#define KEYS_VERSION 4 // 3: shaders named by input id (gfx_shader.c); 4: mesh pipelines, with their task stage
|
|
typedef struct { char magic[8]; uint32_t version, rec_size; } KeysHeader;
|
|
typedef struct {
|
|
uint8_t vs[32], fs[32];
|
|
uint32_t has_fs, pad;
|
|
VkPipeState s;
|
|
} KeyRecV3;
|
|
typedef struct {
|
|
uint8_t vs[32], fs[32], ts[32]; // gfx_shader_input_id of each stage (vs: the mesh stage of a mesh pipeline); fs all
|
|
// zero with has_fs 0, ts with has_ts 0
|
|
uint32_t has_fs, has_ts;
|
|
VkPipeState s; // vs, fs, ts zero
|
|
} KeyRec;
|
|
static struct {
|
|
char path[PLAT_PATH_MAX * 2];
|
|
int header_ok; // the file exists with our header: append
|
|
uint64_t *seen; // open addressing set of record hashes (0 empty)
|
|
uint32_t seen_cap, seen_n;
|
|
KeyRec *pend; // records not written yet
|
|
uint32_t pend_n, pend_cap;
|
|
uint8_t *file; // the file as read at boot, until the prewarm is done with it
|
|
size_t file_n; // records in it
|
|
PlatMutex io; // one writer
|
|
// prebuild progress (under P.lock)
|
|
int loading; // the prewarm still queues records
|
|
int warm; // vk_pipeline_prewarm owns the file (prebuild_start leaves it)
|
|
uint32_t left; // prebuild jobs queued or running
|
|
uint32_t built, failed, promoted;
|
|
uint32_t gate_ended; // boot builders the memory gate ended (warm_builder)
|
|
double t0; // now_ms() when the prebuild started
|
|
double until; // now_ms() after which the boot builders stop (comment at prewarm_run); 0 none
|
|
} K = {.io = PLAT_MUTEX_INIT};
|
|
|
|
static uint64_t rec_hash(const KeyRec *r) {
|
|
uint64_t h = fnv(FNV0, r, sizeof *r);
|
|
return h ? h : 1;
|
|
}
|
|
|
|
// 1 if h was new. Caller holds P.lock.
|
|
static int seen_add(uint64_t h) {
|
|
if ((K.seen_n + 1) * 2 > K.seen_cap) {
|
|
uint32_t cap = K.seen_cap ? K.seen_cap * 2 : 4096;
|
|
uint64_t *t = calloc(cap, sizeof *t);
|
|
if (!t) return 1;
|
|
for (uint32_t i = 0; i < K.seen_cap; i++) {
|
|
if (!K.seen[i]) continue;
|
|
uint32_t j = (uint32_t)K.seen[i] & (cap - 1);
|
|
while (t[j]) j = (j + 1) & (cap - 1);
|
|
t[j] = K.seen[i];
|
|
}
|
|
free(K.seen);
|
|
K.seen = t, K.seen_cap = cap;
|
|
}
|
|
uint32_t j = (uint32_t)h & (K.seen_cap - 1);
|
|
while (K.seen[j]) {
|
|
if (K.seen[j] == h) return 0;
|
|
j = (j + 1) & (K.seen_cap - 1);
|
|
}
|
|
K.seen[j] = h;
|
|
K.seen_n++;
|
|
return 1;
|
|
}
|
|
|
|
static void rec_fill(KeyRec *r, const VkPipeState *s) {
|
|
memset(r, 0, sizeof *r);
|
|
memcpy(r->vs, gfx_shader_input_id(s->vs), 32);
|
|
if (s->fs) memcpy(r->fs, gfx_shader_input_id(s->fs), 32), r->has_fs = 1;
|
|
if (s->ts) memcpy(r->ts, gfx_shader_input_id(s->ts), 32), r->has_ts = 1;
|
|
r->s = *s;
|
|
r->s.vs = r->s.fs = r->s.ts = NULL;
|
|
r->s.prim_restart = 0; // dynamic state: the same pipeline either way
|
|
}
|
|
|
|
// A pipeline was built for a draw: queue its record for the file. Caller holds P.lock.
|
|
static void keys_note(const VkPipeState *s) {
|
|
if (!A.on || !K.path[0]) return;
|
|
static const uint8_t none[32];
|
|
if (s->fs && !memcmp(gfx_shader_input_id(s->fs), none, 32)) return; // the runtime's own shader (gfx_shader_own)
|
|
if (s->ts && !memcmp(gfx_shader_input_id(s->ts), none, 32)) return;
|
|
if (!memcmp(gfx_shader_input_id(s->vs), none, 32)) return; // the mesh fallback's vertex stage (vk_exec_draw.c mfb_vs_get)
|
|
KeyRec r;
|
|
rec_fill(&r, s);
|
|
if (!seen_add(rec_hash(&r))) return;
|
|
if (K.pend_n == K.pend_cap) {
|
|
uint32_t cap = K.pend_cap ? K.pend_cap * 2 : 64;
|
|
KeyRec *t = realloc(K.pend, cap * sizeof *t);
|
|
if (!t) return;
|
|
K.pend = t, K.pend_cap = cap;
|
|
}
|
|
K.pend[K.pend_n++] = r;
|
|
}
|
|
|
|
// Appends the pending records (present thread or exit; never the render thread).
|
|
static void keys_flush(void) {
|
|
if (!K.path[0]) return;
|
|
plat_mutex_lock(&K.io);
|
|
plat_mutex_lock(&P.lock);
|
|
KeyRec *r = K.pend;
|
|
uint32_t n = K.pend_n;
|
|
K.pend = NULL, K.pend_n = K.pend_cap = 0;
|
|
plat_mutex_unlock(&P.lock);
|
|
if (n) {
|
|
FILE *f = fopen(K.path, K.header_ok ? "ab" : "wb");
|
|
int ok = f != NULL;
|
|
if (ok && !K.header_ok) {
|
|
KeysHeader h = {{'N', 'X', '2', 'N', 'P', 'K', 'E', 'Y'}, KEYS_VERSION, sizeof(KeyRec)};
|
|
ok = fwrite(&h, sizeof h, 1, f) == 1;
|
|
}
|
|
if (ok) ok = fwrite(r, sizeof *r, n, f) == n;
|
|
if (ok) ok = plat_file_sync(f); // appended records on the disk before the next append
|
|
if (f) ok = !fclose(f) && ok;
|
|
if (ok) K.header_ok = 1;
|
|
else rt_log("vk_pipeline: cannot write %s", K.path);
|
|
}
|
|
free(r);
|
|
plat_mutex_unlock(&K.io);
|
|
}
|
|
|
|
// Reads the key file into K.file and its record hashes into the set. Caller holds P.lock (ensure_init).
|
|
static void keys_open(void) {
|
|
if (!A.on || !P.cache_path[0]) return;
|
|
snprintf(K.path, sizeof K.path, "%s", P.cache_path);
|
|
char *slash = strrchr(K.path, '\\'), *fslash = strrchr(K.path, '/');
|
|
if (fslash > slash) slash = fslash;
|
|
snprintf(slash ? slash + 1 : K.path, sizeof K.path - (size_t)(slash ? slash + 1 - K.path : 0), "pipeline-keys.bin");
|
|
gfx_shader_record_at(K.path); // shader-answers.bin next to it (gfx_shader.c)
|
|
FILE *f = fopen(K.path, "rb");
|
|
if (!f) return;
|
|
KeysHeader h;
|
|
fseek(f, 0, SEEK_END);
|
|
long len = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
int head = len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8);
|
|
if (head && h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec)) {
|
|
K.header_ok = 1;
|
|
size_t n = ((size_t)len - sizeof h) / sizeof(KeyRec);
|
|
if (n && (K.file = malloc(n * sizeof(KeyRec))) && fread(K.file, sizeof(KeyRec), n, f) == n) K.file_n = n;
|
|
else free(K.file), K.file = NULL;
|
|
for (size_t i = 0; i < K.file_n; i++) seen_add(rec_hash((const KeyRec *)K.file + i));
|
|
if (len != (long)(sizeof h + n * sizeof(KeyRec))) // a torn append: rewrite the file from what was read
|
|
K.header_ok = 0;
|
|
} else if (head && h.version == 3 && h.rec_size == sizeof(KeyRecV3)) {
|
|
// the vertex pipelines of a version 3 file keep their prewarm: each record without a task stage, and the
|
|
// file rewritten in this version below (header_ok 0)
|
|
size_t n = ((size_t)len - sizeof h) / sizeof(KeyRecV3);
|
|
KeyRecV3 *old = n ? malloc(n * sizeof *old) : NULL;
|
|
if (old && fread(old, sizeof *old, n, f) == n && (K.file = calloc(n, sizeof(KeyRec)))) {
|
|
KeyRec *r = (KeyRec *)K.file;
|
|
for (size_t i = 0; i < n; i++) {
|
|
memcpy(r[i].vs, old[i].vs, 32), memcpy(r[i].fs, old[i].fs, 32);
|
|
r[i].has_fs = old[i].has_fs, r[i].s = old[i].s;
|
|
seen_add(rec_hash(&r[i]));
|
|
}
|
|
K.file_n = n;
|
|
rt_log("vk_pipeline: %s: %zu version 3 records converted; the file is rewritten", K.path, n);
|
|
}
|
|
free(old);
|
|
} else {
|
|
rt_log("vk_pipeline: %s has another format or version; it is rewritten", K.path);
|
|
}
|
|
fclose(f);
|
|
if (!K.header_ok && K.file_n) { // keep the whole records of a torn file
|
|
KeyRec *t = malloc(K.file_n * sizeof *t);
|
|
if (t) memcpy(t, K.file, K.file_n * sizeof *t), K.pend = t, K.pend_n = K.pend_cap = (uint32_t)K.file_n;
|
|
}
|
|
}
|
|
|
|
static double now_ms(void) { return (double)plat_now_ns() / 1e6; }
|
|
|
|
// Caller holds P.lock.
|
|
// p, which a draw missed, has its first usable pipeline: how long that took is how long its draws were skipped or
|
|
// waited. fast 0: it had no fast link and this is the optimised build. Caller holds P.lock.
|
|
// The summary (save_with) counts what a slow one can be: a mesh pipeline (no stage libraries, so its whole build), one
|
|
// the key file names whose prebuild had not come to it, one with a shader translated in this session (nothing of it
|
|
// was prepared, so its libraries compile cold). The first READY_LOGS that took READY_SLOW_MS or longer get a line
|
|
// each with those facts, so a log says which pipelines left their draws out of the picture and for which reason.
|
|
#define READY_SLOW_MS 50.0
|
|
#define READY_LOGS 48
|
|
static void ready_note(Pipe *p, int fast) {
|
|
if (!p->t_miss) return;
|
|
double ms = (double)(plat_now_ns() - p->t_miss) / 1e6;
|
|
p->t_miss = 0;
|
|
P.g.ready_n[ms < 4 ? 0 : ms < 8 ? 1 : ms < 17 ? 2 : ms < 50 ? 3 : ms < 150 ? 4 : 5]++;
|
|
P.g.ready_ms += ms;
|
|
if (ms > P.g.ready_max) P.g.ready_max = ms;
|
|
P.g.ready_nofast += !fast;
|
|
const Job *j = p->job;
|
|
if (!j) return;
|
|
int mesh = is_mesh(&j->s);
|
|
int fresh = gfx_shader_origin(j->s.vs) == 2 || (j->s.fs && gfx_shader_origin(j->s.fs) == 2) ||
|
|
(j->s.ts && gfx_shader_origin(j->s.ts) == 2);
|
|
P.g.ready_mesh += (uint64_t)mesh, P.g.ready_recorded += j->prebuild, P.g.ready_fresh += (uint64_t)fresh;
|
|
if (ms < READY_SLOW_MS || P.g.ready_logged >= READY_LOGS) return;
|
|
P.g.ready_logged++;
|
|
const uint8_t *vi = gfx_shader_input_id(j->s.vs), *fi = j->s.fs ? gfx_shader_input_id(j->s.fs) : NULL;
|
|
char fid[16] = "none";
|
|
if (fi) snprintf(fid, sizeof fid, "%02x%02x%02x%02x", fi[0], fi[1], fi[2], fi[3]);
|
|
rt_log("vk_pipeline: a pipeline a draw missed was usable after %.0f ms: a %s pipeline, %s; %s; %s shader "
|
|
"%02x%02x%02x%02x (%s), fragment shader %s (%s)%s%s",
|
|
ms, mesh ? "mesh" : "vertex",
|
|
fast ? "fast-linked, so the time went to its stage libraries (the lines \"a ... library took\" name the slow ones)"
|
|
: mesh ? "which has no stage libraries: this is its whole build"
|
|
: "which got no fast link: this is its whole build",
|
|
j->prebuild ? "pipeline-keys.bin names it and its prebuild had not come to it" : "not in pipeline-keys.bin before",
|
|
mesh ? "mesh" : "vertex", vi[0], vi[1], vi[2], vi[3], origin_name(j->s.vs), fid, origin_name(j->s.fs),
|
|
j->k.fbl ? "; feedback loop" : "", P.g.ready_logged == READY_LOGS ? " (the last such line of this run)" : "");
|
|
}
|
|
|
|
static void prebuild_report(void) {
|
|
rt_log("vk_pipeline: prebuild done: %u built, %u failed, %u asked for by a draw first; %.0f ms after the device was "
|
|
"ready", K.built, K.failed, K.promoted, now_ms() - K.t0);
|
|
}
|
|
|
|
static void job_lower(Job *j);
|
|
|
|
// Runs a waiting draw's job taken from U on a worker: a library build or, for a job with p, p's fast link (gpl_link
|
|
// build 1 builds a library still missing and waits for one another worker builds). Releases P.lock while building.
|
|
static void u_run(LibJob *lj) {
|
|
U.by_worker++, U.running++;
|
|
if (lj->p) {
|
|
Pipe *p = lj->p;
|
|
if (!p->info.pipeline) {
|
|
int final = 0;
|
|
VkPipeline f = gpl_link(&lj->s, &lj->k, lj->l, 1, &final); // releases P.lock while linking
|
|
if (f && !p->info.pipeline) {
|
|
p->fast = p->info.pipeline = f;
|
|
ready_note(p, 1);
|
|
job_lower(p->job); // its job now only does the optimised build
|
|
} else if (f) {
|
|
vkDestroyPipeline(P.device, f, NULL); // the job published one meanwhile
|
|
}
|
|
}
|
|
p->link_queued = 0;
|
|
U.links++;
|
|
plat_cond_broadcast(&done_cv);
|
|
} else {
|
|
lib_build(lj->b, lj->kind, &lj->s, &lj->k, lj->l);
|
|
}
|
|
U.running--;
|
|
free(lj);
|
|
}
|
|
|
|
// NX2NAVI_PIPE_FAST_WORKERS (default 1; 0 = as before): the library workers also take the fast link of a draw-queue job
|
|
// (fast_take). A draw that is skipped rather than waiting leaves its pipeline to the pool, where a job is first
|
|
// fast-linked and then goes back in the queue for its optimised build. The pool keeps one worker out of optimised
|
|
// builds, so with the others inside one (about 100 ms each with the driver's cache warm, 350 ms cold) that single
|
|
// worker linked every new pipeline of a burst, one after another at 5 to 10 ms each. On a first visit to Kokiri
|
|
// Forest (RTX 5090, libraries in the driver's cache) 180 of the 444 pipelines draws missed took over 50 ms to become
|
|
// usable, the longest 1.3 s, and their draws were missing from every frame until then; with the library workers
|
|
// linking, 23 did, 19 of them mesh pipelines, which have no fast link.
|
|
static int fast_workers(void) {
|
|
static int on = -1;
|
|
if (on < 0) on = cfg_bool("NX2NAVI_PIPE_FAST_WORKERS");
|
|
return on;
|
|
}
|
|
|
|
// The first draw-queue job whose pipeline still lacks its fast link, taken out of the queue; NULL when there is none.
|
|
// Caller holds P.lock.
|
|
static Job *fast_take(void) {
|
|
if (!P.gpl || !fast_workers()) return NULL;
|
|
for (Job *j = A.head, *prev = NULL; j; prev = j, j = j->next) {
|
|
Pipe *p = j->p;
|
|
if (p->info.pipeline || p->fast_tried || p->link_queued || !p->lay || is_mesh(&j->s)) continue;
|
|
if (prev) prev->next = j->next;
|
|
else A.head = j->next;
|
|
if (A.tail == j) A.tail = prev;
|
|
j->next = j->prev = NULL;
|
|
return j;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
// A worker for stage libraries and fast links only: a waiting draw's jobs (U) first, so a draw that waits finds one
|
|
// free even while the pool builds pipelines and the drawing thread never builds a library itself; then the fast link
|
|
// of a draw-queue job (fast_take), whose optimised build goes on to the pool.
|
|
static void lib_worker(void *arg) {
|
|
(void)arg;
|
|
plat_mutex_lock(&P.lock);
|
|
for (;;) {
|
|
Job *j = NULL;
|
|
while (!U.head && !(j = fast_take())) plat_cond_wait(&lib_cv, &P.lock);
|
|
if (!j) {
|
|
LibJob *lj = U.head;
|
|
if (!(U.head = lj->next)) U.tail = NULL;
|
|
if (lj->b) lj->b->job = NULL;
|
|
u_run(lj);
|
|
continue;
|
|
}
|
|
Pipe *p = j->p;
|
|
int final;
|
|
p->fast_tried = 1; // no other thread links it meanwhile; a draw that waits for it waits for this link
|
|
VkPipeline f = gpl_link(&j->s, &j->k, p->lay, 1, &final); // releases P.lock while building
|
|
if (f && !p->info.pipeline) {
|
|
p->fast = p->info.pipeline = f, P.g.wlinks++, P.g.flinks++;
|
|
ready_note(p, 1);
|
|
plat_cond_broadcast(&done_cv);
|
|
} else if (f) {
|
|
vkDestroyPipeline(P.device, f, NULL);
|
|
}
|
|
if (p->info.pipeline) { // the optimised build: ahead of the prebuild, behind other draws' misses
|
|
j->low = 1, P.g.requeued++;
|
|
j->next = A.low;
|
|
if (A.low) A.low->prev = j;
|
|
else A.low_tail = j;
|
|
A.low = j;
|
|
} else { // no fast link for this one: its whole build, next in the draw queue
|
|
if (!(j->next = A.head)) A.tail = j;
|
|
A.head = j;
|
|
}
|
|
plat_cond_signal(&work_cv);
|
|
}
|
|
}
|
|
|
|
// ---- the memory gate of the background builds (vk_build_gate.h)
|
|
// Every pipeline built ahead of its first use passes here before it starts: the recorded pipelines the boot builds
|
|
// (warm_builder), the pool's low queue during play (pipe_worker: the rest of the recorded pipelines and the optimised
|
|
// builds behind a fast link), the compute pipelines and stage libraries of the boot (lb_worker), the library warm
|
|
// (lw_worker) and the large compute shaders (bg_main). A build a draw or a dispatch waits for never does: it is needed
|
|
// now.
|
|
// NX2NAVI_BUILD_FREE_MB (default 4096; 0 no check): the system commit headroom that stays free of them.
|
|
// NX2NAVI_BUILD_RESERVE_MB (default 64): what one build under way is taken to cost the driver's compiler while it
|
|
// runs; a large compute shader reserves GATE_BIG_MB. A build starts only when the headroom holds the part kept free,
|
|
// the reservations of the builds under way and its own.
|
|
// Measured on the machine this was written on (RTX 5090, 73,154 MB commit limit, warm driver cache): the boot's 3,898
|
|
// recorded pipelines on 16 threads raised the process commit by 3,054 MB in 2.2 s, and the 4,066 compute pipelines and
|
|
// libraries after them, together with the game's own start, by another 9,824 MB. A device was lost on that machine
|
|
// at 56,673 of 65,531 MB system commit right after such a prebuild.
|
|
// The commit is read at most every GATE_SAMPLE_MS: sixteen builders finishing a cached pipeline every half millisecond
|
|
// would otherwise ask the system thousands of times a second. What the builds took since the last reading is covered
|
|
// by their reservations.
|
|
#define GATE_BIG_MB 4096.0
|
|
#define GATE_SAMPLE_MS 100.0
|
|
#define GATE_WAIT_MS 30000.0 // how long a boot or warm builder waits for room before it leaves the rest to their first use
|
|
static struct {
|
|
PlatMutex lock;
|
|
BuildGate g;
|
|
double est, used, limit, at;
|
|
int init;
|
|
int off; // a preparation run (seed_pipelines): nothing is refused, there is no game to keep room for
|
|
uint64_t refused;
|
|
} GATE = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
// 1 when a build estimated at est MB may start, and then est is reserved until gate_leave. est 0: an ordinary build
|
|
// (NX2NAVI_BUILD_RESERVE_MB), whose estimate *took returns for gate_leave.
|
|
static int gate_enter(double est, double *took) {
|
|
plat_mutex_lock(&GATE.lock);
|
|
if (!GATE.init) {
|
|
GATE.init = 1;
|
|
GATE.g.free_mb = strtod(cfg_value("NX2NAVI_BUILD_FREE_MB"), NULL);
|
|
GATE.est = strtod(cfg_value("NX2NAVI_BUILD_RESERVE_MB"), NULL);
|
|
if (GATE.est < 0) GATE.est = 0;
|
|
GATE.at = -GATE_SAMPLE_MS;
|
|
}
|
|
if (GATE.off) GATE.g.free_mb = 0;
|
|
double now = now_ms();
|
|
if (now - GATE.at >= GATE_SAMPLE_MS) {
|
|
double proc;
|
|
plat_commit_info(&proc, &GATE.used, &GATE.limit);
|
|
GATE.at = now;
|
|
}
|
|
if (est <= 0) est = GATE.est;
|
|
int ok = build_gate_enter(&GATE.g, GATE.used, GATE.limit, est);
|
|
if (!ok) GATE.refused++;
|
|
plat_mutex_unlock(&GATE.lock);
|
|
if (took) *took = est;
|
|
return ok;
|
|
}
|
|
|
|
static void gate_leave(double est) {
|
|
plat_mutex_lock(&GATE.lock);
|
|
build_gate_leave(&GATE.g, est);
|
|
plat_mutex_unlock(&GATE.lock);
|
|
}
|
|
|
|
// The gate's numbers for a log line that says why a builder stopped or waits.
|
|
static const char *gate_text(char *buf, size_t n) {
|
|
plat_mutex_lock(&GATE.lock);
|
|
snprintf(buf, n, "system commit %.0f of %.0f MB, %.0f MB reserved for builds under way, %.0f MB kept free "
|
|
"(NX2NAVI_BUILD_FREE_MB), %.0f MB a build (NX2NAVI_BUILD_RESERVE_MB)", GATE.used, GATE.limit,
|
|
GATE.g.reserved_mb, GATE.g.free_mb, GATE.est);
|
|
plat_mutex_unlock(&GATE.lock);
|
|
return buf;
|
|
}
|
|
|
|
// A boot or warm builder's wait for room: 1 with est reserved (*took), 0 when *stop was set or GATE_WAIT_MS passed
|
|
// without room, which the caller takes as the end of its work. who names the builder for the log's one line a wait.
|
|
static int gate_wait(const char *who, volatile int32_t *stop, double *took) {
|
|
double w0 = now_ms();
|
|
int waited = 0;
|
|
while (!gate_enter(0, took)) {
|
|
char g[256];
|
|
if (*stop) return 0;
|
|
if (!waited++) rt_log("vk_pipeline: %s: waiting for commit headroom (%s)", who, gate_text(g, sizeof g));
|
|
if (now_ms() - w0 > GATE_WAIT_MS) return 0;
|
|
for (int i = 0; i < 20 && !*stop; i++) plat_sleep_ms(100);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
// --probe-buildgate (host.h): the gate with this machine's commit and the settings in effect, without a device and
|
|
// without a build. It prints what the settings and the system give, asks for sixteen ordinary builds and one large
|
|
// one, and checks the answers against the same numbers: the builds admitted are those that fit with the earlier
|
|
// ones' reservations, a refused one reserves nothing, and a builder's wait ends at once when its stop flag is set.
|
|
// tests/build_gate_check.py runs it with build_free_mb at 0 (everything admitted) and at a value no machine has free
|
|
// (everything refused).
|
|
int vk_pipeline_probe_buildgate(void) {
|
|
double took = 0, big = 0;
|
|
int fails = 0, in = 0, want = 0;
|
|
int first = gate_enter(0, &took);
|
|
double free_mb = GATE.g.free_mb, used = GATE.used, limit = GATE.limit, est = took;
|
|
if (first) gate_leave(took);
|
|
printf("probe-buildgate: system commit %.0f of %.0f MB, %.0f MB kept free (NX2NAVI_BUILD_FREE_MB), %.0f MB a build "
|
|
"(NX2NAVI_BUILD_RESERVE_MB)\n", used, limit, free_mb, est);
|
|
double room = limit - used - free_mb;
|
|
for (int i = 0; i < 16; i++) {
|
|
want += free_mb <= 0 || limit < 0 || room - i * est >= est; // the commit sample stays for GATE_SAMPLE_MS
|
|
in += gate_enter(0, &took);
|
|
}
|
|
double reserved = GATE.g.reserved_mb;
|
|
int big_in = gate_enter(GATE_BIG_MB, &big);
|
|
int big_want = free_mb <= 0 || limit < 0 || room - reserved >= GATE_BIG_MB;
|
|
printf("probe-buildgate: %d of 16 ordinary builds admitted (%d fit), %.0f MB reserved; a large compute build beside "
|
|
"them %s\n", in, want, reserved, big_in ? "admitted" : "refused");
|
|
if (in != want) fails++, printf("probe-buildgate: FAIL: %d admitted, %d fit\n", in, want);
|
|
if (reserved != in * est) fails++, printf("probe-buildgate: FAIL: %.0f MB reserved for %d builds\n", reserved, in);
|
|
if (big_in != big_want) fails++, printf("probe-buildgate: FAIL: the large build was %s\n", big_in ? "admitted" : "refused");
|
|
if (big_in) gate_leave(big);
|
|
for (int i = 0; i < in; i++) gate_leave(est);
|
|
if (GATE.g.reserved_mb != 0) fails++, printf("probe-buildgate: FAIL: %.0f MB still reserved\n", GATE.g.reserved_mb);
|
|
volatile int32_t stop = 1;
|
|
double t0 = now_ms();
|
|
int got = gate_wait("probe", &stop, &took);
|
|
if (got) gate_leave(took);
|
|
if (got != first || now_ms() - t0 > 1000)
|
|
fails++, printf("probe-buildgate: FAIL: a stopped builder's wait gave %d after %.0f ms\n", got, now_ms() - t0);
|
|
printf("probe-buildgate: an ordinary build is %s now; %s\n", first ? "admitted" : "refused", fails ? "FAILED" : "passed");
|
|
return fails ? 1 : 0;
|
|
}
|
|
|
|
// The whole build of job j, which the caller took off its queue, and the job's end: its pipeline is entered, the
|
|
// threads waiting for it are woken and j is freed. lowrun: 1 on a pool worker that took j from the low queue. Caller
|
|
// holds P.lock, which is released for the build.
|
|
static void job_build(Job *j, int lowrun) {
|
|
plat_mutex_unlock(&P.lock);
|
|
VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off);
|
|
plat_mutex_lock(&P.lock);
|
|
if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one
|
|
ready_note(j->p, 0);
|
|
if (pipe && j->p->fast) P.g.swaps++;
|
|
j->p->building = 0;
|
|
j->p->job = NULL;
|
|
if (pipe) {
|
|
P.stats.graphics++, P.stats.async_built++;
|
|
if (!j->prebuild) keys_note(&j->s);
|
|
} else {
|
|
P.stats.failures++;
|
|
rt_log("vk_pipeline: graphics pipeline not built: pipeline creation failed");
|
|
}
|
|
if (j->prebuild) {
|
|
if (pipe) K.built++;
|
|
else K.failed++;
|
|
K.promoted += j->demanded;
|
|
if (!--K.left && !K.loading) prebuild_report();
|
|
}
|
|
A.low_running -= lowrun;
|
|
if (!--A.inflight) plat_cond_broadcast(&A.idle);
|
|
plat_cond_broadcast(&done_cv);
|
|
if (pipe && (!j->prebuild || j->demanded)) perf_count(PERF_CNT_PIPE_ASYNC); // a draw waited for it
|
|
free(j);
|
|
}
|
|
|
|
static void pipe_worker(void *arg) {
|
|
int me = (int)(intptr_t)arg;
|
|
plat_mutex_lock(&P.lock);
|
|
for (;;) {
|
|
// With 3 or more workers one never takes low-queue work, so a waiting draw's library starts at once
|
|
// (not with NX2NAVI_PIPE_WAIT=draw).
|
|
while (!U.head && !A.head && !(A.low && (A.threads < 3 || !U.on || A.low_running < A.threads - 1))) {
|
|
U.idle++;
|
|
plat_cond_wait(&work_cv, &P.lock);
|
|
U.idle--;
|
|
}
|
|
if (U.head) { // a library a draw waits for
|
|
LibJob *lj = U.head;
|
|
if (!(U.head = lj->next)) U.tail = NULL;
|
|
if (lj->b) lj->b->job = NULL;
|
|
if (A.prio_low[me]) A.prio_low[me] = 0, plat_thread_priority(PLAT_PRIO_NORMAL);
|
|
u_run(lj); // releases P.lock while building
|
|
continue;
|
|
}
|
|
Job *j = A.head;
|
|
int lowrun = !j;
|
|
double held = 0; // the low job's reservation at the memory gate
|
|
if (!j && !gate_enter(0, &held)) {
|
|
// No room for a background build now (gate_enter). The job stays queued: a draw that needs its pipeline
|
|
// moves it to the draw queue (promote), which wakes this wait; otherwise the gate is asked again in 2 s.
|
|
if (!A.gate_shut++) {
|
|
char g[256];
|
|
rt_log("vk_pipeline: background pipeline builds wait for commit headroom (%s); a pipeline a draw needs "
|
|
"is still built at once", gate_text(g, sizeof g));
|
|
}
|
|
plat_cond_wait_ns(&work_cv, &P.lock, 2000000000ull);
|
|
continue;
|
|
}
|
|
if (!j && A.gate_shut) {
|
|
A.gate_shut = 0;
|
|
rt_log("vk_pipeline: background pipeline builds go on: the commit headroom is back");
|
|
}
|
|
if (j) {
|
|
if (!(A.head = j->next)) A.tail = NULL;
|
|
} else {
|
|
j = A.low;
|
|
if ((A.low = j->next)) A.low->prev = NULL;
|
|
else A.low_tail = NULL;
|
|
A.low_running++;
|
|
}
|
|
j->next = j->prev = NULL;
|
|
j->worker = me;
|
|
if (j->low != A.prio_low[me]) { // under P.lock, so a draw raising this thread (promote) is not undone
|
|
A.prio_low[me] = j->low;
|
|
plat_thread_priority(j->low ? PLAT_PRIO_BELOW_NORMAL : PLAT_PRIO_NORMAL);
|
|
}
|
|
// Step 1 for a job a draw waits for: its stage libraries and the fast link. Step 2, the optimised build, goes to
|
|
// the head of the low queue, so a burst of new pipelines gets every fast link before any optimised build.
|
|
if (P.gpl && (!j->prebuild || j->demanded) && !j->p->info.pipeline && !j->p->fast_tried && j->p->lay &&
|
|
!is_mesh(&j->s)) {
|
|
int final;
|
|
VkPipeline f = gpl_link(&j->s, &j->k, j->p->lay, 1, &final); // releases P.lock while building
|
|
j->p->fast_tried = 1;
|
|
if (f && !j->p->info.pipeline) {
|
|
j->p->fast = j->p->info.pipeline = f, P.g.wlinks++;
|
|
ready_note(j->p, 1);
|
|
plat_cond_broadcast(&done_cv);
|
|
j->low = 1, j->worker = -1, P.g.requeued++;
|
|
j->prev = NULL, j->next = A.low;
|
|
if (A.low) A.low->prev = j;
|
|
else A.low_tail = j;
|
|
A.low = j;
|
|
A.low_running -= lowrun;
|
|
if (lowrun) gate_leave(held);
|
|
continue;
|
|
} else if (f) {
|
|
vkDestroyPipeline(P.device, f, NULL);
|
|
}
|
|
}
|
|
job_build(j, lowrun);
|
|
if (lowrun) gate_leave(held);
|
|
}
|
|
return;
|
|
}
|
|
|
|
static int hint_on(void);
|
|
|
|
// Starts the pool on first use: a quarter of the logical processors, 2 to 4. Caller holds P.lock.
|
|
static int workers_start(void) {
|
|
if (A.threads) return 1;
|
|
uint32_t cpus = plat_cpu_usable();
|
|
int n = cpus / 4 < 2 ? 2 : cpus / 4 > MAX_WORKERS ? MAX_WORKERS : (int)(cpus / 4);
|
|
for (int i = 0; i < n; i++) {
|
|
PlatThread *t = plat_thread_new(pipe_worker, (void *)(intptr_t)A.threads, "pipe worker");
|
|
if (!t) break;
|
|
A.thread[A.threads++] = t; // kept: promote() raises a worker running a prebuild job
|
|
}
|
|
// Workers for stage libraries and fast links alone: two (the vertex and fragment libraries of one waiting draw in
|
|
// parallel), and as many as the pool has with the look-ahead, which queues the libraries of a whole submission's
|
|
// new pipelines at once (vk_pipeline_hint), or with the fast links of skipped draws (lib_worker). They sleep
|
|
// unless a library or a fast link is queued.
|
|
int nlib = hint_on() || fast_workers() ? n : 2;
|
|
for (int i = 0; A.threads && U.on && P.gpl && i < nlib; i++)
|
|
if (plat_thread_new(lib_worker, NULL, "pipe lib worker")) U.lib_workers++;
|
|
if (A.threads)
|
|
rt_log("vk_pipeline: %d background graphics pipeline workers, %d for waiting draws' libraries", A.threads,
|
|
U.lib_workers);
|
|
else rt_log("vk_pipeline: cannot start pipeline workers; building synchronously");
|
|
return A.threads != 0;
|
|
}
|
|
|
|
static void workers_drain(void) {
|
|
plat_mutex_lock(&P.lock);
|
|
K.loading = 0;
|
|
for (Job *j = A.low, *n; j; j = n) { // queued prebuild jobs are dropped, not built
|
|
n = j->next;
|
|
j->p->building = 0, j->p->job = NULL;
|
|
A.inflight--, K.left--;
|
|
free(j);
|
|
}
|
|
A.low = A.low_tail = NULL;
|
|
for (LibJob *j = U.head, *n; j; j = n) { // queued library jobs too: their entries read as failed builds
|
|
n = j->next;
|
|
j->b->job = NULL, j->b->building = 0;
|
|
free(j);
|
|
}
|
|
U.head = U.tail = NULL;
|
|
while (A.inflight) plat_cond_wait(&A.idle, &P.lock);
|
|
while (U.running) plat_cond_wait(&done_cv, &P.lock);
|
|
plat_mutex_unlock(&P.lock);
|
|
}
|
|
|
|
void vk_pipeline_async_enable(void) {
|
|
U.on = strcmp(cfg_value("NX2NAVI_PIPE_WAIT"), "draw") != 0;
|
|
const char *e = cfg_get("NX2NAVI_PIPE_SYNC");
|
|
if (e && *e && strcmp(e, "0")) return;
|
|
A.on = 1;
|
|
}
|
|
|
|
int vk_pipeline_async_on(void) { return A.on; }
|
|
|
|
// A draw needs a pipeline a prebuild job owns: the job moves to the draw queue, or its worker to normal priority.
|
|
// Caller holds P.lock.
|
|
static void promote(Pipe *p) {
|
|
Job *j = p->job;
|
|
if (!j || !j->low) return;
|
|
if (j->prebuild && !j->demanded && !p->info.pipeline) p->t_miss = plat_now_ns();
|
|
j->low = 0, j->demanded = 1;
|
|
if (j->worker == -1) {
|
|
if (j->prev) j->prev->next = j->next;
|
|
else A.low = j->next;
|
|
if (j->next) j->next->prev = j->prev;
|
|
else A.low_tail = j->prev;
|
|
j->next = j->prev = NULL;
|
|
if (A.tail) A.tail->next = j;
|
|
else A.head = j;
|
|
A.tail = j;
|
|
plat_cond_signal(&work_cv);
|
|
if (U.lib_workers) plat_cond_signal(&lib_cv); // its fast link (lib_worker)
|
|
} else if (j->worker >= 0 && A.prio_low[j->worker]) {
|
|
A.prio_low[j->worker] = 0;
|
|
plat_thread_set_priority(A.thread[j->worker], PLAT_PRIO_NORMAL);
|
|
}
|
|
}
|
|
|
|
// A draw waits up to wait_ms for the pipeline of job j, which has no fast link (a mesh pipeline, a pipeline without
|
|
// stage libraries, or one whose link failed) and so needs its whole build. While the job is still queued the wait
|
|
// buys nothing: the draw queue is first in, first out, and the jobs before it are the fast links and builds of other
|
|
// draws' misses, so on a burst of new pipelines the one a draw stood waiting for started last and the draw was
|
|
// skipped at the end of its wait all the same.
|
|
// - The job goes to the head of the draw queue and a worker is woken, so the next free worker takes it.
|
|
// - A wait longer than TAKE_WAIT_MS is not one of a frame's budgets (vk_exec_draw.c frame_wait is at most 500 ms): the
|
|
// caller wants this pipeline whatever it takes (a one-shot draw with NX2NAVI_ONESHOT_BUDGET=0). With no worker idle
|
|
// the job is then built here at once (job_build), as a library no worker took is (lib_wait). A wait inside a
|
|
// budget is never turned into a build on this thread: a compile takes 0.2 to 1.1 s for a mesh pipeline, and the
|
|
// budget exists so that the replay thread is not held that long.
|
|
// Caller holds P.lock; it is released while the job is built here.
|
|
#define TAKE_WAIT_MS 500u
|
|
static void job_wanted(Job *j, uint32_t wait_ms) {
|
|
if (!j || j->worker != -1 || j->low) return; // running already, or gone
|
|
Job *q = NULL;
|
|
if (A.head != j) {
|
|
for (q = A.head; q && q->next != j; q = q->next) {}
|
|
if (!q) return;
|
|
}
|
|
if (wait_ms > TAKE_WAIT_MS && !U.idle) {
|
|
if (q) {
|
|
if (!(q->next = j->next)) A.tail = q;
|
|
} else if (!(A.head = j->next)) {
|
|
A.tail = NULL;
|
|
}
|
|
j->next = j->prev = NULL;
|
|
j->worker = JOB_ON_DRAW;
|
|
A.taken++;
|
|
job_build(j, 0);
|
|
return;
|
|
}
|
|
if (q) { // to the head
|
|
if (!(q->next = j->next)) A.tail = q;
|
|
j->next = A.head, A.head = j;
|
|
A.first++;
|
|
}
|
|
plat_cond_signal(&work_cv);
|
|
}
|
|
|
|
// Enters a missing pipeline: built here (mode 0), or queued for a draw (1) or for the prebuild (2). Failures are
|
|
// entered too. *queued: 1 when a job owns it. Caller holds P.lock.
|
|
static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode, int *queued) {
|
|
*queued = 0;
|
|
const char *why = NULL;
|
|
Bindings B = {0};
|
|
Layout *l = NULL;
|
|
uint32_t span = binding_span(s->vs);
|
|
int mesh = is_mesh(s);
|
|
if (mesh) {
|
|
// Mesh draws (VK_EXT_mesh_shader): one set holding the mesh stage's bindings, the fragment stage's after them and
|
|
// the task stage's after those (create_graphics renumbers both modules); no libraries (the GPL fast link is for
|
|
// vertex pipelines). They go to the workers like the others: built here, each new one stopped the frame for its
|
|
// whole compile (0.26-0.51 s in Kokiri Forest), and they are recorded for the boot prewarm (keys_note).
|
|
uint32_t fspan = s->fs ? binding_span(s->fs) : 1;
|
|
if (!vk_ctx()->has_mesh_shader) why = "the GPU has no mesh shaders";
|
|
else if ((s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS) || (s->ts && gfx_shader_stage(s->ts) != GFX_STAGE_TS)) why = "stage mismatch";
|
|
else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1)) ||
|
|
(s->ts && !add_stage(&B, s->ts, span - 1 + fspan - 1))) why = "too many bindings";
|
|
else if (!(l = layout_get(&B))) why = "descriptor layout creation failed";
|
|
} else if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) why = "stage mismatch";
|
|
else if (k->topo_class == 3) why = "patches need tessellation stages, which are not translated";
|
|
else if (P.indep) {
|
|
Bindings F = {0};
|
|
if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&F, s->fs, 0))) why = "too many bindings";
|
|
else if (!(l = layout_pair(&B, &F))) why = "descriptor layout creation failed";
|
|
B.pool |= F.pool;
|
|
span = 1;
|
|
} else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1))) why = "too many bindings";
|
|
else if (!(l = layout_get(&B))) why = "descriptor layout creation failed";
|
|
char lack[128];
|
|
if (!why && !layout_covers(s, l, span - 1, lack, sizeof lack)) why = lack, l = NULL;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
Job *j = NULL;
|
|
if (!why && mode && workers_start() && (j = calloc(1, sizeof *j))) {
|
|
*j = (Job){.l = l, .fs_off = span - 1, .low = mode == 2, .prebuild = mode == 2, .worker = -1, .s = *s, .k = *k};
|
|
} else if (!why && !(pipe = create_graphics(s, k, l, span - 1))) {
|
|
why = "pipeline creation failed";
|
|
}
|
|
if (why) {
|
|
P.stats.failures++;
|
|
rt_log("vk_pipeline: graphics pipeline not built: %s", why);
|
|
} else if (!j) {
|
|
P.stats.graphics++;
|
|
if (mode != 2) keys_note(s);
|
|
}
|
|
Pipe *p = pipe_add(k, sizeof *k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_GRAPHICS, gfx_shader_inputs(s->vs, NULL));
|
|
if (!j || !p) {
|
|
free(j);
|
|
return p;
|
|
}
|
|
j->p = p;
|
|
p->building = 1, p->job = j;
|
|
if (mode != 2) p->t_miss = plat_now_ns();
|
|
if (mode == 2) {
|
|
j->prev = A.low_tail;
|
|
if (A.low_tail) A.low_tail->next = j;
|
|
else A.low = j;
|
|
A.low_tail = j;
|
|
K.left++;
|
|
} else {
|
|
if (A.tail) A.tail->next = j;
|
|
else A.head = j;
|
|
A.tail = j;
|
|
if (U.lib_workers && fast_workers()) plat_cond_signal(&lib_cv); // its fast link (lib_worker)
|
|
}
|
|
A.inflight++;
|
|
plat_cond_signal(&work_cv);
|
|
*queued = 1;
|
|
return p;
|
|
}
|
|
|
|
// A job still in the draw queue goes to the head of the low queue: its pipeline has (or is about to have) a fast-linked
|
|
// one, so the optimised build waits behind other draws' misses. Caller holds P.lock.
|
|
static void job_lower(Job *j) {
|
|
if (!j || j->worker != -1 || j->low) return;
|
|
if (A.head == j) {
|
|
if (!(A.head = j->next)) A.tail = NULL;
|
|
} else {
|
|
Job *q = A.head;
|
|
while (q && q->next != j) q = q->next;
|
|
if (!q) return;
|
|
if (!(q->next = j->next)) A.tail = q;
|
|
}
|
|
j->low = 1, P.g.requeued++;
|
|
j->prev = NULL, j->next = A.low;
|
|
if (A.low) A.low->prev = j;
|
|
else A.low_tail = j;
|
|
A.low = j;
|
|
}
|
|
|
|
// A draw waits for pipeline s: its missing pre-rasterization and fragment shader libraries go to the head of the
|
|
// worker queues (U), so they build in parallel on idle workers, and this thread sleeps until both are published or
|
|
// end (now_ms) passes. A job no worker took within 1 ms (all busy) is built here. Caller holds P.lock.
|
|
static void u_push(LibJob *j) {
|
|
if (U.tail) U.tail->next = j;
|
|
else U.head = j;
|
|
U.tail = j;
|
|
}
|
|
|
|
// Queues s's missing pre-rasterization and fragment shader libraries (b: both entries). 0 when one cannot be entered
|
|
// (no layout half, out of memory). Caller holds P.lock.
|
|
static int lib_queue(const VkPipeState *s, const GKey *k, Layout *l, Lib *b[2]) {
|
|
ViKey vk_;
|
|
PrKey pk;
|
|
FsKey fk;
|
|
FoKey ok;
|
|
lib_keys(s, k, &vk_, &pk, &fk, &ok);
|
|
U.waits++;
|
|
const void *key[2] = {&pk, &fk};
|
|
uint32_t size[2] = {sizeof pk, sizeof fk};
|
|
int queued = 0;
|
|
for (int i = 0; i < 2; i++) {
|
|
uint64_t h = key_hash(key[i], size[i]);
|
|
if ((b[i] = lib_find(h, key[i], size[i]))) continue;
|
|
if (!half_layout(l, i)) return 0; // gpl_link reports it
|
|
LibJob *j = calloc(1, sizeof *j);
|
|
if (!j || !(b[i] = lib_new(h, key[i], size[i]))) {
|
|
free(j);
|
|
return 0;
|
|
}
|
|
*j = (LibJob){.b = b[i], .kind = i ? LIB_FS : LIB_PR, .l = l, .s = *s, .k = *k};
|
|
b[i]->job = j;
|
|
u_push(j);
|
|
U.queued++, queued++;
|
|
}
|
|
if (queued) plat_cond_broadcast(&work_cv), plat_cond_broadcast(&lib_cv);
|
|
return 1;
|
|
}
|
|
|
|
// A waiting draw's whole fast link of p on the library workers: its library jobs, then the link job behind them.
|
|
// The caller then waits for p->info.pipeline. 0 when nothing was queued (the caller links as before). Under P.lock.
|
|
static int link_queue(Pipe *p, const VkPipeState *s, const GKey *k, Layout *l) {
|
|
Lib *b[2];
|
|
if (!U.lib_workers || !lib_queue(s, k, l, b)) return 0;
|
|
LibJob *j = calloc(1, sizeof *j);
|
|
if (!j) return 0;
|
|
*j = (LibJob){.l = l, .s = *s, .k = *k, .p = p};
|
|
p->link_queued = 1;
|
|
u_push(j);
|
|
plat_cond_broadcast(&work_cv), plat_cond_broadcast(&lib_cv);
|
|
return 1;
|
|
}
|
|
|
|
static void lib_wait(const VkPipeState *s, const GKey *k, Layout *l, double end) {
|
|
Lib *b[2];
|
|
if (!lib_queue(s, k, l, b)) return;
|
|
double steal_at = now_ms() + 1.0;
|
|
for (;;) {
|
|
if (!b[0]->building && !b[1]->building) return;
|
|
double now = now_ms();
|
|
if (now >= end) {
|
|
U.timeouts++;
|
|
return;
|
|
}
|
|
Lib *q = b[0]->job ? b[0] : b[1]->job ? b[1] : NULL;
|
|
if (q && !U.lib_workers && (now >= steal_at || !U.idle)) {
|
|
U.stolen++;
|
|
lib_steal(q); // releases P.lock while building
|
|
continue;
|
|
}
|
|
double until = q && steal_at < end ? steal_at : end;
|
|
plat_cond_wait_ms(&done_cv, &P.lock, (uint32_t)(until - now) + 1);
|
|
}
|
|
}
|
|
|
|
// ---- stage libraries ahead of their draws (NX2NAVI_PIPE_AHEAD, default 1; 0 off; vk_exec.c "looking ahead")
|
|
// The replay names the state of a draw it has not reached yet. When no pipeline is known for it, its pre-rasterization
|
|
// and fragment shader libraries are queued to the library workers now, so they build while the draws before it are
|
|
// recorded, in parallel with each other and with those of the other draws named: the draw's own lookup then finds both
|
|
// and links on its thread in microseconds instead of waiting for them or being skipped while they build. Nothing else
|
|
// is entered, no pipeline, no optimised build and no key record: a state the scan guessed wrong costs two libraries
|
|
// at most. The feedback loop bits, the topology and the targets do not matter here, as no shader library depends on
|
|
// them. Returns the libraries queued.
|
|
static int hint_on(void) {
|
|
static int on = -1;
|
|
if (on < 0) on = cfg_bool("NX2NAVI_PIPE_AHEAD");
|
|
return on;
|
|
}
|
|
|
|
int vk_pipeline_hint(const VkPipeState *s) {
|
|
if (!A.on || !U.on || !hint_on() || !s->vs || !ensure_init() || !P.gpl || !P.indep || is_mesh(s)) return 0;
|
|
if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) return 0;
|
|
GKey k;
|
|
gkey_fill(s, &k);
|
|
if (k.topo_class == 3) return 0;
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
int queued = 0;
|
|
plat_mutex_lock(&P.lock);
|
|
if (!pipe_find(&k, sizeof k, h)) {
|
|
Bindings B = {0}, F = {0};
|
|
Layout *l = add_stage(&B, s->vs, 0) && (!s->fs || add_stage(&F, s->fs, 0)) ? layout_pair(&B, &F) : NULL;
|
|
Lib *b[2];
|
|
uint64_t q0 = U.queued, w0 = U.waits;
|
|
if (l && l->set1 && workers_start() && lib_queue(s, &k, l, b)) queued = (int)(U.queued - q0);
|
|
U.waits = w0, U.queued = q0; // those count waiting draws' jobs
|
|
P.g.hints++, P.g.hint_libs += (uint64_t)queued;
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
return queued;
|
|
}
|
|
|
|
static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int async, int *pending, uint32_t wait_ms) {
|
|
if (info) *info = NULL;
|
|
if (pending) *pending = 0;
|
|
if (!s->vs || !ensure_init()) return VK_NULL_HANDLE;
|
|
GKey k;
|
|
gkey_fill(s, &k);
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
plat_mutex_lock(&P.lock);
|
|
Pipe *p = pipe_find(&k, sizeof k, h);
|
|
int queued;
|
|
if (p) {
|
|
if (!p->building) P.stats.hits++;
|
|
} else {
|
|
p = pipe_miss(s, &k, h, async, &queued);
|
|
}
|
|
// A draw that would wait (wait_ms) gets its missing shader libraries built at once, the vertex and fragment ones in
|
|
// parallel on idle workers (lib_wait; about 0.12 ms per library from the driver's disk cache, 5-180 ms cold), and
|
|
// links them here: its own job may queue behind optimised builds of earlier pipelines. The job then only does the
|
|
// optimised build, from the low queue. Other draws leave the libraries to the job and are skipped meanwhile.
|
|
// NX2NAVI_PIPE_WAIT=draw builds them on this thread one after the other instead, as before; NX2NAVI_GPL_DRAW_BUILD=0
|
|
// leaves them to the job.
|
|
static int draw_build = -1;
|
|
if (draw_build < 0) {
|
|
const char *e = cfg_get("NX2NAVI_GPL_DRAW_BUILD");
|
|
draw_build = !(e && !strcmp(e, "0"));
|
|
rt_log("vk_pipeline: a draw that waits for its pipeline %s", !draw_build ? "leaves its libraries to its job"
|
|
: U.on ? "has its libraries built in parallel on the workers, ahead of other jobs"
|
|
: "builds its libraries itself (NX2NAVI_PIPE_WAIT=draw)");
|
|
}
|
|
double end = wait_ms ? now_ms() + wait_ms : 0;
|
|
if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && !p->link_queued && p->lay && !is_mesh(s) &&
|
|
wait_ms && draw_build && U.on && link_queue(p, s, &k, (Layout *)p->lay)) {
|
|
// the libraries and the link on the library workers; this thread waits below without building
|
|
p->fast_tried = 1;
|
|
job_lower(p->job);
|
|
} else if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && !p->link_queued && p->lay &&
|
|
!is_mesh(s)) {
|
|
int final = 0, build = wait_ms != 0 && draw_build;
|
|
uint64_t p0 = perf_begin(), t0 = qpc();
|
|
if (build && U.on) {
|
|
job_lower(p->job); // its worker would only wait for the same libraries; it links them if this draw cannot
|
|
lib_wait(s, &k, (Layout *)p->lay, end); // releases P.lock while waiting
|
|
build = 0; // links here when both libraries exist (gpl_link build 0)
|
|
} else if (build) {
|
|
p->fast_tried = 1; // the job, if a worker takes it meanwhile, goes straight to the optimised build
|
|
}
|
|
VkPipeline f = p->info.pipeline ? VK_NULL_HANDLE : gpl_link(s, &k, p->lay, build, &final); // build 1 releases P.lock
|
|
perf_end(PERF_PIPE, p0);
|
|
if (wait_ms && draw_build) {
|
|
double ms = ms_since(t0);
|
|
P.g.dlinks++, P.g.dlink_ms += ms;
|
|
if (ms > P.g.dlink_max) P.g.dlink_max = ms;
|
|
if (ms > 5.0 && P.g.slow++ < 16)
|
|
rt_log("vk_pipeline: gpl: a waiting draw %s its libraries in %.1f ms", U.on ? "got" : "built", ms);
|
|
}
|
|
if (final) p->fast_tried = 1;
|
|
if (f && !p->info.pipeline) {
|
|
p->fast = p->info.pipeline = f;
|
|
ready_note(p, 1);
|
|
job_lower(p->job);
|
|
plat_cond_broadcast(&done_cv);
|
|
} else if (f) {
|
|
vkDestroyPipeline(P.device, f, NULL); // a worker published one while the lock was free
|
|
}
|
|
}
|
|
if (p && p->building && !p->info.pipeline) {
|
|
promote(p);
|
|
uint64_t p0 = wait_ms ? perf_begin() : 0; // the wait counts as the render thread's pipeline time
|
|
double now;
|
|
if (wait_ms) job_wanted(p->job, wait_ms); // releases P.lock when it builds the job here
|
|
while (p->building && !p->info.pipeline && wait_ms && (now = now_ms()) < end)
|
|
plat_cond_wait_ms(&done_cv, &P.lock, (uint32_t)(end - now) + 1);
|
|
if (wait_ms) perf_end(PERF_PIPE, p0);
|
|
if (wait_ms && !p->building) perf_count(PERF_CNT_PIPE_WAIT);
|
|
}
|
|
int wait = p && p->building && !p->info.pipeline;
|
|
if (wait) P.stats.pending_skips++;
|
|
VkPipeline pipe = p ? p->info.pipeline : VK_NULL_HANDLE;
|
|
plat_mutex_unlock(&P.lock);
|
|
if (wait) {
|
|
if (pending) *pending = 1;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
if (!p) return VK_NULL_HANDLE;
|
|
if (info) *info = &p->info;
|
|
return pipe;
|
|
}
|
|
|
|
VkPipeline vk_pipeline_graphics(const VkPipeState *s, const VkPipeInfo **info) {
|
|
return graphics(s, info, 0, NULL, 0);
|
|
}
|
|
|
|
VkPipeline vk_pipeline_graphics_async(const VkPipeState *s, const VkPipeInfo **info, int *pending, uint32_t wait_ms) {
|
|
return graphics(s, info, A.on, pending, wait_ms);
|
|
}
|
|
|
|
double vk_pipeline_building_ms(const VkPipeState *s) {
|
|
if (!s->vs || !P.ready) return 0;
|
|
GKey k;
|
|
gkey_fill(s, &k);
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
plat_mutex_lock(&P.lock);
|
|
const Pipe *p = pipe_find(&k, sizeof k, h);
|
|
double ms = p && p->building && !p->info.pipeline && p->t_miss ? (double)(plat_now_ns() - p->t_miss) / 1e6 : 0;
|
|
plat_mutex_unlock(&P.lock);
|
|
return ms;
|
|
}
|
|
|
|
// ---- boot prewarm from the key file (comment at "pipeline keys")
|
|
|
|
static int id_cmp(const void *x, const void *y) { return memcmp(x, y, 32); }
|
|
|
|
// Builds queued prebuild jobs until the low queue is empty or K.until passes, beside the pipe_worker pool (same
|
|
// bookkeeping). Jobs left queued at K.until stay for the pool. So do those the memory gate refuses (gate_enter): a
|
|
// builder it refuses ends, which leaves fewer builders compiling side by side while the others' reservations stand,
|
|
// and none when the machine has no room; the pool builds what is left during play, when the gate lets it.
|
|
// arg non-NULL: one job at most. The builders start with one pipeline built alone on the starting thread:
|
|
// a driver's first graphics pipeline sets up its compiler, and NVIDIA's 621.14 was reported to fault in a sizable
|
|
// share of starts when every builder began its first pipeline at the same moment.
|
|
static void warm_builder(void *arg) {
|
|
int one = arg != NULL;
|
|
plat_mutex_lock(&P.lock);
|
|
while (A.low && !(K.until && now_ms() >= K.until)) {
|
|
double held;
|
|
if (!gate_enter(0, &held)) {
|
|
if (!K.gate_ended++) {
|
|
char g[256];
|
|
rt_log("vk_pipeline: prewarm: the boot builds fewer recorded pipelines side by side, or none: the commit "
|
|
"headroom is short (%s). The rest are built in the background when there is room, or at their "
|
|
"first draw", gate_text(g, sizeof g));
|
|
}
|
|
break;
|
|
}
|
|
Job *j = A.low;
|
|
if ((A.low = j->next)) A.low->prev = NULL;
|
|
else A.low_tail = NULL;
|
|
j->next = j->prev = NULL;
|
|
j->low = 0; // not on a pool thread: promote() has nothing to raise
|
|
plat_mutex_unlock(&P.lock);
|
|
VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off);
|
|
plat_mutex_lock(&P.lock);
|
|
if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one
|
|
ready_note(j->p, 0);
|
|
if (pipe && j->p->fast) P.g.swaps++;
|
|
j->p->building = 0;
|
|
j->p->job = NULL;
|
|
if (pipe) {
|
|
P.stats.graphics++, P.stats.async_built++, K.built++;
|
|
} else {
|
|
P.stats.failures++, K.failed++;
|
|
rt_log("vk_pipeline: graphics pipeline not built: pipeline creation failed");
|
|
}
|
|
K.promoted += j->demanded;
|
|
K.left--;
|
|
if (!--A.inflight) plat_cond_broadcast(&A.idle);
|
|
plat_cond_broadcast(&done_cv);
|
|
free(j);
|
|
gate_leave(held);
|
|
if (one) break;
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
return;
|
|
}
|
|
|
|
static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]);
|
|
static int pm_skipped(void);
|
|
static void pm_set(void);
|
|
|
|
// vk_pipeline_prewarm_progress's numbers: the phase, and the pipelines finished (built or failed) of those queued, copied
|
|
// while prewarm_run waits for its builders.
|
|
static int32_t pw_phase;
|
|
static int64_t pw_pipe_done, pw_pipe_total;
|
|
|
|
void vk_pipeline_prewarm_progress(VkPrewarmProgress *p) {
|
|
p->phase = __atomic_load_n(&pw_phase, __ATOMIC_ACQUIRE);
|
|
gfx_shader_prewarm_progress(&p->sh_done, &p->sh_total, &p->sh_translated);
|
|
p->pipe_done = __atomic_load_n(&pw_pipe_done, __ATOMIC_RELAXED);
|
|
p->pipe_total = __atomic_load_n(&pw_pipe_total, __ATOMIC_RELAXED);
|
|
}
|
|
|
|
static void prewarm_run(void);
|
|
|
|
// The boot's share of the pipeline prebuild with the graphics pipeline library. The boot builds the recorded
|
|
// pipelines in record order, so those met first in play (the title screen, the first places) come first, for at most
|
|
// this long after the shaders are ready; the rest stay queued for the pool workers, which build them below normal
|
|
// priority while the game runs. A draw that meets one of them before it is built gets a pipeline linked from stage
|
|
// libraries at once (gpl_link), as for any new pipeline in play. After a driver update, when every pipeline is built
|
|
// from nothing, the boot thus waits this long instead of for all of them (15 minutes and more with a long play
|
|
// history). Without the library a draw would have to wait for a whole build, so the boot waits for all; so it does
|
|
// with NX2NAVI_PREWARM_FULL=1.
|
|
#define PREWARM_PIPE_MS 20000.0
|
|
|
|
void vk_pipeline_prewarm(void) {
|
|
prewarm_run();
|
|
__atomic_store_n(&pw_phase, 3, __ATOMIC_RELEASE);
|
|
}
|
|
|
|
static void prewarm_run(void) {
|
|
const char *gpu = rt_config()->gpu;
|
|
uint32_t pr_hist[16] = {0}, fs_hist[4] = {0}; // library state of the recorded pipelines (lb_start)
|
|
if (!A.on || (gpu && !strcmp(gpu, "null"))) return;
|
|
if (!cfg_bool("NX2NAVI_PIPE_PREBUILD")) { // prebuild_start logs the off case
|
|
if (ensure_init()) lb_start(pr_hist, fs_hist);
|
|
return;
|
|
}
|
|
double t0 = now_ms();
|
|
K.warm = 1;
|
|
if (!ensure_init() || !K.file_n) {
|
|
rt_log("vk_pipeline: prewarm: no pipeline keys (%s)", K.path[0] ? K.path : "<none>");
|
|
if (P.ready) lb_start(pr_hist, fs_hist);
|
|
return;
|
|
}
|
|
const KeyRec *r = (const KeyRec *)K.file;
|
|
size_t n = K.file_n, m = 0;
|
|
uint8_t (*ids)[32] = malloc(3 * n * 32);
|
|
const GfxShader **sh = calloc(3 * n, sizeof *sh);
|
|
if (!ids || !sh) {
|
|
free(ids), free(sh);
|
|
return;
|
|
}
|
|
for (size_t i = 0; i < n; i++) {
|
|
memcpy(ids[m++], r[i].vs, 32);
|
|
if (r[i].has_fs) memcpy(ids[m++], r[i].fs, 32);
|
|
if (r[i].has_ts) memcpy(ids[m++], r[i].ts, 32);
|
|
}
|
|
qsort(ids, m, 32, id_cmp);
|
|
size_t u = 0;
|
|
for (size_t i = 0; i < m; i++)
|
|
if (!u || memcmp(ids[u - 1], ids[i], 32)) memcpy(ids[u++], ids[i], 32);
|
|
int cpus = (int)plat_cpu_usable();
|
|
const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset
|
|
int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2;
|
|
__atomic_store_n(&pw_phase, 1, __ATOMIC_RELEASE);
|
|
gfx_shader_prewarm((const uint8_t(*)[32])ids, u, sh, cpus, procs < 1 ? 1 : procs);
|
|
double t1 = now_ms();
|
|
size_t carried = 0, fresh = 0; // the recorded pipelines' shaders by where their translation came from
|
|
for (size_t i = 0; i < u; i++)
|
|
if (sh[i]) carried += gfx_shader_origin(sh[i]) == 1, fresh += gfx_shader_origin(sh[i]) == 2;
|
|
vk_mem_stage_log("the shader prewarm");
|
|
|
|
if (P.indep) { // vertex output unions first: workers may take a job before the queue is complete
|
|
uint64_t pair = 0, uni = 0, pairs = 0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp);
|
|
const uint8_t (*f)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL;
|
|
if (!v || !f || !sh[v - ids] || !sh[f - ids] || gfx_shader_stage(sh[v - ids]) != GFX_STAGE_VS) continue;
|
|
GfxShaderIo io[32];
|
|
vs_outs(sh[v - ids], sh[f - ids], io);
|
|
pair += (uint64_t)__builtin_popcount(gfx_shader_inputs(sh[f - ids], NULL)), pairs++;
|
|
}
|
|
for (uint32_t j = 0; j < VS_OUTS_CAP; j++) uni += VO.vs[j] ? (uint64_t)__builtin_popcount(VO.mask[j]) : 0;
|
|
rt_log("vk_pipeline: vertex output unions: %u vertex shaders, %.2f locations each; %llu pairs read %.2f each",
|
|
VO.n, VO.n ? (double)uni / VO.n : 0.0, (unsigned long long)pairs, pairs ? (double)pair / pairs : 0.0);
|
|
}
|
|
if (pm_skipped()) { // the boot's pipeline builds killed an earlier start on this GPU and driver
|
|
plat_mutex_lock(&P.lock);
|
|
free(K.file);
|
|
K.file = NULL;
|
|
plat_mutex_unlock(&P.lock);
|
|
free(ids), free(sh);
|
|
lb_start(pr_hist, fs_hist);
|
|
return;
|
|
}
|
|
pm_set();
|
|
uint32_t queued = 0, dropped = 0, known = 0;
|
|
plat_mutex_lock(&P.lock);
|
|
K.t0 = t1;
|
|
K.loading = 1;
|
|
for (size_t i = 0; i < n; i++) {
|
|
const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp);
|
|
const uint8_t (*f)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL;
|
|
const uint8_t (*t)[32] = r[i].has_ts ? bsearch(r[i].ts, ids, u, 32, id_cmp) : NULL;
|
|
const GfxShader *vs = v ? sh[v - ids] : NULL, *fs = f ? sh[f - ids] : NULL, *ts = t ? sh[t - ids] : NULL;
|
|
if (!vs || (r[i].has_fs && !fs) || (r[i].has_ts && !ts)) {
|
|
dropped++;
|
|
continue;
|
|
}
|
|
VkPipeState s = r[i].s;
|
|
s.vs = vs, s.fs = fs, s.ts = ts;
|
|
GKey k;
|
|
gkey_fill(&s, &k);
|
|
if (!is_mesh(&s)) { // the stage libraries lb_start prebuilds are for vertex pipelines
|
|
pr_hist[(k.clamp & 1) | (k.poly_mode & 3) << 1 | (k.minus_w & 1) << 3]++;
|
|
fs_hist[k.samples >= 8 ? 3 : k.samples >= 4 ? 2 : k.samples >= 2 ? 1 : 0]++;
|
|
}
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
int q = 0;
|
|
if (pipe_find(&k, sizeof k, h)) known++;
|
|
else pipe_miss(&s, &k, h, 2, &q), queued += (uint32_t)q;
|
|
}
|
|
K.loading = 0;
|
|
plat_mutex_unlock(&P.lock);
|
|
free(ids), free(sh);
|
|
|
|
__atomic_store_n(&pw_pipe_done, 0, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&pw_pipe_total, (int64_t)queued, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&pw_phase, 2, __ATOMIC_RELEASE);
|
|
const char *full = cfg_get("NX2NAVI_PREWARM_FULL");
|
|
K.until = P.gpl && !(full && !strcmp(full, "1")) ? now_ms() + PREWARM_PIPE_MS : 0; // before the builders start
|
|
PlatThread *t[64];
|
|
int nt = 0;
|
|
if (queued) warm_builder((void *)1); // the first alone (warm_builder)
|
|
for (int i = 0; i < cpus && i < 64 && queued; i++)
|
|
if ((t[nt] = plat_thread_new(warm_builder, NULL, "pipe prewarm"))) nt++;
|
|
if (!nt) warm_builder(NULL);
|
|
double logged = now_ms();
|
|
for (int i = 0; i < nt; i++)
|
|
while (!plat_thread_join_ns(t[i], 100000000u)) { // every 100 ms the numbers, every 2 s a log line
|
|
uint32_t fin = __atomic_load_n(&K.built, __ATOMIC_RELAXED) + __atomic_load_n(&K.failed, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&pw_pipe_done, (int64_t)fin, __ATOMIC_RELAXED);
|
|
if (now_ms() - logged < 2000) continue;
|
|
logged = now_ms();
|
|
rt_log("vk_pipeline: prewarm: %u of %u pipelines, %.0f ms", fin, queued, now_ms() - t1);
|
|
}
|
|
__atomic_store_n(&pw_pipe_done, (int64_t)queued, __ATOMIC_RELAXED);
|
|
plat_mutex_lock(&P.lock);
|
|
uint32_t later = 0;
|
|
for (Job *j = A.low; j; j = j->next) later += j->prebuild;
|
|
if (!later) // all built: wait for those a pool worker took
|
|
while (K.left) plat_cond_wait(&done_cv, &P.lock);
|
|
else // the boot's time is up: the pool builds the rest while the game runs
|
|
plat_cond_broadcast(&work_cv);
|
|
rt_log("vk_pipeline: prewarm: %zu records in %s: %u pipelines built, %u left to build in the background, %u "
|
|
"failed, %u already built, %u dropped (shader neither cached nor stored); shaders %.0f ms, pipelines %.0f "
|
|
"ms on %d threads, total %.0f ms", n, K.path, K.built, later, K.failed, known, dropped, t1 - t0,
|
|
now_ms() - t1, nt, now_ms() - t0);
|
|
free(K.file);
|
|
K.file = NULL;
|
|
plat_mutex_unlock(&P.lock);
|
|
// After an update with another translator: a record names its shaders by input id, which no translator changes,
|
|
// so every record still finds its shaders, and the pipeline cache gives the pipeline back wherever the translation
|
|
// in use is the one it was built from. The line below this one says how many it gave back.
|
|
if (carried || fresh)
|
|
rt_log("vk_pipeline: prewarm: of the %zu shaders the recorded pipelines name, %zu are a previous translator's "
|
|
"translations and %zu were translated now; a recorded pipeline is compiled again only where its shaders' "
|
|
"translation changed, which the pipeline-cache.bin hits below show", u, carried, fresh);
|
|
fb_log("after the boot prewarm");
|
|
vk_mem_stage_log("the recorded pipelines' prebuild");
|
|
vk_pipeline_prewarm_marker_clear(); // the boot's share is done; what the pool builds later runs during play
|
|
lb_start(pr_hist, fs_hist);
|
|
}
|
|
|
|
// ---- boot prebuild crash marker
|
|
// The boot builds the recorded pipelines (warm_builder) before the game starts. A driver whose compiler crashes on one
|
|
// of them would kill every start at the same point, and the preparation journal above does not cover these builds.
|
|
// So <dir>/pipeline-prewarm-running.txt exists while they run: written before the first one, deleted when the boot's
|
|
// share ends (or when the boot stops waiting and lets the game start). A start that finds it left over records this
|
|
// GPU and driver in <dir>/pipeline-prewarm-skip.txt, and starts on them build no recorded pipelines ahead of play:
|
|
// each one is built at its first draw instead, as with NX2NAVI_PIPE_PREBUILD=0. Another driver version, or deleting
|
|
// the file, brings the prebuild back. Only the boot's own share is covered: a crash during play, where the pool also
|
|
// builds what the boot left, says nothing about these builds. An orderly end of the process during the boot (the
|
|
// window closed, the guest's ExitProcess, a --seconds run) deletes the marker as well: only a process that died leaves it.
|
|
static int cache_dir(char *dir, size_t n);
|
|
static void gpu_key(char *out, size_t n);
|
|
static char PM_run[PLAT_PATH_MAX * 2 + 48];
|
|
static volatile int32_t pm_on;
|
|
|
|
static int pm_skipped(void) {
|
|
char dir[PLAT_PATH_MAX * 2], sp[sizeof PM_run], key[32];
|
|
if (!cache_dir(dir, sizeof dir)) return 0;
|
|
snprintf(PM_run, sizeof PM_run, "%s/pipeline-prewarm-running.txt", dir);
|
|
snprintf(sp, sizeof sp, "%s/pipeline-prewarm-skip.txt", dir);
|
|
gpu_key(key, sizeof key);
|
|
uint64_t n = 0;
|
|
uint8_t *left = plat_file_read(PM_run, &n);
|
|
if (left) {
|
|
char text[256];
|
|
int len = snprintf(text, sizeof text,
|
|
"# The boot's pipeline prebuild did not finish on this GPU and driver (runtime/vk_pipeline.c "
|
|
"pm_skipped): starts skip it\n# while this line matches. Delete this file to build ahead of play "
|
|
"again.\n%s\n", key);
|
|
if (plat_file_write_atomic(sp, text, (size_t)len)) plat_file_delete(PM_run);
|
|
rt_log("vk_pipeline: the last start died while the boot built pipelines (%s was left); %s now skips that "
|
|
"prebuild on this GPU and driver (%s)", PM_run, sp, key);
|
|
free(left);
|
|
}
|
|
uint8_t *skip = plat_file_read(sp, &n);
|
|
int on = 0;
|
|
for (char *line = (char *)skip, *next; line && *line && !on; line = next) {
|
|
next = strchr(line, '\n');
|
|
if (next) *next++ = 0;
|
|
size_t l = strlen(line);
|
|
while (l && (line[l - 1] == '\r' || line[l - 1] == ' ')) line[--l] = 0;
|
|
on = line[0] != '#' && !strcmp(line, key);
|
|
}
|
|
free(skip);
|
|
if (on)
|
|
rt_log("vk_pipeline: prewarm: %zu recorded pipelines not built ahead of play: an earlier start died while building "
|
|
"them on this GPU and driver (%s lists %s); each builds at its first draw", K.file_n, sp, key);
|
|
return on;
|
|
}
|
|
|
|
static void pm_set(void) {
|
|
if (!PM_run[0]) return;
|
|
char key[32];
|
|
gpu_key(key, sizeof key);
|
|
// straight to the disk: the file has to be there if the driver takes the process down a moment later
|
|
if (plat_file_write_atomic(PM_run, key, strlen(key))) plat_xchg32(&pm_on, 1);
|
|
}
|
|
|
|
void vk_pipeline_prewarm_marker_clear(void) {
|
|
if (plat_xchg32(&pm_on, 0)) plat_file_delete(PM_run);
|
|
}
|
|
|
|
// ensure_init's hook: without a prewarm (NX2NAVI_PIPE_PREBUILD=0, or the device came up first elsewhere) the records
|
|
// are not built. Caller holds no lock.
|
|
static void prebuild_start(void) {
|
|
plat_mutex_lock(&P.lock);
|
|
if (K.file && !K.warm) {
|
|
rt_log("vk_pipeline: prebuild off%s; %zu records not built",
|
|
!cfg_bool("NX2NAVI_PIPE_PREBUILD") ? " (NX2NAVI_PIPE_PREBUILD=0)" : " (no boot prewarm)", K.file_n);
|
|
free(K.file);
|
|
K.file = NULL;
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
}
|
|
|
|
// NX2NAVI_SPV_OVERRIDE=<dir> (test): a compute shader whose SPIR-V hashes (FNV-1a over its bytes, the name vk_dump.c
|
|
// gives it) to a file <dir>/<hash>.spv runs that SPIR-V instead, an edited copy with the same interface, to see a
|
|
// shader's intermediate values in a frame dump. NULL: no override (malloc'd words otherwise).
|
|
static uint32_t *spv_override(const uint32_t *code, uint64_t words, uint64_t *out) {
|
|
const char *dir = cfg_get("NX2NAVI_SPV_OVERRIDE");
|
|
if (!dir || !*dir) return NULL;
|
|
uint64_t h = 0xcbf29ce484222325ull;
|
|
for (uint64_t i = 0; i < words * 4; i++) h = (h ^ ((const uint8_t *)code)[i]) * 0x100000001b3ull;
|
|
char path[600];
|
|
snprintf(path, sizeof path, "%s/%016llx.spv", dir, (unsigned long long)h);
|
|
FILE *f = fopen(path, "rb");
|
|
if (!f) return NULL;
|
|
fseek(f, 0, SEEK_END);
|
|
long n = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
uint32_t *w = n > 20 && n % 4 == 0 ? malloc((size_t)n) : NULL;
|
|
if (w && fread(w, 1, (size_t)n, f) != (size_t)n) free(w), w = NULL;
|
|
fclose(f);
|
|
if (w) *out = (uint64_t)n / 4, rt_log("vk_pipeline: compute shader %016llx replaced by %s", (unsigned long long)h, path);
|
|
return w;
|
|
}
|
|
|
|
// A skipped froxel's exit in the fog's first compute pass: the block at word at (an OpLabel) makes five image writes, four
|
|
// of (0,0,0,1) and last the mask value (1/255,0,0,0), and returns.
|
|
static int fog_skip_block(const uint32_t *w, uint64_t n, uint64_t at, const uint64_t *def, uint32_t bound) {
|
|
int writes = 0;
|
|
for (uint64_t i = at + (w[at] >> 16); i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (!len || i + len > n) return 0;
|
|
if (op == 253) return writes == 5; // OpReturn
|
|
if (op == 248 || op == 249 || op == 250) return 0; // OpLabel, OpBranch, OpBranchConditional
|
|
if (op == 99) { // OpImageWrite
|
|
if (len < 4 || w[i + 3] >= bound || ++writes > 5) return 0;
|
|
uint64_t c = def[w[i + 3]];
|
|
if (!c || (w[c] & 0xffff) != 80 || w[c] >> 16 != 7) return 0; // OpCompositeConstruct of four
|
|
float v[4];
|
|
for (int k = 0; k < 4; k++) {
|
|
uint32_t id = w[c + 3 + k];
|
|
uint64_t f = id < bound ? def[id] : 0;
|
|
if (!f || (w[f] & 0xffff) != 43 || w[f] >> 16 != 4) return 0; // OpConstant of one word
|
|
memcpy(&v[k], &w[f + 3], 4);
|
|
}
|
|
if (writes == 5 ? !(v[0] > 0.0039f && v[0] < 0.0040f && v[1] == 0 && v[2] == 0 && v[3] == 0)
|
|
: !(v[0] == 0 && v[1] == 0 && v[2] == 0 && v[3] == 1)) return 0;
|
|
}
|
|
i += len;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// The volumetric fog's first compute pass skips a froxel behind a coarse depth table and writes it as empty, and the next
|
|
// pass's mask keeps it empty. That table comes from the frame before, so in the columns a near object has just left, every
|
|
// froxel past the object's old depth came out with no fog: a dark block of the fog grid beside the object, which the fog's
|
|
// history then smeared into a trail, and worst at the screen edges, where near objects move fastest. A froxel behind
|
|
// today's depth is never seen, so computing it only costs a little time: the comparison is made never true (x > x).
|
|
// Returns the patched copy of that shader, NULL for any other. Found by its shape rather than a hash, so it holds across
|
|
// translator updates: a branch whose other side is that exit, taken on !(distance > table) || flag.
|
|
// NX2NAVI_FOG_CULL=1 keeps the game's skip.
|
|
static uint32_t *spirv_fog_keep_froxels(const uint32_t *w, uint64_t n) {
|
|
const char *e = cfg_get("NX2NAVI_FOG_CULL");
|
|
if (n < 6 || (e && !strcmp(e, "1"))) return NULL;
|
|
uint32_t bound = w[3];
|
|
int writes = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t len = w[i] >> 16;
|
|
if (!len || i + len > n) return NULL;
|
|
writes += (w[i] & 0xffff) == 99;
|
|
i += len;
|
|
}
|
|
if (writes < 5) return NULL;
|
|
uint64_t *def = calloc(bound, sizeof *def); // word of each id's definition, for the instructions looked at below
|
|
if (!def) return NULL;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (op == 248 && len >= 2 && w[i + 1] < bound) def[w[i + 1]] = i; // OpLabel
|
|
else if ((op == 43 || op == 80 || op == 166 || op == 168 || op == 186) && len >= 3 && w[i + 2] < bound) def[w[i + 2]] = i;
|
|
i += len;
|
|
}
|
|
uint64_t cmp = 0, prev = 0;
|
|
int found = 0;
|
|
for (uint64_t i = 5; i < n;) {
|
|
uint32_t op = w[i] & 0xffff, len = w[i] >> 16;
|
|
if (op == 250 && len >= 4 && prev && (w[prev] & 0xffff) == 247 && w[prev + 1] == w[i + 3] && w[i + 1] < bound &&
|
|
w[i + 3] < bound && def[w[i + 3]] && fog_skip_block(w, n, def[w[i + 3]], def, bound)) { // OpSelectionMerge just before
|
|
uint64_t c = def[w[i + 1]];
|
|
for (int k = 0; c && (w[c] & 0xffff) == 166 && k < 2; k++) { // OpLogicalOr
|
|
uint32_t a = w[c + 3 + k];
|
|
uint64_t no = a < bound ? def[a] : 0;
|
|
if (!no || (w[no] & 0xffff) != 168 || w[no + 3] >= bound) continue; // OpLogicalNot
|
|
uint64_t gt = def[w[no + 3]];
|
|
if (gt && (w[gt] & 0xffff) == 186) cmp = gt, found++; // OpFOrdGreaterThan
|
|
}
|
|
}
|
|
prev = i;
|
|
i += len;
|
|
}
|
|
free(def);
|
|
uint32_t *o = found == 1 ? malloc(n * 4) : NULL;
|
|
if (!o) return NULL;
|
|
memcpy(o, w, n * 4);
|
|
o[cmp + 4] = o[cmp + 3];
|
|
LOG_ONCE("vk_pipeline: fog: froxels behind last frame's depth are computed too");
|
|
return o;
|
|
}
|
|
|
|
// A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised).
|
|
// cached_only (needs P.cache_control): from P.cache or the driver's own disk cache, never compiled; VK_NULL_HANDLE when
|
|
// neither holds it (the driver returns VK_PIPELINE_COMPILE_REQUIRED, a success code, with no pipeline).
|
|
static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cached_only) {
|
|
uint64_t words;
|
|
const uint32_t *code = gfx_shader_spirv(cs, &words);
|
|
uint64_t ow = 0;
|
|
uint32_t *over = spv_override(code, words, &ow);
|
|
if (over) code = over, words = ow;
|
|
uint32_t *fog = spirv_fog_keep_froxels(code, words);
|
|
if (fog) free(over), over = fog, code = fog;
|
|
VkShaderModule m = make_module(code, words, 0);
|
|
if (!m) return free(over), VK_NULL_HANDLE;
|
|
VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqc;
|
|
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
|
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
|
.stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"},
|
|
.layout = l->layout, .flags = noopt_flag()};
|
|
if (cached_only) ci.flags |= VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
|
|
pin_subgroup32(&ci.stage, &rqc, code, words);
|
|
// Subgroup operations in a workgroup whose width is a whole number of subgroups need every subgroup full: the
|
|
// driver may otherwise run partial ones, and a ballot over the culling's candidates then counted differently from
|
|
// one frame to the next
|
|
uint32_t lx = local_size_x(code, words);
|
|
if (P.f_full_subgroups && vk_caps.sg_size && lx && lx % vk_caps.sg_size == 0 && uses_subgroup_ops(code, words))
|
|
ci.stage.flags |= VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT;
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
VkPipelineCreationFeedbackCreateInfo fci;
|
|
VkPipelineCreationFeedback fb;
|
|
ci.pNext = fb_chain(&fci, &fb, ci.pNext);
|
|
crash_note("a compute pipeline", cs, NULL, NULL);
|
|
if (pipe_create(2, &ci, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
|
plat_crash_context(NULL);
|
|
if (pipe) fb_note(FB_COMPUTE, &fb);
|
|
cc_hit = !pipe ? -1 : pb_last ? 1
|
|
: !P.feedback || !P.cache || !(fb.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) ? -1
|
|
: (fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0;
|
|
if (P.feedback >= 2 && (pipe || !cached_only))
|
|
rt_log("vk_pipeline: compute %016llx: %llu words, module %016llx, %s, %.1f ms", (unsigned long long)fnv(FNV0, code, words * 4),
|
|
(unsigned long long)words, (unsigned long long)mm_hash,
|
|
!pipe ? "failed" : pb_last ? "from pipeline-binaries.bin" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled",
|
|
(double)fb.duration / 1e6);
|
|
vkDestroyShaderModule(P.device, m, NULL);
|
|
free(over);
|
|
return pipe;
|
|
}
|
|
|
|
// The layout of a compute shader in *B; NULL with *why on failure. Caller holds P.lock.
|
|
static Layout *compute_layout(const GfxShader *cs, Bindings *B, const char **why) {
|
|
Layout *l = NULL;
|
|
if (gfx_shader_stage(cs) != GFX_STAGE_CS) *why = "not a compute shader";
|
|
else if (!add_stage(B, cs, 0)) *why = "too many bindings";
|
|
else if (!(l = layout_get(B))) *why = "descriptor layout creation failed";
|
|
return l;
|
|
}
|
|
|
|
// ---- stage library prebuild
|
|
// Two places build what a draw would otherwise build while the game waits:
|
|
// - nx2navi.exe --prepare-shaders (lb_warm_prepared), after the translation, on every core: for each prepared variant
|
|
// (shader-prepared.bin) the pre-rasterization library of a vertex shader, the fragment-shader library of a fragment
|
|
// shader and the pipeline of a compute shader, each built from a private shader copy and destroyed at once. What
|
|
// stays is the driver's own disk cache: on AMD a library built again in a later process takes 0.1 ms instead of
|
|
// 55 ms (vertex) / 28 ms (fragment). Our pipeline-cache.bin is not written there:
|
|
// it would grow by about 10 KB per library (~240 MB) and be read into memory at every boot.
|
|
// - boot (lb_worker), after the pipeline-key prewarm, on two threads (lb_start) at PLAT_PRIO_IDLE (the short P.lock
|
|
// sections and shader loads at normal priority, so a preempted thread never holds the lock the replay needs; a
|
|
// compute pipeline's compile below normal priority, lb_compute):
|
|
// compute pipelines of every recorded and prepared compute shader (vk_pipeline_compute builds compute
|
|
// synchronously on the replay thread; 163 prepared): those of the shaders dispatched most recently stay, the others
|
|
// are built into the pipeline cache and destroyed (comment at compute-used.bin). Then the libraries of the first
|
|
// NX2NAVI_LIB_PREBUILD_KEEP variants play recorded (priority 0), kept in libs[] with their shaders. KEEP defaults to
|
|
// 0: a kept library costs 140-460 KB of process memory (10,000 kept took 9 GB in run 1) to save a rebuild the driver
|
|
// cache makes 0.1 ms.
|
|
// Library state: the vertex output union known now (vs_union_peek; the prewarm fills it from pipeline-keys.bin, the
|
|
// preparation has none), the clamp, polygon mode and depth mode most recorded pipelines use (defaults without a key
|
|
// file: clamp 0, fill, -w..w), single-sampled, no dual source; the state dynamic on this GPU is 0 in every key.
|
|
// A draw that meets such a variant in a new pipeline then gets its libraries from libs[] or the driver cache and
|
|
// fast-links (gpl_link). NX2NAVI_LIB_PREBUILD=0 turns the boot part off, =N stops it after N variants;
|
|
// NX2NAVI_LIB_PREBUILD_THREADS sets its thread count, NX2NAVI_LIB_PREBUILD_BIG its compute size limit (LB_BIG_WORDS).
|
|
#define LB_MAX_THREADS 16
|
|
// The boot part builds a compute shader over LB_BIG_WORDS SPIR-V words only from the pipeline caches (compute_create
|
|
// cached_only); one that no cache holds is left to its first dispatch, like any shader outside the warm list. Play
|
|
// recorded four straight-line compute shaders of 0.87-4.4 million words (one function of up to 888K instructions; the
|
|
// median of 3,648 compute shaders is 7,351 words, p99 64,844). NVIDIA 621.01 on an RTX 4090 compiled them on four
|
|
// prebuild threads for minutes at 100% CPU each while the process grew from 12 GB to 28-31 GB, until RAM ran out and the
|
|
// game stuttered and went black a few minutes after boot. None finished, so no cache held one and every boot did it
|
|
// again. NX2NAVI_LIB_PREBUILD_BIG sets the limit in words; 0 builds every size.
|
|
#define LB_BIG_WORDS (512u * 1024)
|
|
|
|
static struct {
|
|
GfxWarmEnt *e;
|
|
size_t n;
|
|
size_t n_keep; // e[0..n_keep): compute shaders whose pipeline stays (cu_keep)
|
|
volatile int64_t next;
|
|
volatile int32_t stop, running;
|
|
volatile int32_t short_mem; // the memory gate refused a build for GATE_WAIT_MS: the workers ended
|
|
int threads;
|
|
PlatThread *th[LB_MAX_THREADS];
|
|
uint8_t clamp, poly, minus_w, samples; // library state (GKey fields)
|
|
double t0, ms[3]; // ms: build time per kind (vertex, fragment, compute)
|
|
// warmed: built for the caches only and destroyed; nolib: vertex and fragment variants skipped with the graphics
|
|
// pipeline library off
|
|
// skipped: variants of the skip list (SK below)
|
|
uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, skipped, done, cs_draw;
|
|
uint64_t big_words, deferred; // LB_BIG_WORDS; compute shaders over it left to their dispatch
|
|
uint64_t big_unused; // compute shaders over it that no run dispatched: not built at all
|
|
double warm_ms[3];
|
|
volatile int32_t kept; // libraries entered in libs[] so far
|
|
int32_t keep_n;
|
|
int idle; // build at PLAT_PRIO_IDLE (boot), normal (preparation)
|
|
int tail_one; // automatic boot policy: only worker 0 takes the warming tail
|
|
PlatMutex stat;
|
|
} LB = {.stat = PLAT_MUTEX_INIT};
|
|
static __thread int lb_me = -1; // on a boot prebuild thread (lb_worker): its index in LB.th
|
|
|
|
// Claim a boot job without dropping one when the extra workers finish the recent-pipeline prefix. Recent pipelines
|
|
// retain parallel creation; a single worker continues the complete tail. Explicit thread counts override this
|
|
// policy, and installer preparation uses its own queue at full concurrency. The CAS makes the prefix boundary
|
|
// exact even when two workers reach it together. No pipeline creation or dispatch waits on this queue lock.
|
|
static size_t lb_next(int worker) {
|
|
int64_t next = __atomic_load_n(&LB.next, __ATOMIC_RELAXED);
|
|
for (;;) {
|
|
if (LB.stop || (size_t)next >= LB.n ||
|
|
(LB.idle && LB.tail_one && worker > 0 && (size_t)next >= LB.n_keep)) return LB.n;
|
|
if (__atomic_compare_exchange_n(&LB.next, &next, next + 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
|
|
return (size_t)next;
|
|
}
|
|
}
|
|
|
|
// ---- preparation that outlives a driver crash. A bug in the driver's shader compiler can kill the process inside
|
|
// vkCreate*Pipelines: NVIDIA 615.78.08 on an RTX 5080 crashed on the same vertex library at the same point of every
|
|
// preparation, natively and under Wine, so preparation never finished (issue #30 and a report on PR #60).
|
|
// While lb_warm_prepared builds, slot k of <dir>/shader-prepare-journal.bin holds the variant its worker k has handed to
|
|
// the driver (input id and stage; zeros between variants). The writes go straight to the OS, so they outlast the
|
|
// process, and a run that finishes deletes the file. The next preparation or boot (skip_init) moves the variants a
|
|
// leftover journal names into <dir>/shader-prepare-skip.txt with this GPU and driver version, and preparation and the
|
|
// boot prebuild (lb_worker) skip what that file lists for them. The variants the other workers were building at the
|
|
// crash are skipped along with the culprit; like it, they build at their first draw, where the driver can still crash.
|
|
// Another driver version builds them again, and so does deleting the file. tools/prepare_shaders.py runs the build
|
|
// again (--warm-shaders) after a run that left a journal.
|
|
#define JR_SLOT 64 // bytes per journal slot: input id, stage, zeros
|
|
#define JR_SLOTS 64 // lb_warm_prepared's thread cap
|
|
|
|
static struct {
|
|
PlatFile *journal; // while lb_warm_prepared builds
|
|
uint8_t (*iid)[32]; // the skip list's variants for this GPU and driver
|
|
size_t n;
|
|
int loaded;
|
|
} SK;
|
|
|
|
static int cache_dir(char *dir, size_t n) { // the folder of pipeline-cache.bin and shader-prepared.bin
|
|
snprintf(dir, n, "%s", P.cache_path);
|
|
char *cut = plat_last_sep(dir);
|
|
if (!cut) return 0;
|
|
*cut = 0;
|
|
return 1;
|
|
}
|
|
|
|
static void gpu_key(char *out, size_t n) { // vendor:device:driver version, as the skip list names a GPU and driver
|
|
const VkPhysicalDeviceProperties *p = &vk_ctx()->props;
|
|
snprintf(out, n, "%04x:%04x:%08x", p->vendorID, p->deviceID, p->driverVersion);
|
|
}
|
|
|
|
static const char *stage_name(int st) {
|
|
return st == GFX_STAGE_VS ? "vertex" : st == GFX_STAGE_FS ? "fragment" : st == GFX_STAGE_CS ? "compute" : "other";
|
|
}
|
|
|
|
static int hex_digit(int c) {
|
|
return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1;
|
|
}
|
|
|
|
// A journal a crashed run left into the skip list, then the skip list's variants for this GPU and driver into SK.
|
|
static void skip_init(const char *dir) {
|
|
if (SK.loaded) return;
|
|
SK.loaded = 1;
|
|
char jp[PLAT_PATH_MAX * 2 + 48], sp[PLAT_PATH_MAX * 2 + 48], key[32];
|
|
snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir);
|
|
snprintf(sp, sizeof sp, "%s/shader-prepare-skip.txt", dir);
|
|
gpu_key(key, sizeof key);
|
|
uint64_t jn = 0, sn = 0;
|
|
uint8_t *j = plat_file_read(jp, &jn);
|
|
char *s = (char *)plat_file_read(sp, &sn);
|
|
int keep_journal = 0;
|
|
if (j) {
|
|
static const char head[] =
|
|
"# Shader variants the graphics driver was building when the process died during preparation\n"
|
|
"# (runtime/vk_pipeline.c skip_init): input id, stage, GPU vendor:device:driver version. Preparation skips\n"
|
|
"# them on that GPU and driver. Delete this file to build them again.\n";
|
|
size_t cap = (s ? sn : sizeof head) + (jn / JR_SLOT + 1) * 128, len = 0;
|
|
char *out = malloc(cap);
|
|
int added = 0;
|
|
if (out) {
|
|
len = s ? sn : sizeof head - 1;
|
|
memcpy(out, s ? s : head, len);
|
|
if (len && out[len - 1] != '\n') out[len++] = '\n';
|
|
for (uint64_t o = 0; o + 33 <= jn; o += JR_SLOT) {
|
|
int set = 0;
|
|
for (int b = 0; b < 32; b++) set |= j[o + b];
|
|
if (!set) continue;
|
|
char hex[65];
|
|
for (int b = 0; b < 32; b++) snprintf(hex + 2 * b, 3, "%02x", j[o + b]);
|
|
len += (size_t)snprintf(out + len, cap - len, "%s %s %s\n", hex, stage_name(j[o + 32]), key);
|
|
added++;
|
|
}
|
|
out[len] = 0; // parsed below as the list's text
|
|
}
|
|
if (added && plat_file_write_atomic(sp, out, len)) {
|
|
rt_log("vk_pipeline: the last shader preparation died while the driver built %d variants; %s now lists them "
|
|
"and preparation skips them on this GPU and driver (%s)", added, sp, key);
|
|
free(s), s = out, sn = len, out = NULL;
|
|
} else if (added) {
|
|
rt_log("vk_pipeline: cannot write %s; %s stays for the next run", sp, jp);
|
|
keep_journal = 1;
|
|
}
|
|
free(out);
|
|
free(j);
|
|
}
|
|
if (!keep_journal) plat_file_delete(jp);
|
|
for (char *line = s, *next; line && *line; line = next) {
|
|
next = strchr(line, '\n');
|
|
if (next) *next++ = 0;
|
|
char hex[65], st[16], k[32];
|
|
uint8_t iid[32];
|
|
int ok = line[0] != '#' && sscanf(line, "%64s %15s %31s", hex, st, k) == 3 && strlen(hex) == 64 &&
|
|
!strcmp(k, key);
|
|
for (int b = 0; ok && b < 32; b++) {
|
|
int hi = hex_digit(hex[2 * b]), lo = hex_digit(hex[2 * b + 1]);
|
|
if (hi < 0 || lo < 0) ok = 0;
|
|
else iid[b] = (uint8_t)(hi << 4 | lo);
|
|
}
|
|
uint8_t (*grown)[32] = ok ? realloc(SK.iid, (SK.n + 1) * sizeof *SK.iid) : NULL;
|
|
if (grown) memcpy(grown[SK.n++], iid, 32), SK.iid = grown;
|
|
}
|
|
free(s);
|
|
if (SK.n) rt_log("vk_pipeline: %zu shader variants skipped ahead of play on this GPU and driver (%s)", SK.n, sp);
|
|
}
|
|
|
|
static int skip_listed(const uint8_t iid[32]) {
|
|
for (size_t i = 0; i < SK.n; i++)
|
|
if (!memcmp(SK.iid[i], iid, 32)) return 1;
|
|
return 0;
|
|
}
|
|
|
|
static void skip_count(void) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.skipped++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
}
|
|
|
|
// Journal slot `slot`: the variant e its worker hands to the driver next, or zeros (e NULL) when the build returned.
|
|
static void journal_mark(int slot, const GfxWarmEnt *e) {
|
|
if (!SK.journal || slot < 0 || slot >= JR_SLOTS) return;
|
|
uint8_t rec[JR_SLOT] = {0};
|
|
if (e) memcpy(rec, e->iid, 32), rec[32] = e->stage;
|
|
plat_file_pwrite(SK.journal, rec, sizeof rec, (uint64_t)slot * JR_SLOT);
|
|
}
|
|
|
|
// Compute shaders whose pipeline a thread is compiling now (P.lock): a cold driver takes about 2 s for one of Ocarina's
|
|
// large compute shaders. The replay used to compile under P.lock, which held every other pipeline lookup that long, and
|
|
// compiled a second copy of a pipeline the boot prebuild (lb_compute) was already compiling.
|
|
#define CS_FLIGHT 32
|
|
static const GfxShader *cs_flight[CS_FLIGHT];
|
|
|
|
static int cs_flight_has(const GfxShader *cs) {
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (cs_flight[i] == cs) return 1;
|
|
return 0;
|
|
}
|
|
|
|
static volatile uint8_t cs_flight_want[CS_FLIGHT]; // a dispatch waits for the slot's pipeline (vk_pipeline_compute)
|
|
static PlatThread *cs_flight_th[CS_FLIGHT]; // the boot prebuild thread compiling it below normal priority
|
|
|
|
// th: the thread a waiting dispatch raises to normal priority, NULL when it compiles at normal priority already.
|
|
static int cs_flight_add(const GfxShader *cs, PlatThread *th) { // 0 when the table is full (the caller compiles anyway)
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (!cs_flight[i]) return cs_flight[i] = cs, cs_flight_want[i] = 0, cs_flight_th[i] = th, 1;
|
|
return 0;
|
|
}
|
|
|
|
// A dispatch waits for cs's build: a builder that would only have filled the pipeline cache keeps the pipeline. Returns
|
|
// the thread to raise, the first time.
|
|
static PlatThread *cs_flight_wanted(const GfxShader *cs) {
|
|
PlatThread *th = NULL;
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (cs_flight[i] == cs && !cs_flight_want[i]) cs_flight_want[i] = 1, th = cs_flight_th[i];
|
|
return th;
|
|
}
|
|
|
|
// Without P.lock, for the thread that holds cs's slot: whether a dispatch came to wait.
|
|
static int cs_flight_is_wanted(const GfxShader *cs) {
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (__atomic_load_n(&cs_flight[i], __ATOMIC_RELAXED) == cs) return cs_flight_want[i];
|
|
return 0;
|
|
}
|
|
|
|
static int cs_flight_remove(const GfxShader *cs) { // 1 when a dispatch waited for it
|
|
int want = 0;
|
|
for (int i = 0; i < CS_FLIGHT; i++)
|
|
if (cs_flight[i] == cs) cs_flight[i] = NULL, want |= cs_flight_want[i];
|
|
return want;
|
|
}
|
|
|
|
// ---- compute pipelines the boot keeps (compute-used.bin next to pipeline-keys.bin; NX2NAVI_COMPUTE_KEEP)
|
|
// The boot prebuild (lb_compute) used to build the pipeline of every compute shader of the warm list (the ones play
|
|
// recorded and the prepared ones, shader-prepared.bin) and keep all of them for the process: 4,066 on an installed
|
|
// copy (42,344 variants in its list, 42,142 of them prepared), of which a boot into the first village dispatched 95.
|
|
// What that cost was twenty of them. A copy's list has about twenty compute shaders of 0.9 to 2.0 million SPIR-V
|
|
// words: the game's tensor-core upscaler programs, which the game never dispatches natively (vk_exec_draw.c
|
|
// native_tensor) and which no run of the usage record below ever dispatched. A pipeline of one holds about 100 MiB
|
|
// of device memory from the moment it exists, used or not, and the driver keeps that memory when the pipeline is
|
|
// destroyed. On an RTX 5090 (NVIDIA 621.01) a boot pass that read all 4,066 pipelines from pipeline-cache.bin took the
|
|
// device memory from 386 to 2,398 MiB although it destroyed 3,949 of them at once; the same pass before the twenty
|
|
// were in the cache left it at 386, and building only the twenty took it to 1,370 (--probe-prewarm, 2026-11-10; the
|
|
// game showed 608 to 2,743 MiB over the same stage). The 4,046 others are cheap: all of them kept cost 2 MiB of device
|
|
// memory and 0.7 GB of the process's commit. So a copy with prepared shaders and a filled pipeline cache held 2 GiB
|
|
// for nothing from the title screen on, on cards that need that memory for textures.
|
|
// What the boot does now:
|
|
// - A compute shader over LB.big_words is built ahead only when a run has dispatched it (lb_compute, its keep
|
|
// argument). One that no run dispatched is not created at all: not from the pipeline caches and not by the
|
|
// background builder (bg_main). If the game does dispatch one for the first time, that dispatch builds it, which
|
|
// holds the frame for the compile once (0.65 to 1.9 s measured for such a shader), and the record has it built ahead
|
|
// from the next boot on.
|
|
// - Of the others the boot keeps the pipelines of the NX2NAVI_COMPUTE_KEEP (default 512) dispatched most recently, and
|
|
// builds every other one into the pipeline cache and destroys it at once (lb_compute keep 0), from a copy of its
|
|
// shader that is released again (lb_worker): the cache then holds it, so its first dispatch creates it without
|
|
// compiling. On that machine 94 first dispatches in play took 0.24 ms on average and 1.6 ms at most, 14 of them in
|
|
// the busiest 33 ms; reading the shader from the disk at that moment adds 0.05 to 0.15 ms (--probe-prewarm).
|
|
// The record: one entry per compute shader a run dispatched (its input id) with the number of the last run that did;
|
|
// each run that dispatches anything takes the next number. Kept at boot are the keep_n entries with the highest
|
|
// numbers, so a short run does not push out what a long one used, and nothing is forgotten until newer shaders take
|
|
// its place. Without a record (the first run that has one) nothing is kept and the run pays the first dispatches once.
|
|
// NX2NAVI_COMPUTE_KEEP=all builds and keeps every compute shader of the warm list, the large ones too, as before; 0
|
|
// keeps none.
|
|
#define CU_VERSION 1
|
|
typedef struct { char magic[8]; uint32_t version, rec_size, run, pad; } CuHeader;
|
|
typedef struct { uint8_t iid[32]; uint32_t run, pad; } CuRec;
|
|
static struct {
|
|
PlatMutex lock;
|
|
int opened, dirty;
|
|
char path[PLAT_PATH_MAX * 2 + 32];
|
|
CuRec *rec;
|
|
uint8_t *keep; // per record read at boot: among the keep_n dispatched most recently
|
|
uint32_t n, cap, n_file; // n_file: the records read at boot (keep has one byte for each)
|
|
uint32_t *slot; // open addressing by the id's first bytes: record index + 1, 0 empty
|
|
uint32_t slot_cap;
|
|
uint32_t run; // this run's number
|
|
int quiet; // --probe-prewarm: its dispatches are not the game's, the file stays as it is
|
|
int32_t keep_n; // NX2NAVI_COMPUTE_KEEP; -1: all
|
|
uint32_t kept; // records flagged in keep
|
|
} CU = {.lock = PLAT_MUTEX_INIT};
|
|
|
|
// What the first dispatches of compute shaders cost this run (under P.lock): each creates its pipeline on the replay
|
|
// thread. burst: the creations that fell into one window of CF_WINDOW_MS, the worst such window kept, which is what a
|
|
// player would feel as one long frame when a new area starts.
|
|
#define CF_WINDOW_MS 33.0
|
|
static struct {
|
|
uint64_t n, hit, pre_used; // hit: the pipeline cache held it (cc_hit 1); pre_used: kept from the boot and dispatched
|
|
double ms, max_ms, hit_ms;
|
|
double win_t0, win_ms, worst_ms;
|
|
uint32_t win_n, worst_n;
|
|
} CF;
|
|
|
|
static uint32_t *cu_slot(const uint8_t iid[32]) { // the id's slot, or the empty one it would take. Caller holds CU.lock.
|
|
uint64_t h;
|
|
memcpy(&h, iid, 8);
|
|
for (uint32_t j = (uint32_t)(h * 0x9e3779b97f4a7c15ull >> 32) & (CU.slot_cap - 1);; j = (j + 1) & (CU.slot_cap - 1))
|
|
if (!CU.slot[j] || !memcmp(CU.rec[CU.slot[j] - 1].iid, iid, 32)) return &CU.slot[j];
|
|
}
|
|
|
|
// Room for one more record and a table at most half full. 0 when out of memory. Caller holds CU.lock.
|
|
static int cu_room(void) {
|
|
if (CU.n == CU.cap) {
|
|
uint32_t cap = CU.cap ? CU.cap * 2 : 1024;
|
|
CuRec *t = realloc(CU.rec, cap * sizeof *t);
|
|
if (!t) return 0;
|
|
CU.rec = t, CU.cap = cap;
|
|
}
|
|
if ((CU.n + 1) * 2 > CU.slot_cap) {
|
|
uint32_t cap = CU.slot_cap ? CU.slot_cap * 2 : 4096;
|
|
uint32_t *t = calloc(cap, sizeof *t), *old = CU.slot;
|
|
if (!t) return 0;
|
|
CU.slot = t, CU.slot_cap = cap;
|
|
for (uint32_t i = 0; i < CU.n; i++) *cu_slot(CU.rec[i].iid) = i + 1;
|
|
free(old);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
static int cu_run_cmp(const void *a, const void *b) { // {run, index} pairs, the highest run first
|
|
const uint32_t *x = a, *y = b;
|
|
return x[0] != y[0] ? (x[0] < y[0] ? 1 : -1) : (x[1] > y[1]) - (x[1] < y[1]);
|
|
}
|
|
|
|
// The number of records in a compute-used.bin image (f, size) with its header in *h, or -1 when it is not one.
|
|
static long cu_parse(const uint8_t *f, uint64_t size, CuHeader *h) {
|
|
if (!f || size < sizeof *h) return -1;
|
|
memcpy(h, f, sizeof *h);
|
|
if (memcmp(h->magic, "NX2NCUSE", 8) || h->version != CU_VERSION || h->rec_size != sizeof(CuRec)) return -1;
|
|
return (long)((size - sizeof *h) / sizeof(CuRec));
|
|
}
|
|
|
|
// Reads the record and marks what the boot keeps. Caller holds P.lock (ensure_init), after keys_open.
|
|
static void cu_open(void) {
|
|
if (!A.on || !K.path[0]) return;
|
|
const char *e = cfg_value("NX2NAVI_COMPUTE_KEEP");
|
|
plat_mutex_lock(&CU.lock);
|
|
int count = cfg_int("NX2NAVI_COMPUTE_KEEP"); // a value that is neither "all" nor a number reads as the default
|
|
CU.keep_n = !strcmp(e, "all") ? -1 : count < 0 ? 0 : count;
|
|
snprintf(CU.path, sizeof CU.path, "%s", K.path);
|
|
char *cut = plat_last_sep(CU.path);
|
|
snprintf(cut ? cut + 1 : CU.path, sizeof CU.path - (size_t)(cut ? cut + 1 - CU.path : 0), "compute-used.bin");
|
|
uint64_t size = 0;
|
|
uint8_t *f = plat_file_read(CU.path, &size);
|
|
CuHeader h;
|
|
long n = cu_parse(f, size, &h);
|
|
if (n >= 0) {
|
|
for (long i = 0; i < n && cu_room(); i++) {
|
|
CuRec r;
|
|
memcpy(&r, f + sizeof h + (size_t)i * sizeof r, sizeof r);
|
|
uint32_t *s = cu_slot(r.iid);
|
|
if (*s) continue; // an id twice: the first entry stands
|
|
CU.rec[CU.n] = r, *s = ++CU.n;
|
|
}
|
|
CU.run = h.run;
|
|
} else if (f) {
|
|
rt_log("vk_pipeline: %s has another format or version; it is rewritten", CU.path);
|
|
}
|
|
free(f);
|
|
CU.n_file = CU.n, CU.run++;
|
|
uint32_t (*order)[2] = CU.n ? malloc(CU.n * sizeof *order) : NULL;
|
|
CU.keep = CU.n ? calloc(CU.n, 1) : NULL;
|
|
if (order && CU.keep) {
|
|
for (uint32_t i = 0; i < CU.n; i++) order[i][0] = CU.rec[i].run, order[i][1] = i;
|
|
qsort(order, CU.n, sizeof *order, cu_run_cmp);
|
|
for (uint32_t i = 0; i < CU.n && (CU.keep_n < 0 || i < (uint32_t)CU.keep_n); i++) CU.keep[order[i][1]] = 1, CU.kept++;
|
|
}
|
|
free(order);
|
|
CU.opened = 1;
|
|
rt_log("vk_pipeline: %s: %u compute shaders dispatched in earlier runs; the boot keeps the pipelines of %s%u "
|
|
"dispatched most recently (NX2NAVI_COMPUTE_KEEP %s)", CU.path, CU.n,
|
|
CU.keep_n < 0 ? "every recorded compute shader, among them the " : "the ", CU.kept, e);
|
|
plat_mutex_unlock(&CU.lock);
|
|
}
|
|
|
|
// 1 when the boot keeps the pipeline of the compute shader with this input id.
|
|
static int cu_keep(const uint8_t iid[32]) {
|
|
plat_mutex_lock(&CU.lock);
|
|
int keep = CU.keep_n < 0 || !CU.opened; // without the record (no cache folder): as before, every one
|
|
if (!keep && CU.slot_cap) {
|
|
uint32_t s = *cu_slot(iid);
|
|
keep = s && s <= CU.n_file && CU.keep && CU.keep[s - 1];
|
|
}
|
|
plat_mutex_unlock(&CU.lock);
|
|
return keep;
|
|
}
|
|
|
|
// The first dispatch of cs in this run: its entry takes this run's number. The runtime's own shaders (no input id)
|
|
// are built at their first use every run and are not recorded.
|
|
static void cu_note(const GfxShader *cs) {
|
|
static const uint8_t none[32];
|
|
const uint8_t *iid = gfx_shader_input_id(cs);
|
|
if (!CU.opened || !memcmp(iid, none, 32)) return;
|
|
plat_mutex_lock(&CU.lock);
|
|
if (cu_room()) {
|
|
uint32_t *s = cu_slot(iid);
|
|
if (!*s) {
|
|
memset(&CU.rec[CU.n], 0, sizeof *CU.rec);
|
|
memcpy(CU.rec[CU.n].iid, iid, 32);
|
|
*s = ++CU.n;
|
|
}
|
|
if (CU.rec[*s - 1].run != CU.run) CU.rec[*s - 1].run = CU.run, CU.dirty = 1;
|
|
}
|
|
plat_mutex_unlock(&CU.lock);
|
|
}
|
|
|
|
// Writes the record to path. 1 when written.
|
|
// One writer at a time, from the copy of the record to the end of the file write: the autosave thread and a save at
|
|
// exit can both be here, and the older copy must not land on the newer file.
|
|
static PlatMutex cu_io = PLAT_MUTEX_INIT;
|
|
|
|
static int cu_write(const char *path) {
|
|
plat_mutex_lock(&cu_io);
|
|
plat_mutex_lock(&CU.lock);
|
|
uint32_t n = CU.n;
|
|
plat_mutex_unlock(&CU.lock);
|
|
// allocated outside CU.lock, which a dispatch takes (cu_note); room for the entries that come meanwhile
|
|
size_t cap = sizeof(CuHeader) + ((size_t)n + 256) * sizeof(CuRec), size = 0;
|
|
uint8_t *out = malloc(cap);
|
|
plat_mutex_lock(&CU.lock);
|
|
CuHeader h = {{'N', 'X', '2', 'N', 'C', 'U', 'S', 'E'}, CU_VERSION, sizeof(CuRec), CU.run, 0};
|
|
if (out && sizeof h + CU.n * sizeof(CuRec) <= cap) {
|
|
size = sizeof h + CU.n * sizeof(CuRec);
|
|
memcpy(out, &h, sizeof h);
|
|
if (CU.n) memcpy(out + sizeof h, CU.rec, CU.n * sizeof(CuRec));
|
|
CU.dirty = 0;
|
|
}
|
|
plat_mutex_unlock(&CU.lock);
|
|
int ok = size && plat_file_write_atomic(path, out, size);
|
|
if (!ok) { // still to be written: the next flush tries again
|
|
plat_mutex_lock(&CU.lock);
|
|
CU.dirty = 1;
|
|
plat_mutex_unlock(&CU.lock);
|
|
}
|
|
free(out);
|
|
plat_mutex_unlock(&cu_io);
|
|
return ok;
|
|
}
|
|
|
|
// Writes the record when a dispatch changed it (the autosave thread and the save at exit).
|
|
static void cu_flush(void) {
|
|
if (!CU.opened || CU.quiet || !*(volatile int *)&CU.dirty) return;
|
|
if (!cu_write(CU.path)) LOG_ONCE("vk_pipeline: cannot write %s; the next saves try again", CU.path);
|
|
}
|
|
|
|
// A compute pipeline created at a dispatch took ms. Caller holds P.lock.
|
|
static void cf_note(double ms, int hit) {
|
|
double now = now_ms();
|
|
CF.n++, CF.ms += ms;
|
|
if (ms > CF.max_ms) CF.max_ms = ms;
|
|
if (hit == 1) CF.hit++, CF.hit_ms += ms;
|
|
if (now - CF.win_t0 > CF_WINDOW_MS) CF.win_t0 = now, CF.win_n = 0, CF.win_ms = 0;
|
|
CF.win_n++, CF.win_ms += ms;
|
|
if (CF.win_ms > CF.worst_ms) CF.worst_ms = CF.win_ms, CF.worst_n = CF.win_n;
|
|
}
|
|
|
|
static void cf_log(void) {
|
|
plat_mutex_lock_shared(&P.lock);
|
|
typeof(CF) c = CF; // the line is written after the unlock: every dispatch needs P.lock, and this
|
|
plat_mutex_unlock_shared(&P.lock); // runs on the autosave thread below normal priority
|
|
if (c.n || c.pre_used)
|
|
rt_log("vk_pipeline: compute pipelines created at their first dispatch: %llu (avg %.2f ms, longest %.2f ms; the "
|
|
"pipeline cache held %llu of them, avg %.2f ms); the %.0f ms that held the most of it: %u pipelines, "
|
|
"%.1f ms; of the compute pipelines kept from the boot, %llu were dispatched",
|
|
(unsigned long long)c.n, c.n ? c.ms / c.n : 0.0, c.max_ms, (unsigned long long)c.hit,
|
|
c.hit ? c.hit_ms / c.hit : 0.0, CF_WINDOW_MS, c.worst_n, c.worst_ms, (unsigned long long)c.pre_used);
|
|
}
|
|
|
|
// The background builder of large compute shaders (comment at bg_main).
|
|
static struct {
|
|
const GfxShader **cs;
|
|
uint8_t (*iid)[32];
|
|
uint64_t *words;
|
|
size_t n, cap;
|
|
PlatThread *th;
|
|
const GfxShader *volatile building; // the shader on the driver now, NULL between builds
|
|
volatile int32_t raised; // a dispatch waited for it and raised the thread
|
|
uint64_t built, failed, there;
|
|
double ms;
|
|
} BG;
|
|
|
|
VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) {
|
|
if (info) *info = NULL;
|
|
if (!cs || !ensure_init()) return VK_NULL_HANDLE;
|
|
struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
plat_mutex_lock(&P.lock);
|
|
Pipe *p = pipe_find(&k, sizeof k, h);
|
|
double w0 = !p && cs_flight_has(cs) ? now_ms() : 0;
|
|
int bg = 0;
|
|
while (!p && cs_flight_has(cs)) { // another thread is compiling it: its result, not a second compile
|
|
// a build that only fills the pipeline cache keeps its pipeline for this dispatch, and a boot prebuild thread
|
|
// compiling it below normal priority goes to normal
|
|
PlatThread *builder = cs_flight_wanted(cs);
|
|
if (builder) plat_thread_set_priority(builder, PLAT_PRIO_NORMAL);
|
|
if (BG.building == cs && BG.th && !plat_xchg32(&BG.raised, 1)) // the background builder: normal priority now
|
|
plat_thread_set_priority(BG.th, PLAT_PRIO_NORMAL), bg = 1;
|
|
plat_mutex_unlock(&P.lock);
|
|
plat_sleep_ms(1);
|
|
plat_mutex_lock(&P.lock);
|
|
p = pipe_find(&k, sizeof k, h);
|
|
}
|
|
if (w0 && now_ms() - w0 >= 2.0) {
|
|
static int logs;
|
|
if (logs++ < 16)
|
|
rt_log("vk_exec: a dispatch waited %.0f ms for its compute pipeline (%s)", now_ms() - w0,
|
|
bg ? "built on the background thread" : "built by another thread");
|
|
}
|
|
if (p) {
|
|
P.stats.hits++;
|
|
} else {
|
|
const char *why = NULL;
|
|
Bindings B = {0};
|
|
Layout *l = compute_layout(cs, &B, &why);
|
|
VkPipeline pipe = VK_NULL_HANDLE;
|
|
if (!why) {
|
|
int flight = cs_flight_add(cs, NULL);
|
|
plat_mutex_unlock(&P.lock); // the compile runs without the lock
|
|
uint64_t p0 = perf_begin();
|
|
double c0 = now_ms();
|
|
pipe = compute_create(cs, l, 0);
|
|
double cms = now_ms() - c0;
|
|
int hit = cc_hit;
|
|
perf_end(PERF_PIPE, p0);
|
|
plat_mutex_lock(&P.lock);
|
|
if (flight) cs_flight_remove(cs);
|
|
if (!pipe) why = "pipeline creation failed";
|
|
else cf_note(cms, hit);
|
|
}
|
|
if ((p = pipe_find(&k, sizeof k, h))) { // another thread added it meanwhile (no flight slot was free)
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
} else {
|
|
if (why) {
|
|
P.stats.failures++;
|
|
rt_log("vk_pipeline: compute pipeline not built: %s", why);
|
|
} else {
|
|
P.stats.compute++;
|
|
LB.cs_draw++;
|
|
}
|
|
p = pipe_add(&k, sizeof k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_COMPUTE, 0);
|
|
}
|
|
}
|
|
if (p && !p->used) { // the first dispatch of this run: the boot of the next runs keeps this shader's pipeline
|
|
p->used = 1;
|
|
CF.pre_used += p->pre;
|
|
if (p->info.pipeline) cu_note(cs);
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
if (!p) return VK_NULL_HANDLE;
|
|
if (info) *info = &p->info;
|
|
return p->info.pipeline;
|
|
}
|
|
|
|
static void lb_lock(void) {
|
|
plat_thread_priority(PLAT_PRIO_NORMAL);
|
|
plat_mutex_lock(&P.lock);
|
|
}
|
|
|
|
static void lb_unlock(void) {
|
|
plat_mutex_unlock(&P.lock);
|
|
if (LB.idle) plat_thread_priority(PLAT_PRIO_IDLE);
|
|
}
|
|
|
|
static void lb_count(int kind, int what, double ms) { // what: 0 built, 1 already there, 2 failed, 3 cache only
|
|
plat_mutex_lock(&LB.stat);
|
|
if (what == 3) LB.warmed[kind]++, LB.warm_ms[kind] += ms;
|
|
else (what == 0 ? LB.built : what == 1 ? LB.have : LB.fail)[kind]++, LB.ms[kind] += ms;
|
|
plat_mutex_unlock(&LB.stat);
|
|
}
|
|
|
|
// vs_union_missing for vs alone, without entering vs into the union table (it holds VS_OUTS_CAP shaders).
|
|
static uint32_t vs_union_peek(const GfxShader *vs, GfxShaderIo fin[32]) {
|
|
GfxShaderIo vout[32];
|
|
uint32_t need = 0, have = gfx_shader_outputs(vs, vout), missing = 0;
|
|
memset(fin, 0, 32 * sizeof *fin);
|
|
plat_mutex_lock_shared(&VO.lock);
|
|
uint32_t j = (uint32_t)(((uintptr_t)vs >> 4) * 0x9e3779b1u) & (VS_OUTS_CAP - 1);
|
|
while (VO.vs[j] && VO.vs[j] != vs) j = (j + 1) & (VS_OUTS_CAP - 1);
|
|
if (VO.vs[j]) need = VO.mask[j], memcpy(fin, VO.io[j], 32 * sizeof *fin);
|
|
plat_mutex_unlock_shared(&VO.lock);
|
|
if (P.vs_outs_all)
|
|
for (int i = 0; i < 32; i++)
|
|
if (!(need >> i & 1)) fin[i] = (GfxShaderIo){GFX_TYPE_FLOAT, 4, 0}, need |= 1u << i;
|
|
for (int i = 0; i < 32; i++)
|
|
if (need >> i & 1 && !(have >> i & 1)) missing |= 1u << i;
|
|
return missing;
|
|
}
|
|
|
|
// A compute shader over LB.big_words that no pipeline cache holds: its first dispatch builds it (vk_pipeline_compute).
|
|
static void lb_defer(const uint8_t iid[32], uint64_t words) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.deferred++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
rt_log("vk_pipeline: stage library prebuild: compute shader %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) %s: "
|
|
"left to its first dispatch (NX2NAVI_LIB_PREBUILD_BIG %llu)", iid[0], iid[1], iid[2], iid[3], iid[4], iid[5],
|
|
iid[6], iid[7], (unsigned long long)words,
|
|
P.cache_control ? "not in the pipeline caches" : "too large to build without a cache probe (no pipelineCreationCacheControl)",
|
|
(unsigned long long)LB.big_words);
|
|
}
|
|
|
|
// ---- large compute shaders on a background thread (NX2NAVI_LIB_PREBUILD_BIG_BG, default 1; 0 leaves them to their
|
|
// first dispatch as before)
|
|
// The boot prebuild leaves a compute shader over LB.big_words that no pipeline cache holds (lb_compute). Its first
|
|
// dispatch then compiled it on the replay thread: frames of 0.65-1.9 s in Kokiri Forest on an RTX 5090 after a
|
|
// translator change. One thread builds them instead, once the boot prebuild is done, one at a time and the smallest
|
|
// first: four compiled side by side grew the process by 16-19 GB on an RTX 4090 (comment at LB_BIG_WORDS). The shader
|
|
// being built is in cs_flight, so a dispatch that needs it waits for this build (vk_pipeline_compute raises the
|
|
// thread to normal priority then) instead of compiling a second copy. What is built is entered for the process, as
|
|
// lb_compute's pipelines are, and lands in pipeline-cache.bin for the next boot.
|
|
|
|
static int bg_on(void) {
|
|
static int on = -1;
|
|
if (on < 0) on = cfg_bool("NX2NAVI_LIB_PREBUILD_BIG_BG");
|
|
return on;
|
|
}
|
|
|
|
// A large compute shader no cache holds goes to the background builder (lb_worker threads; under LB.stat).
|
|
static void bg_queue(const GfxShader *cs, const uint8_t iid[32], uint64_t words) {
|
|
plat_mutex_lock(&LB.stat);
|
|
if (BG.n == BG.cap) {
|
|
size_t cap = BG.cap ? BG.cap * 2 : 16;
|
|
const GfxShader **c = realloc(BG.cs, cap * sizeof *c);
|
|
if (c) BG.cs = c;
|
|
uint8_t (*i)[32] = realloc(BG.iid, cap * sizeof *i);
|
|
if (i) BG.iid = i;
|
|
uint64_t *w = realloc(BG.words, cap * sizeof *w);
|
|
if (w) BG.words = w;
|
|
if (c && i && w) BG.cap = cap;
|
|
}
|
|
int ok = BG.n < BG.cap;
|
|
if (ok) BG.cs[BG.n] = cs, memcpy(BG.iid[BG.n], iid, 32), BG.words[BG.n] = words, BG.n++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
if (!ok) {
|
|
lb_defer(iid, words);
|
|
return;
|
|
}
|
|
rt_log("vk_pipeline: stage library prebuild: compute shader %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) not in "
|
|
"the pipeline caches: built on the background thread after the prebuild (NX2NAVI_LIB_PREBUILD_BIG_BG)",
|
|
iid[0], iid[1], iid[2], iid[3], iid[4], iid[5], iid[6], iid[7], (unsigned long long)words);
|
|
}
|
|
|
|
// NX2NAVI_BIG_BUILD_FREE_MB (default 8192; 0 no check): the system commit headroom a large compute build waits for.
|
|
// A build of one of these shaders took the process up by 1-3 GB on an RTX 5090 and four at once by 16-19 GB on an
|
|
// RTX 4090; started on a machine near its commit limit it can end the game. While the headroom is short the shader
|
|
// waits, and if the game dispatches it first it is built there, as without the background builder.
|
|
// The builds under way elsewhere count through their reservations at the memory gate (gate_enter), where this build
|
|
// then reserves GATE_BIG_MB of its own for as long as it runs, so the other background builds leave it its room.
|
|
static int bg_room(double *used, double *limit, double *held) {
|
|
static double need = -1;
|
|
if (need < 0) need = strtod(cfg_value("NX2NAVI_BIG_BUILD_FREE_MB"), NULL);
|
|
double proc;
|
|
plat_commit_info(&proc, used, limit);
|
|
if (!(need <= 0 || *limit < 0 || *limit - *used >= need)) return 0;
|
|
return gate_enter(GATE_BIG_MB, held);
|
|
}
|
|
|
|
// NX2NAVI_BIG_BUILD_MAX_MB (default 4096; 0 no cap): how far the builder may raise the process commit before it stops
|
|
// and leaves the rest to their first dispatch. One of these shaders costs the driver 1-4 GB: on a 31 GB machine four of
|
|
// them took the process from 15 to 29 GB and system commit to 60 of 65 GB, which is where a player feels the game
|
|
// stutter. The builder exists to remove hitches, not to trade them for paging, so a machine that cannot hold them all
|
|
// builds only what it can and pays one hitch per shader later.
|
|
static double bg_growth_cap(void) {
|
|
static double cap = -1;
|
|
if (cap < 0) cap = strtod(cfg_value("NX2NAVI_BIG_BUILD_MAX_MB"), NULL);
|
|
return cap;
|
|
}
|
|
|
|
// The builder compiles into a pipeline cache of its own (own_cache at pipe_create_in), not into P.cache. A create into
|
|
// P.cache holds cache_rw shared from start to end, and the cache is only saved at a moment when nobody holds it: on an
|
|
// RX 7900 XT (26.10.1) the largest of these shaders (4,874,502 SPIR-V words) was still compiling 590 s after the boot,
|
|
// pipeline-cache.bin was not written once in that time, and the game was closed before the build ended, so the next
|
|
// boot found neither that shader nor the pipelines built in play and started the same build again. bg_merge moves the
|
|
// result into P.cache when the build is done, at a moment with no build into P.cache running: the lock is tried as
|
|
// cache_save tries it, because a waiting exclusive request would hold back every new build.
|
|
static void bg_merge(VkPipelineCache own, int built) {
|
|
int locked = 0;
|
|
while (built && !LB.stop && !(locked = plat_mutex_trylock(&cache_rw))) plat_sleep_ms(2);
|
|
if (locked) {
|
|
// what play compiled: into the play cache when there is one, so its file gets it and not pipeline-cache.bin
|
|
VkResult r = vkMergePipelineCaches(P.device, P.play ? P.play : P.cache, 1, &own);
|
|
plat_mutex_unlock(&cache_rw);
|
|
if (r == VK_SUCCESS) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED);
|
|
else rt_log("vk_pipeline: background compute build: its cache did not merge into %s (%d)",
|
|
P.play ? P.play_path : P.cache_path, (int)r);
|
|
}
|
|
vkDestroyPipelineCache(P.device, own, NULL);
|
|
}
|
|
|
|
static void bg_main(void *arg) {
|
|
(void)arg;
|
|
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
// The queue can grow while this thread builds (bg_queue runs on the replay thread and reallocs these arrays), so
|
|
// the sort and every read below happen under LB.stat; an entry is copied out before its compile so no pointer into
|
|
// the arrays is held across it.
|
|
plat_mutex_lock(&LB.stat);
|
|
for (size_t i = 1; i < BG.n; i++) // smallest first: insertion sort of a handful
|
|
for (size_t j = i; j > 0 && BG.words[j] < BG.words[j - 1]; j--) {
|
|
const GfxShader *c = BG.cs[j];
|
|
BG.cs[j] = BG.cs[j - 1], BG.cs[j - 1] = c;
|
|
uint64_t w = BG.words[j];
|
|
BG.words[j] = BG.words[j - 1], BG.words[j - 1] = w;
|
|
uint8_t t[32];
|
|
memcpy(t, BG.iid[j], 32), memcpy(BG.iid[j], BG.iid[j - 1], 32), memcpy(BG.iid[j - 1], t, 32);
|
|
}
|
|
plat_mutex_unlock(&LB.stat);
|
|
double t_all = now_ms();
|
|
double c_start, c_used0, c_limit0;
|
|
plat_commit_info(&c_start, &c_used0, &c_limit0);
|
|
for (size_t i = 0; !LB.stop; i++) {
|
|
uint8_t idbuf[32];
|
|
uint64_t words = 0;
|
|
const GfxShader *cs;
|
|
plat_mutex_lock(&LB.stat);
|
|
int have = i < BG.n;
|
|
if (have) {
|
|
cs = BG.cs[i];
|
|
memcpy(idbuf, BG.iid[i], 32);
|
|
words = BG.words[i];
|
|
}
|
|
plat_mutex_unlock(&LB.stat);
|
|
if (!have) break;
|
|
const uint8_t *iid = idbuf;
|
|
struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
Bindings B = {0};
|
|
const char *why = NULL;
|
|
double used, limit, c_now;
|
|
plat_commit_info(&c_now, &used, &limit);
|
|
double cap = bg_growth_cap();
|
|
if (cap > 0 && c_now - c_start >= cap) {
|
|
rt_log("vk_pipeline: background compute build: stopping, the process grew %.0f MB in all (NX2NAVI_BIG_BUILD_MAX_MB "
|
|
"%.0f); the shaders left build at their first dispatch", c_now - c_start, cap);
|
|
break;
|
|
}
|
|
int waited = 0, no_room = 0;
|
|
double w0 = now_ms(), held = -1; // held: this build's reservation at the memory gate, -1 none
|
|
while (!LB.stop && !bg_room(&used, &limit, &held)) { // not under P.lock: a dispatch builds it meanwhile
|
|
held = -1;
|
|
if (!waited++)
|
|
rt_log("vk_pipeline: background compute build: waiting for commit headroom (system %.0f of %.0f MB, "
|
|
"NX2NAVI_BIG_BUILD_FREE_MB %s)", used, limit, cfg_value("NX2NAVI_BIG_BUILD_FREE_MB"));
|
|
if (now_ms() - w0 > 30000.0) { // waiting is only useful when the headroom will come back
|
|
rt_log("vk_pipeline: background compute build: no commit headroom after %.0f s; the shaders left build at "
|
|
"their first dispatch", (now_ms() - w0) / 1000.0);
|
|
no_room = 1;
|
|
break;
|
|
}
|
|
plat_sleep_ms(2000);
|
|
}
|
|
if (LB.stop || no_room) {
|
|
if (held >= 0 && !no_room) gate_leave(held); // stopped right after the gate let it in
|
|
break;
|
|
}
|
|
plat_mutex_lock(&P.lock);
|
|
int there = cs_flight_has(cs) || pipe_find(&k, sizeof k, h); // a dispatch got to it first
|
|
Layout *l = there ? NULL : compute_layout(cs, &B, &why);
|
|
int flight = l ? cs_flight_add(cs, NULL) : 0; // vk_pipeline_compute raises this thread through BG.raised
|
|
if (l) BG.building = cs;
|
|
plat_mutex_unlock(&P.lock);
|
|
if (!l) {
|
|
if (held >= 0) gate_leave(held);
|
|
if (there) BG.there++;
|
|
else BG.failed++, rt_log("vk_pipeline: background compute build: %s", why ? why : "no layout");
|
|
continue;
|
|
}
|
|
VkPipelineCache own = VK_NULL_HANDLE;
|
|
VkPipelineCacheCreateInfo oci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO};
|
|
if (P.cache && vkCreatePipelineCache(P.device, &oci, NULL, &own) != VK_SUCCESS) own = VK_NULL_HANDLE;
|
|
double t0 = now_ms();
|
|
own_cache = own; // without one the build goes into P.cache as before
|
|
VkPipeline pipe = compute_create(cs, l, 0);
|
|
own_cache = VK_NULL_HANDLE;
|
|
double ms = now_ms() - t0;
|
|
if (held >= 0) gate_leave(held);
|
|
// The pipeline stays: only a shader a run dispatched is queued here (lb_compute), and it is entered before
|
|
// the merge below, so a dispatch that comes meanwhile finds it.
|
|
plat_mutex_lock(&P.lock);
|
|
if (flight) cs_flight_remove(cs);
|
|
BG.building = NULL;
|
|
Pipe *e = NULL;
|
|
if (pipe && !pipe_find(&k, sizeof k, h) &&
|
|
(e = pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0))) {
|
|
e->pre = 1;
|
|
P.stats.compute++;
|
|
} else if (pipe) { // a dispatch built its own meanwhile (no flight slot was free), or out of memory
|
|
vkDestroyPipeline(P.device, pipe, NULL);
|
|
}
|
|
plat_mutex_unlock(&P.lock);
|
|
if (own) bg_merge(own, pipe != VK_NULL_HANDLE); // after the entry: a dispatch waiting for it does not wait for this
|
|
if (plat_xchg32(&BG.raised, 0)) plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
if (pipe) BG.built++, BG.ms += ms;
|
|
else BG.failed++;
|
|
double c_proc, c_used, c_limit;
|
|
plat_commit_info(&c_proc, &c_used, &c_limit);
|
|
rt_log("vk_pipeline: background compute build: %02x%02x%02x%02x%02x%02x%02x%02x (%llu SPIR-V words) %s in %.0f ms; "
|
|
"commit: process %.0f MB, system %.0f of %.0f MB", iid[0], iid[1], iid[2], iid[3], iid[4], iid[5], iid[6],
|
|
iid[7], (unsigned long long)words, pipe ? "built" : "failed", ms, c_proc, c_used, c_limit);
|
|
}
|
|
rt_log("vk_pipeline: background compute builds %s: %llu built (%.0f ms in all), %llu failed, %llu dispatched before "
|
|
"their turn; %.0f ms after the prebuild", LB.stop ? "stopped" : "done", (unsigned long long)BG.built, BG.ms,
|
|
(unsigned long long)BG.failed, (unsigned long long)BG.there, now_ms() - t_all);
|
|
}
|
|
|
|
// The last boot prebuild worker starts the builder when it queued anything.
|
|
static void bg_start(void) {
|
|
if (!BG.n || LB.stop || BG.th) return;
|
|
if (!(BG.th = plat_thread_new(bg_main, NULL, "big compute")))
|
|
rt_log("vk_pipeline: cannot start the background compute builder; %zu large compute shaders build at their first "
|
|
"dispatch", BG.n);
|
|
}
|
|
|
|
// keep 0: the pipeline is built for the pipeline cache only and destroyed at once, unless a dispatch came to wait for
|
|
// it meanwhile; a shader over LB.big_words is then not built at all (comment at compute-used.bin). cs may be a copy
|
|
// the caller releases afterwards when keep is 0: nothing here holds on to it then.
|
|
static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep) {
|
|
struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes
|
|
uint64_t h = key_hash(&k, sizeof k);
|
|
Bindings B = {0};
|
|
const char *why = NULL;
|
|
uint64_t words = 0;
|
|
gfx_shader_spirv(cs, &words);
|
|
int big = LB.big_words && words > LB.big_words; // only from the caches (LB_BIG_WORDS)
|
|
if (big && !keep) { // no run dispatched it: a pipeline of it would hold about 100 MiB of device memory unused
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.big_unused++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
return;
|
|
}
|
|
if (big && !P.cache_control) {
|
|
if (bg_on()) bg_queue(cs, iid, words);
|
|
else lb_defer(iid, words);
|
|
return;
|
|
}
|
|
lb_lock();
|
|
int busy = cs_flight_has(cs); // the replay compiles it now (vk_pipeline_compute)
|
|
Layout *l = busy || pipe_find(&k, sizeof k, h) ? NULL : compute_layout(cs, &B, &why);
|
|
PlatThread *me = LB.idle && lb_me >= 0 ? LB.th[lb_me] : NULL;
|
|
int have = !l && !why, flight = l ? cs_flight_add(cs, me) : 0;
|
|
lb_unlock();
|
|
if (!l) {
|
|
lb_count(2, have ? 1 : 2, 0);
|
|
return;
|
|
}
|
|
double t0 = now_ms();
|
|
// The compile runs below normal priority: after an update or a driver update every one of these pipelines is
|
|
// compiled again (about 140 ms each on an RTX 5090's driver, minutes of two threads for a long play history), and
|
|
// at normal priority those threads took cores from the game's own on a CPU with few of them. While the shader is in
|
|
// flight a dispatch of it waits for this build (vk_pipeline_compute) and raises the thread to normal: a compile left
|
|
// at idle priority under load held that frame for seconds. The dispatch may have come between the unlock and the
|
|
// call below, which would undo its raise, hence the look at the slot after it. Without a slot or a handle nobody
|
|
// could raise the thread, so it compiles at normal priority.
|
|
if (LB.idle && flight && me) {
|
|
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
if (cs_flight_is_wanted(cs)) plat_thread_priority(PLAT_PRIO_NORMAL);
|
|
} else if (LB.idle) {
|
|
plat_thread_priority(PLAT_PRIO_NORMAL);
|
|
}
|
|
VkPipeline pipe = compute_create(cs, l, big);
|
|
double ms = now_ms() - t0;
|
|
int compiled = pipe && cc_hit != 1, kept = 0;
|
|
lb_lock();
|
|
if (flight && cs_flight_remove(cs)) keep = 1; // a dispatch waits for it
|
|
Pipe *e = NULL;
|
|
if (pipe && pipe_find(&k, sizeof k, h)) {
|
|
vkDestroyPipeline(P.device, pipe, NULL);
|
|
have = 1;
|
|
} else if (pipe && keep && (e = pipe_add(&k, sizeof k, h, pipe, l, &B, VK_PIPELINE_BIND_POINT_COMPUTE, 0))) {
|
|
e->pre = 1, kept = 1;
|
|
P.stats.compute++;
|
|
} else if (pipe) {
|
|
vkDestroyPipeline(P.device, pipe, NULL);
|
|
// P.stats.compute does not count it, so the autosave would not see what its compile added to P.cache
|
|
if (compiled) __atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED);
|
|
}
|
|
lb_unlock();
|
|
if (!pipe && big && bg_on()) bg_queue(cs, iid, words);
|
|
else if (!pipe && big) lb_defer(iid, words);
|
|
else lb_count(2, !pipe ? 2 : have ? 1 : kept ? 0 : 3, ms);
|
|
}
|
|
|
|
// The pre-rasterization (vertex) or fragment-shader library of one shader with the LB state.
|
|
// keep 0: build into the driver's pipeline cache only (sh is a private copy; the library is destroyed).
|
|
static void lb_library(const GfxShader *sh, int vertex, int keep) {
|
|
int kind = vertex ? LIB_PR : LIB_FS, stat = vertex ? 0 : 1;
|
|
VkPipeState s;
|
|
vk_pipe_state_defaults(&s);
|
|
GKey k;
|
|
memset(&k, 0, sizeof k);
|
|
k.clamp = LB.clamp, k.poly_mode = LB.poly, k.minus_w = LB.minus_w, k.samples = LB.samples;
|
|
PrKey pk;
|
|
FsKey fk;
|
|
memset(&pk, 0, sizeof pk), memset(&fk, 0, sizeof fk);
|
|
const void *key;
|
|
uint32_t size;
|
|
if (vertex) {
|
|
GfxShaderIo fin[32];
|
|
s.vs = sh;
|
|
pk.kind = LIB_PR, pk.vs = sh, pk.clamp = k.clamp, pk.poly_mode = k.poly_mode, pk.minus_w = k.minus_w;
|
|
pk.missing = vs_union_peek(sh, fin);
|
|
for (int i = 0; i < 32; i++)
|
|
if (pk.missing >> i & 1) pk.fin[i] = fin[i];
|
|
key = &pk, size = sizeof pk;
|
|
} else {
|
|
s.fs = sh;
|
|
fk.kind = LIB_FS, fk.fs = sh, fk.samples = k.samples;
|
|
key = &fk, size = sizeof fk;
|
|
}
|
|
uint64_t h = key_hash(key, size);
|
|
Bindings B = {0}, F = {0};
|
|
VkPipelineLayout lay = VK_NULL_HANDLE;
|
|
lb_lock();
|
|
int have = keep && lib_find(h, key, size) != NULL;
|
|
if (!have && add_stage(vertex ? &B : &F, sh, 0)) {
|
|
Layout *l = layout_pair(&B, &F);
|
|
if (l) lay = half_layout(l, vertex ? 0 : 1);
|
|
}
|
|
lb_unlock();
|
|
if (have || !lay) {
|
|
lb_count(stat, have ? 1 : 2, 0);
|
|
return;
|
|
}
|
|
double t0 = now_ms();
|
|
VkPipeline pipe = lib_create(kind, &s, &k, lay, key);
|
|
double ms = now_ms() - t0;
|
|
if (!keep) {
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
lb_count(stat, pipe ? 3 : 2, ms);
|
|
return;
|
|
}
|
|
lb_lock();
|
|
have = lib_find(h, key, size) != NULL;
|
|
VkPipeline got = lib_put(h, key, size, pipe);
|
|
if (!have && pipe) {
|
|
Lib *b = lib_find(h, key, size);
|
|
if (b) b->pre = 1;
|
|
}
|
|
lb_unlock();
|
|
lb_count(stat, !pipe ? 2 : have || got != pipe ? 1 : 0, ms);
|
|
}
|
|
|
|
// A vertex or fragment variant with the graphics pipeline library off: no stage library to build ahead.
|
|
static void lb_skip_lib(void) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.nolib++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
}
|
|
|
|
static void lb_report(const char *how) {
|
|
float mem_used, mem_total, mem_proc;
|
|
plat_mem_info(&mem_used, &mem_total, &mem_proc);
|
|
plat_mutex_lock_shared(&LB.stat);
|
|
rt_log("vk_pipeline: stage library prebuild %s: %llu of %zu variants in %.0f ms; kept: vertex %llu built (avg %.1f ms), "
|
|
"%llu there; fragment %llu built (avg %.1f ms), %llu there; compute %llu built (avg %.1f ms), %llu there; driver "
|
|
"cache only: vertex %llu (avg %.1f ms), fragment %llu (avg %.1f ms), compute %llu (avg %.1f ms); %llu failed, %llu not in the shader cache, "
|
|
"%llu vertex/fragment skipped (graphics pipeline library off), %llu skipped (the driver crashed on them before, "
|
|
"shader-prepare-skip.txt), %llu large compute shaders left to their first dispatch (NX2NAVI_LIB_PREBUILD_BIG), %zu "
|
|
"queued for the background builder (NX2NAVI_LIB_PREBUILD_BIG_BG), %llu large compute shaders not built because "
|
|
"no run has dispatched them; process working set %.0f MB", how, (unsigned long long)LB.done, LB.n,
|
|
now_ms() - LB.t0, (unsigned long long)LB.built[0], LB.built[0] ? LB.ms[0] / LB.built[0] : 0.0,
|
|
(unsigned long long)LB.have[0], (unsigned long long)LB.built[1], LB.built[1] ? LB.ms[1] / LB.built[1] : 0.0,
|
|
(unsigned long long)LB.have[1], (unsigned long long)LB.built[2], LB.built[2] ? LB.ms[2] / LB.built[2] : 0.0,
|
|
(unsigned long long)LB.have[2], (unsigned long long)LB.warmed[0], LB.warmed[0] ? LB.warm_ms[0] / LB.warmed[0] : 0.0,
|
|
(unsigned long long)LB.warmed[1], LB.warmed[1] ? LB.warm_ms[1] / LB.warmed[1] : 0.0,
|
|
(unsigned long long)LB.warmed[2], LB.warmed[2] ? LB.warm_ms[2] / LB.warmed[2] : 0.0,
|
|
(unsigned long long)(LB.fail[0] + LB.fail[1] + LB.fail[2]),
|
|
(unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (unsigned long long)LB.skipped,
|
|
(unsigned long long)LB.deferred, BG.n, (unsigned long long)LB.big_unused, (double)mem_proc);
|
|
plat_mutex_unlock_shared(&LB.stat);
|
|
}
|
|
|
|
// Boot: compute pipelines (the first LB.n_keep entered for the process, the others only built into the pipeline
|
|
// cache) and kept libraries.
|
|
static void lb_worker(void *arg) {
|
|
lb_me = (int)(intptr_t)arg; // its handle is LB.th[lb_me]
|
|
plat_thread_priority(PLAT_PRIO_IDLE);
|
|
for (;;) {
|
|
size_t i = lb_next(lb_me);
|
|
if (i >= LB.n) break;
|
|
if (skip_listed(LB.e[i].iid)) {
|
|
skip_count();
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.done++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
continue;
|
|
}
|
|
// The memory gate (gate_wait): without room for another build the worker waits, and after GATE_WAIT_MS it and
|
|
// the other workers end. What they leave is built at its first dispatch or draw, as what a run never recorded.
|
|
double held;
|
|
if (LB.short_mem || !gate_wait("stage library prebuild", &LB.stop, &held)) {
|
|
if (!LB.stop && !plat_xchg32(&LB.short_mem, 1)) {
|
|
char g[256];
|
|
rt_log("vk_pipeline: stage library prebuild: ends here, at %zu of %zu variants: no commit headroom for "
|
|
"another build (%s). The others are built when the game first uses them", i, LB.n,
|
|
gate_text(g, sizeof g));
|
|
}
|
|
break;
|
|
}
|
|
// A compute shader whose pipeline does not stay is read as a copy of this thread's own and released after its
|
|
// build: kept in the shader cache's memory, the 3,900 of an installed copy's list were 240 MB of the process's
|
|
// commit for shaders most runs never dispatch. A dispatch that does come reads its shader then (0.05 to
|
|
// 0.15 ms from the files this pass has just read). The shaders of kept pipelines and libraries stay loaded:
|
|
// their pipeline and library entries point at them.
|
|
int keep = i < LB.n_keep, owned = 0;
|
|
plat_thread_priority(PLAT_PRIO_NORMAL); // gfx_shader's cache lock, disk reads
|
|
const GfxShader *sh = keep || LB.e[i].stage != GFX_STAGE_CS ? gfx_shader_load_input(LB.e[i].iid, 0)
|
|
: gfx_shader_load_private(LB.e[i].iid, &owned);
|
|
plat_thread_priority(PLAT_PRIO_IDLE);
|
|
if (!sh) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.noshader++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
} else if (gfx_shader_stage(sh) == GFX_STAGE_CS) {
|
|
lb_compute(sh, LB.e[i].iid, keep);
|
|
} else if (!owned && (gfx_shader_stage(sh) == GFX_STAGE_VS || gfx_shader_stage(sh) == GFX_STAGE_FS)) {
|
|
if (P.gpl) lb_library(sh, gfx_shader_stage(sh) == GFX_STAGE_VS, 1); // a kept library points at sh: never a copy
|
|
else lb_skip_lib();
|
|
}
|
|
if (owned) gfx_shader_release_private(sh);
|
|
gate_leave(held);
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.done++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
}
|
|
if (!plat_dec32(&LB.running)) {
|
|
lb_report(LB.stop ? "stopped" : LB.short_mem ? "ended for want of commit headroom" : "done");
|
|
fb_log("after the stage library prebuild");
|
|
vk_mem_stage_log("the compute pipeline prebuild");
|
|
bg_start();
|
|
}
|
|
return;
|
|
}
|
|
|
|
// The library state from the key file's records (prewarm); defaults without one.
|
|
static void lb_state(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) {
|
|
// state index: clamp | poly (0 fill, 1 line, 2 point -> VkPolygonMode) << 1 | minus_w << 3 (pr); samples log2 (fs)
|
|
uint32_t best = 0, bf = 0;
|
|
for (uint32_t i = 1; i < 16; i++) best = pr_hist[i] > pr_hist[best] ? i : best;
|
|
for (uint32_t i = 1; i < 4; i++) bf = fs_hist[i] > fs_hist[bf] ? i : bf;
|
|
if (!pr_hist[best]) best = 8; // no key file: NX2navi clips -w..w (all 2706 records of a recorded key file)
|
|
LB.clamp = (uint8_t)(best & 1), LB.poly = (uint8_t)(best >> 1 & 3), LB.minus_w = (uint8_t)(best >> 3 & 1);
|
|
LB.samples = (uint8_t)(1u << bf);
|
|
}
|
|
|
|
// ---- the driver's cache of stage libraries, warmed again during play
|
|
//
|
|
// The preparation (lb_warm_prepared) builds every prepared variant's stage library once, so the driver's disk cache
|
|
// holds it and the library a draw needs later takes a few milliseconds instead of a compile: 3 ms against 100 ms for
|
|
// a vertex and 340 ms for a fragment library on an RX 7900 XT (driver 26.10.1). The driver's cache does not outlive
|
|
// everything, though: a driver update empties it, the AMD driver keeps a limited amount and drops old entries, and a
|
|
// change to what a library is built with (the dynamic states it lists, as when dynamic vertex input was turned off for
|
|
// this driver) gives every library another entry. After any of them each pipeline a draw first needs compiles cold:
|
|
// the draw is skipped for the frames that takes, or waits for it. Measured on that card with the cache cold, a walk
|
|
// through a village not seen before skipped 11,180 draws while their pipelines built and 94 draws waited 5.5 s in
|
|
// all, one of them 1.9 s.
|
|
//
|
|
// So every start checks: a sample of the prepared fragment libraries is built (lw_probe), and when their median time
|
|
// says the cache is cold, the whole prepared list is built again into the driver's cache on a few idle-priority
|
|
// threads while the game runs (lw_worker), as the preparation does but without stopping play. shader-warm.txt beside
|
|
// the caches says where that stands: "warming" while it runs, so a start after a run that ended meanwhile goes on,
|
|
// and "done" with the driver and a count of the times the cache was cold again right after a finished warm. At two in
|
|
// a row the driver does not keep stage libraries at all and warming stops being tried (NX2NAVI_LIB_WARM=0 turns it off;
|
|
// =force warms at every start).
|
|
#define LW_SAMPLE 12
|
|
#define LW_COLD_MS 15.0
|
|
#define LW_MAX_THREADS 6
|
|
static struct {
|
|
GfxWarmEnt *e;
|
|
size_t n;
|
|
volatile int64_t next, done;
|
|
volatile int32_t running, stop, started;
|
|
volatile int32_t short_mem; // the memory gate refused a build for GATE_WAIT_MS: the warm ended for this run
|
|
PlatThread *th[LW_MAX_THREADS + 1];
|
|
int threads, strikes;
|
|
uint32_t vendor; // the GPU's vendor id, for what a cold cache's log line adds (vk_warm_hint.h)
|
|
double t0;
|
|
char stamp[PLAT_PATH_MAX * 2 + 32], id[64];
|
|
} LW;
|
|
|
|
static void lw_stamp(const char *state, int strikes) {
|
|
char line[128];
|
|
int n = snprintf(line, sizeof line, "%s %s %d\n", state, LW.id, strikes);
|
|
if (!plat_file_write_atomic(LW.stamp, line, (size_t)n)) rt_log("vk_pipeline: cannot write %s", LW.stamp);
|
|
}
|
|
|
|
// One prepared vertex or fragment variant's library into the driver's cache; its time in ms, negative when the
|
|
// variant is on the skip list, not in the shader cache or of another stage.
|
|
static double lw_build(const GfxWarmEnt *e) {
|
|
if (e->stage != GFX_STAGE_VS && e->stage != GFX_STAGE_FS) return -1;
|
|
if (skip_listed(e->iid)) return -1;
|
|
int owned = 0;
|
|
const GfxShader *sh = gfx_shader_load_private(e->iid, &owned);
|
|
if (!sh) return -1;
|
|
double t0 = now_ms();
|
|
int st = gfx_shader_stage(sh);
|
|
if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) lb_library(sh, st == GFX_STAGE_VS, 0);
|
|
double ms = now_ms() - t0;
|
|
if (owned) gfx_shader_release_private(sh);
|
|
return ms;
|
|
}
|
|
|
|
static void lw_worker(void *arg) {
|
|
(void)arg;
|
|
plat_thread_priority(PLAT_PRIO_IDLE);
|
|
no_cache = 1; // into the driver's own cache only (see no_cache)
|
|
for (;;) {
|
|
size_t i = (size_t)plat_inc64(&LW.next) - 1;
|
|
if (i >= LW.n || LW.stop) break;
|
|
// The memory gate (gate_wait): without room the warm waits, and after GATE_WAIT_MS it ends for this run.
|
|
// shader-warm.txt keeps saying "warming", so the next start goes on with it.
|
|
double held;
|
|
if (LW.short_mem || !gate_wait("library warm", &LW.stop, &held)) {
|
|
if (!LW.stop && !plat_xchg32(&LW.short_mem, 1)) {
|
|
char g[256];
|
|
rt_log("vk_pipeline: library warm: ends for this run at %lld of %zu prepared variants: no commit headroom "
|
|
"for another build (%s). The next start goes on with it", (long long)LW.done, LW.n,
|
|
gate_text(g, sizeof g));
|
|
}
|
|
break;
|
|
}
|
|
lw_build(&LW.e[i]);
|
|
gate_leave(held);
|
|
int64_t d = plat_inc64(&LW.done);
|
|
if (d % 8000 == 0)
|
|
rt_log("vk_pipeline: library warm: %lld of %zu prepared variants in %.0f s", (long long)d, LW.n,
|
|
(now_ms() - LW.t0) / 1000);
|
|
}
|
|
if (!plat_dec32(&LW.running) && !LW.stop && !LW.short_mem) {
|
|
lw_stamp("done", LW.strikes);
|
|
rt_log("vk_pipeline: library warm: done, %zu prepared variants in %.0f s on %d idle threads; new pipelines "
|
|
"link from the driver's cache from here on", LW.n, (now_ms() - LW.t0) / 1000, LW.threads);
|
|
}
|
|
}
|
|
|
|
// The median time of LW_SAMPLE prepared fragment libraries spread over the list; negative without that many.
|
|
static double lw_probe(void) {
|
|
double ms[LW_SAMPLE];
|
|
int got = 0;
|
|
size_t step = LW.n / LW_SAMPLE ? LW.n / LW_SAMPLE : 1, at = (size_t)(plat_now_ns() / 1000) % step;
|
|
for (size_t i = at; i < LW.n && got < LW_SAMPLE && !LW.stop; i += step)
|
|
for (size_t j = i; j < i + step && j < LW.n; j++) { // the next fragment variant from here
|
|
if (LW.e[j].stage != GFX_STAGE_FS) continue;
|
|
double t = lw_build(&LW.e[j]);
|
|
if (t >= 0) ms[got++] = t;
|
|
break;
|
|
}
|
|
if (got < LW_SAMPLE / 2) return -1;
|
|
for (int i = 1; i < got; i++)
|
|
for (int j = i; j > 0 && ms[j] < ms[j - 1]; j--) {
|
|
double t = ms[j];
|
|
ms[j] = ms[j - 1], ms[j - 1] = t;
|
|
}
|
|
return ms[got / 2];
|
|
}
|
|
|
|
static void lw_main(void *arg) {
|
|
(void)arg;
|
|
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
char was[16] = "", id[64] = "";
|
|
int strikes = 0;
|
|
uint64_t size = 0;
|
|
char *f = (char *)plat_file_read(LW.stamp, &size);
|
|
if (f && size < 200) {
|
|
char line[200];
|
|
memcpy(line, f, size), line[size] = 0;
|
|
if (sscanf(line, "%15s %63s %d", was, id, &strikes) < 2) was[0] = 0;
|
|
}
|
|
free(f);
|
|
int same = !strcmp(id, LW.id), resume = same && !strcmp(was, "warming");
|
|
const char *mode = cfg_value("NX2NAVI_LIB_WARM");
|
|
double median = resume ? 0 : lw_probe();
|
|
if (LW.stop) return;
|
|
int cold = median > LW_COLD_MS || !strcmp(mode, "force");
|
|
if (!resume && !cold) {
|
|
if (same && strikes) lw_stamp("done", 0); // the driver kept them this time
|
|
rt_log("vk_pipeline: library warm: the driver's cache has the prepared libraries (a sample's median %.1f ms)",
|
|
median);
|
|
return;
|
|
}
|
|
if (!resume && same && strikes >= 2 && strcmp(mode, "force")) {
|
|
rt_log("vk_pipeline: library warm: the prepared libraries are cold again (a sample's median %.0f ms) after two "
|
|
"finished warms on this driver: it does not keep them, so pipelines compile when a draw first needs "
|
|
"them (NX2NAVI_LIB_WARM=force warms anyway).%s", median,
|
|
lw_cold_hint(LW.vendor, LW_COLD_GAVE_UP, LW_ON_WINDOWS));
|
|
return;
|
|
}
|
|
LW.strikes = resume ? strikes : same && !strcmp(was, "done") ? strikes + 1 : 0;
|
|
lw_stamp("warming", LW.strikes);
|
|
int cpus = (int)plat_cpu_usable(), t = cpus / 4;
|
|
t = t < 2 ? 2 : t > LW_MAX_THREADS ? LW_MAX_THREADS : t;
|
|
if (resume)
|
|
rt_log("vk_pipeline: library warm: going on with the warm an earlier run began: %zu prepared variants on %d "
|
|
"idle threads", LW.n, t);
|
|
else // cold right after a finished warm on this driver says more than a first cold (vk_warm_hint.h)
|
|
rt_log("vk_pipeline: library warm: the driver's cache does not have the prepared libraries (a sample's median "
|
|
"%.0f ms; a driver update or a change of what they are built with): building %zu prepared variants "
|
|
"into it on %d idle threads while the game runs. Until it is done, a pipeline a draw first needs may "
|
|
"compile, which skips that draw for some frames or makes it wait.%s", median, LW.n, t,
|
|
lw_cold_hint(LW.vendor,
|
|
median <= LW_COLD_MS ? LW_WARM_FORCED // warm, but lib_warm = force
|
|
: same && !strcmp(was, "done") ? LW_COLD_AFTER_WARM : LW_COLD_FIRST,
|
|
LW_ON_WINDOWS));
|
|
LW.t0 = now_ms();
|
|
LW.running = t;
|
|
int started = 0;
|
|
for (int i = 0; i < t; i++)
|
|
if ((LW.th[1 + started] = plat_thread_new(lw_worker, NULL, "lib warm"))) started++;
|
|
LW.threads = started;
|
|
if (plat_add32(&LW.running, started - t) == 0 && !started) rt_log("vk_pipeline: library warm: no thread started");
|
|
}
|
|
|
|
// Boot, once the library state is known (lb_state): the prepared list is read and the check starts on its own thread.
|
|
static void lw_start(void) {
|
|
const char *mode = cfg_value("NX2NAVI_LIB_WARM");
|
|
if (!P.gpl || !strcmp(mode, "0") || plat_xchg32(&LW.started, 1)) return;
|
|
char dir[PLAT_PATH_MAX * 2], path[PLAT_PATH_MAX * 2 + 32];
|
|
if (!cache_dir(dir, sizeof dir)) return;
|
|
snprintf(path, sizeof path, "%s/shader-prepared.bin", dir);
|
|
snprintf(LW.stamp, sizeof LW.stamp, "%s/shader-warm.txt", dir);
|
|
size_t n = 0;
|
|
uint8_t *f = gfx_shader_prepared_read(path, &n, NULL, NULL); // the records alone, whichever version the file is
|
|
LW.e = n ? calloc(n, sizeof *LW.e) : NULL;
|
|
for (size_t i = 0; LW.e && i < n; i++)
|
|
memcpy(LW.e[i].iid, f + PRP_REC * i + 4, 32), LW.e[i].stage = f[PRP_REC * i], LW.e[i].prio = f[PRP_REC * i + 1];
|
|
free(f);
|
|
if (!LW.e) return; // nothing prepared (a build without the preparation step)
|
|
LW.n = n;
|
|
VkPhysicalDeviceProperties pp;
|
|
vkGetPhysicalDeviceProperties(vk_ctx()->phys, &pp);
|
|
snprintf(LW.id, sizeof LW.id, "%04x:%04x:%08x", pp.vendorID, pp.deviceID, pp.driverVersion);
|
|
LW.vendor = pp.vendorID;
|
|
LW.th[0] = plat_thread_new(lw_main, NULL, "lib warm check");
|
|
}
|
|
|
|
static void lw_stop(void) {
|
|
plat_xchg32(&LW.stop, 1);
|
|
for (int i = 0; i <= LW_MAX_THREADS; i++)
|
|
if (LW.th[i]) plat_thread_join(LW.th[i]), LW.th[i] = NULL;
|
|
}
|
|
|
|
// Boot: compute shaders of the warm list (recorded and prepared), those whose pipeline stays first, then the first
|
|
// KEEP recorded graphics variants.
|
|
static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]) {
|
|
const char *e = cfg_value("NX2NAVI_LIB_PREBUILD"), *ke = cfg_get("NX2NAVI_LIB_PREBUILD_KEEP");
|
|
const char *te = cfg_get("NX2NAVI_LIB_PREBUILD_THREADS"); // no fixed default: 2 when unset (1 under 4 CPUs)
|
|
const char *be = cfg_get("NX2NAVI_LIB_PREBUILD_BIG"); // SPIR-V words; LB_BIG_WORDS when unset, 0 no limit
|
|
if (!A.on || !cfg_bool("NX2NAVI_LIB_PREBUILD")) return;
|
|
lb_state(pr_hist, fs_hist); // the libraries' state, also for lw_start
|
|
lw_start();
|
|
LB.big_words = be && *be ? strtoull(be, NULL, 10) : LB_BIG_WORDS;
|
|
char dir[PLAT_PATH_MAX * 2];
|
|
if (cache_dir(dir, sizeof dir)) skip_init(dir);
|
|
GfxWarmEnt *all = NULL;
|
|
size_t total = gfx_shader_warm_list(&all), n = 0, n_cs = 0;
|
|
LB.keep_n = ke && *ke ? atoi(ke) : 0;
|
|
LB.e = malloc((total + 1) * sizeof *LB.e);
|
|
// the compute shaders whose pipeline stays come first: they are the ones the game is about to dispatch
|
|
uint8_t *kf = total ? malloc(total) : NULL;
|
|
for (size_t i = 0; LB.e && kf && i < total; i++)
|
|
if (all[i].stage == GFX_STAGE_CS && (kf[i] = (uint8_t)cu_keep(all[i].iid))) LB.e[n++] = all[i];
|
|
LB.n_keep = n;
|
|
for (size_t i = 0; LB.e && i < total; i++)
|
|
if (all[i].stage == GFX_STAGE_CS && !(kf && kf[i])) LB.e[n++] = all[i];
|
|
free(kf);
|
|
n_cs = n;
|
|
for (size_t i = 0; LB.e && i < total && (int32_t)(n - n_cs) < LB.keep_n; i++)
|
|
if (all[i].stage != GFX_STAGE_CS && all[i].prio == 0) LB.e[n++] = all[i];
|
|
free(all);
|
|
if (atoi(e) > 1 && (size_t)atoi(e) < n) n = (size_t)atoi(e); // a count above 1 caps the list; 1 is plain on
|
|
if (LB.n_keep > n) LB.n_keep = n;
|
|
if (!n) {
|
|
free(LB.e), LB.e = NULL;
|
|
return;
|
|
}
|
|
LB.n = n;
|
|
LB.idle = 1;
|
|
int cpus = (int)plat_cpu_usable();
|
|
// Two threads by default, not half the CPUs. On a 32-thread CPU with NVIDIA's driver, 16 builders kept the driver's
|
|
// lock busy: 77% of the process's running time was a spin loop in nvoglv64.dll (about 8 cores), the prebuild had not
|
|
// finished after 34,000 frames, and lib_prebuild = 0 took it to 5% and the game to its frame limit (RTX 4080 SUPER,
|
|
// driver 617.14, 2026-11-08). With the driver's disk cache warm (--prepare-shaders, --warm-shaders) a build takes
|
|
// well under a millisecond, so two threads still finish the list quickly.
|
|
// The two take the shaders dispatched most recently (e[0..n_keep)); one goes on with the rest (lb_next), which is
|
|
// only built into the caches. Where the rest still runs during play, two builders there cost frames: one against
|
|
// two for the whole list, one 20 s route each on an RTX 5090, Castle Town 76.1 against 73.7 fps with 1 frame over
|
|
// 25 ms against 6, Hyrule Field 85.7 against 80.7 fps. NX2NAVI_LIB_PREBUILD_THREADS keeps its count for the whole
|
|
// list.
|
|
int t = te && *te ? atoi(te) : (cpus < 4 ? 1 : 2); // idle priority; a cold driver takes ~2 s per large compute shader
|
|
LB.tail_one = !(te && *te);
|
|
t = t < 1 ? 1 : t > LB_MAX_THREADS ? LB_MAX_THREADS : t;
|
|
if ((size_t)t > n) t = (int)n;
|
|
LB.t0 = now_ms();
|
|
LB.running = t;
|
|
int started = 0;
|
|
for (int i = 0; i < t; i++)
|
|
if ((LB.th[started] = plat_thread_new(lb_worker, (void *)(intptr_t)started, "lib prebuild"))) started++;
|
|
plat_add32(&LB.running, started - t);
|
|
LB.threads = started;
|
|
char big[128];
|
|
if (!LB.big_words) snprintf(big, sizeof big, "compute shaders of every size");
|
|
else snprintf(big, sizeof big, "compute shaders over %llu SPIR-V words %s", (unsigned long long)LB.big_words,
|
|
P.cache_control ? "only from the pipeline caches" : "left to their first dispatch");
|
|
rt_log("vk_pipeline: stage library prebuild: %zu compute pipelines, of which %zu stay (the shaders dispatched most "
|
|
"recently, NX2NAVI_COMPUTE_KEEP) and the others are built into the pipeline cache and destroyed, to be created "
|
|
"at their first dispatch; %zu recorded variants' libraries (NX2NAVI_LIB_PREBUILD_KEEP %d); on %d idle-priority "
|
|
"threads; library state clamp %u, polygon mode %u, depth -w..w %u, samples %u; %s (NX2NAVI_LIB_PREBUILD_BIG)",
|
|
n_cs < n ? n_cs : n, LB.n_keep, n > n_cs ? n - n_cs : 0, LB.keep_n, started, LB.clamp, LB.poly, LB.minus_w,
|
|
LB.samples, big);
|
|
if (LB.tail_one && started > 1)
|
|
rt_log("vk_pipeline: stage library prebuild: after the %zu recent compute pipelines, one worker continues the "
|
|
"remaining list; NX2NAVI_LIB_PREBUILD_THREADS overrides this background limit", LB.n_keep);
|
|
}
|
|
|
|
static void lb_stop(void) {
|
|
lw_stop();
|
|
plat_xchg32(&LB.stop, 1);
|
|
for (int i = 0; i < LB.threads; i++)
|
|
if (LB.th[i]) plat_thread_join(LB.th[i]), LB.th[i] = NULL;
|
|
LB.threads = 0;
|
|
if (BG.th) plat_thread_join(BG.th), BG.th = NULL; // after its current build, if one runs
|
|
}
|
|
|
|
// Preparation: one prepared variant at a time into the driver's disk cache (private shader copy, nothing kept). arg is
|
|
// the worker's journal slot.
|
|
static void lb_warm_worker(void *arg) {
|
|
int slot = (int)(intptr_t)arg;
|
|
for (;;) {
|
|
size_t i = (size_t)plat_inc64(&LB.next) - 1;
|
|
if (i >= LB.n) return;
|
|
int owned = 0, skip = skip_listed(LB.e[i].iid);
|
|
const GfxShader *sh = skip ? NULL : gfx_shader_load_private(LB.e[i].iid, &owned);
|
|
int st = sh ? gfx_shader_stage(sh) : -1;
|
|
if (sh) journal_mark(slot, &LB.e[i]); // from here to its clearing below, a crash names this variant
|
|
if (skip) {
|
|
skip_count();
|
|
} else if (!sh) {
|
|
plat_mutex_lock(&LB.stat);
|
|
LB.noshader++;
|
|
plat_mutex_unlock(&LB.stat);
|
|
} else if (st == GFX_STAGE_CS) {
|
|
Bindings B = {0};
|
|
const char *why = NULL;
|
|
lb_lock();
|
|
Layout *l = compute_layout(sh, &B, &why);
|
|
lb_unlock();
|
|
double t0 = now_ms();
|
|
VkPipeline pipe = l ? compute_create(sh, l, 0) : VK_NULL_HANDLE;
|
|
if (pipe) vkDestroyPipeline(P.device, pipe, NULL);
|
|
lb_count(2, pipe ? 3 : 2, now_ms() - t0);
|
|
} else if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) {
|
|
if (P.gpl) lb_library(sh, st == GFX_STAGE_VS, 0);
|
|
else lb_skip_lib();
|
|
}
|
|
if (sh) journal_mark(slot, NULL);
|
|
if (owned) gfx_shader_release_private(sh);
|
|
plat_mutex_lock(&LB.stat);
|
|
uint64_t d = ++LB.done;
|
|
plat_mutex_unlock(&LB.stat);
|
|
if (d % 4000 == 0) lb_report("preparation progress");
|
|
}
|
|
}
|
|
|
|
// Preparation: every variant of <dir>/shader-prepared.bin on every core at normal priority.
|
|
static void lb_warm_prepared(const char *dir) {
|
|
char path[PLAT_PATH_MAX * 2 + 32];
|
|
snprintf(path, sizeof path, "%s/shader-prepared.bin", dir);
|
|
size_t n = 0;
|
|
uint8_t *f = gfx_shader_prepared_read(path, &n, NULL, NULL); // the records alone, whichever version the file is
|
|
LB.e = n ? calloc(n, sizeof *LB.e) : NULL;
|
|
for (size_t i = 0; LB.e && i < n; i++)
|
|
memcpy(LB.e[i].iid, f + PRP_REC * i + 4, 32), LB.e[i].stage = f[PRP_REC * i], LB.e[i].prio = f[PRP_REC * i + 1];
|
|
free(f);
|
|
if (!LB.e) return;
|
|
skip_init(dir);
|
|
char jp[PLAT_PATH_MAX * 2 + 48];
|
|
snprintf(jp, sizeof jp, "%s/shader-prepare-journal.bin", dir);
|
|
SK.journal = plat_fs_create(jp, NULL); // skip_init removed a leftover one
|
|
if (!SK.journal) rt_log("vk_pipeline: cannot create %s: a driver crash in this run will not be recorded", jp);
|
|
LB.n = n, LB.idle = 0, LB.t0 = now_ms();
|
|
static const uint32_t no_pr[16], no_fs[4];
|
|
lb_state(no_pr, no_fs);
|
|
int cpus = (int)plat_cpu_usable(), k = 0;
|
|
PlatThread *t[JR_SLOTS];
|
|
for (int i = 0; i < cpus && i < JR_SLOTS; i++)
|
|
if ((t[k] = plat_thread_new(lb_warm_worker, (void *)(intptr_t)k, "lib warm"))) k++;
|
|
if (!k) lb_warm_worker((void *)(intptr_t)0);
|
|
// every 2 s a count on stdout for tools/make_game.py (step_shaders), as gfx_shader.c prepare_files prints for the
|
|
// translation: a driver building for a long time still shows the step moving
|
|
for (int i = 0; i < k; i++)
|
|
while (!plat_thread_join_ns(t[i], 2000000000u)) {
|
|
plat_mutex_lock(&LB.stat);
|
|
uint64_t d = LB.done;
|
|
plat_mutex_unlock(&LB.stat);
|
|
printf("prepare-shaders: progress build %llu %zu\n", (unsigned long long)d, n);
|
|
fflush(stdout);
|
|
}
|
|
printf("prepare-shaders: progress build %zu %zu\n", n, n);
|
|
fflush(stdout);
|
|
if (SK.journal) plat_file_close(SK.journal), SK.journal = NULL, plat_file_delete(jp); // every build returned
|
|
lb_report("preparation done");
|
|
// with gpl off only compute variants are built: say so, rather than count the skipped ones as built
|
|
unsigned long long skipped = LB.skipped;
|
|
size_t tried = n - (size_t)skipped;
|
|
if (LB.nolib)
|
|
printf("prepare-shaders: %zu variants on %d threads in %.0f ms, %llu vertex/fragment variants skipped (graphics "
|
|
"pipeline library off, they build at their first draw)\n", tried, k, now_ms() - LB.t0,
|
|
(unsigned long long)LB.nolib);
|
|
else
|
|
printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", tried, k,
|
|
now_ms() - LB.t0);
|
|
if (skipped) // installer/steps.py Prepare reads this line
|
|
printf("prepare-shaders: %llu variants skipped: the graphics driver crashed while building them in an earlier "
|
|
"run (shader-prepare-skip.txt); they build at their first draw\n", skipped);
|
|
free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0, LB.skipped = 0;
|
|
}
|
|
|
|
// ---- the shared pipeline list (tools/pipeline_list.py; seed-pipeline-keys.bin next to the game)
|
|
// tools/prepare_shaders.py writes the list's key records there, in the layout of pipeline-keys.bin, after putting the
|
|
// shader variants they name into the seed. The preparation builds them once, after the seed is translated: each
|
|
// pipeline goes through the pipeline cache and so into pipeline-cache.bin and the driver's own disk cache, and is
|
|
// dropped with the process. Nothing is added to this folder's pipeline-keys.bin: the boot keeps building only what
|
|
// this install drew, and a listed pipeline costs no start-up time or video memory until a draw meets it. That draw
|
|
// then finds its pipeline in the caches: a few milliseconds for a mesh pipeline, which has no stage libraries and
|
|
// otherwise leaves its draws out of the picture for its whole compile (0.2 to 1.1 s each on a first visit of a forest,
|
|
// 18 of them).
|
|
// NX2NAVI_PREPARE_PIPELINES: mesh (default) builds the list's mesh pipelines only; all, every record; 0, none. A
|
|
// vertex pipeline is linked from its prepared stage libraries in 2 to 6 ms when a draw misses it, and its module
|
|
// depends on the vertex output union of the pairs this install has recorded (vs_outs), so one built here is not
|
|
// certain to be the one a draw asks for. Without the graphics pipeline library (vk_caps_gpl: AMD's Windows driver for
|
|
// RDNA 2 and older) neither holds: a vertex pipeline a draw misses is compiled whole while that draw is left out, and
|
|
// its vertex outputs come from its own pair (no independent layouts), so a listed one is the one the draw asks for.
|
|
// There mesh builds every record too.
|
|
// A driver whose compiler crashes on one of these would crash at every preparation: seed-pipelines-running.txt marks
|
|
// the step, and a run that finds the marker of a run that died skips the step for good on this folder (the pipelines
|
|
// are then built at their first draw, as without a list) until the marker is deleted.
|
|
static int cache_save(uint32_t wait_ms, int exiting);
|
|
|
|
static int seed_pipelines(const char *dir) {
|
|
const char *mode = cfg_value("NX2NAVI_PREPARE_PIPELINES");
|
|
int all = !strcmp(mode, "all") || (!strcmp(mode, "mesh") && !P.gpl);
|
|
char path[PLAT_PATH_MAX * 2 + 48], mark[PLAT_PATH_MAX * 2 + 48];
|
|
snprintf(path, sizeof path, "%s/seed-pipeline-keys.bin", dir);
|
|
snprintf(mark, sizeof mark, "%s/seed-pipelines-running.txt", dir);
|
|
FILE *f = fopen(path, "rb");
|
|
if (!f) return 0; // no list: nothing to say
|
|
if (!strcmp(mode, "0")) {
|
|
fclose(f);
|
|
printf("prepare-pipelines: off (NX2NAVI_PREPARE_PIPELINES=0)\n");
|
|
return 0;
|
|
}
|
|
if (plat_file_exists(mark)) {
|
|
fclose(f);
|
|
printf("prepare-pipelines: skipped: an earlier preparation ended while it built the listed pipelines (%s); they "
|
|
"are built at their first draw\n", mark);
|
|
return 0;
|
|
}
|
|
KeysHeader h;
|
|
fseek(f, 0, SEEK_END);
|
|
long len = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
size_t n = 0;
|
|
KeyRec *r = NULL;
|
|
if (len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8) &&
|
|
h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec)) {
|
|
n = ((size_t)len - sizeof h) / sizeof(KeyRec);
|
|
r = malloc(n * sizeof *r + 1);
|
|
if (r && fread(r, sizeof *r, n, f) != n) n = 0;
|
|
} else {
|
|
printf("prepare-pipelines: %s is a list for another version of the game's runtime (its record layout differs); "
|
|
"no pipeline is built from it\n", path);
|
|
}
|
|
fclose(f);
|
|
uint8_t (*ids)[32] = n ? malloc(3 * n * 32) : NULL;
|
|
const GfxShader **sh = n ? calloc(3 * n, sizeof *sh) : NULL;
|
|
if (!r || !ids || !sh) {
|
|
free(r), free(ids), free(sh);
|
|
return 0;
|
|
}
|
|
double t0 = now_ms();
|
|
size_t m = 0, u = 0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
memcpy(ids[m++], r[i].vs, 32);
|
|
if (r[i].has_fs) memcpy(ids[m++], r[i].fs, 32);
|
|
if (r[i].has_ts) memcpy(ids[m++], r[i].ts, 32);
|
|
}
|
|
qsort(ids, m, 32, id_cmp);
|
|
for (size_t i = 0; i < m; i++)
|
|
if (!u || memcmp(ids[u - 1], ids[i], 32)) memcpy(ids[u++], ids[i], 32);
|
|
int cpus = (int)plat_cpu_usable();
|
|
const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE");
|
|
int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2;
|
|
gfx_shader_prewarm((const uint8_t(*)[32])ids, u, sh, cpus, procs < 1 ? 1 : procs);
|
|
static const char note[] = "the listed pipelines are being built; a run that finds this file skips them\n";
|
|
plat_file_write_atomic(mark, note, sizeof note - 1);
|
|
uint32_t queued = 0, dropped = 0, known = 0, other = 0, mesh = 0, no_mesh = 0;
|
|
plat_mutex_lock(&P.lock);
|
|
uint32_t built0 = K.built, failed0 = K.failed;
|
|
K.loading = 1;
|
|
for (size_t i = 0; i < n; i++) {
|
|
const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp);
|
|
const uint8_t (*fi)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL;
|
|
const uint8_t (*t)[32] = r[i].has_ts ? bsearch(r[i].ts, ids, u, 32, id_cmp) : NULL;
|
|
const GfxShader *vs = v ? sh[v - ids] : NULL, *fs = fi ? sh[fi - ids] : NULL, *ts = t ? sh[t - ids] : NULL;
|
|
if (!vs || (r[i].has_fs && !fs) || (r[i].has_ts && !ts)) {
|
|
dropped++;
|
|
continue;
|
|
}
|
|
VkPipeState s = r[i].s;
|
|
s.vs = vs, s.fs = fs, s.ts = ts;
|
|
if (!is_mesh(&s) && !all) {
|
|
other++;
|
|
continue;
|
|
}
|
|
if (is_mesh(&s) && !vk_ctx()->has_mesh_shader) { // this GPU draws no mesh pipeline
|
|
no_mesh++;
|
|
continue;
|
|
}
|
|
if (!is_mesh(&s) && P.indep && fs && gfx_shader_stage(vs) == GFX_STAGE_VS) { // its vertex output union, as the boot
|
|
GfxShaderIo io[32];
|
|
vs_outs(vs, fs, io);
|
|
}
|
|
GKey k;
|
|
gkey_fill(&s, &k);
|
|
uint64_t hk = key_hash(&k, sizeof k);
|
|
int q = 0;
|
|
if (pipe_find(&k, sizeof k, hk)) known++;
|
|
else pipe_miss(&s, &k, hk, 2, &q), queued += (uint32_t)q, mesh += q && is_mesh(&s);
|
|
}
|
|
K.loading = 0;
|
|
K.until = 0; // every one of them: no boot waits for this
|
|
GATE.off = 1; // and no game runs beside it: the memory gate (gate_enter) refuses nothing
|
|
plat_mutex_unlock(&P.lock);
|
|
free(ids), free(sh), free(r);
|
|
PlatThread *th[64];
|
|
int nt = 0;
|
|
if (queued) warm_builder((void *)1); // the first alone (warm_builder)
|
|
for (int i = 0; i < cpus && i < 64 && queued; i++)
|
|
if ((th[nt] = plat_thread_new(warm_builder, NULL, "pipe list"))) nt++;
|
|
if (!nt && queued) warm_builder(NULL);
|
|
for (int i = 0; i < nt; i++)
|
|
while (!plat_thread_join_ns(th[i], 2000000000u)) {
|
|
printf("prepare-pipelines: progress %u %u\n",
|
|
__atomic_load_n(&K.built, __ATOMIC_RELAXED) + __atomic_load_n(&K.failed, __ATOMIC_RELAXED) - built0 - failed0,
|
|
queued);
|
|
fflush(stdout);
|
|
}
|
|
plat_mutex_lock(&P.lock);
|
|
while (K.left) plat_cond_wait(&done_cv, &P.lock); // those a pool worker took
|
|
uint32_t built = K.built - built0, failed = K.failed - failed0;
|
|
plat_mutex_unlock(&P.lock);
|
|
int saved = cache_save(5000, 0) > 0; // explicit preparation waits for a moment with no build running
|
|
plat_file_delete(mark);
|
|
char msg[640];
|
|
snprintf(msg, sizeof msg, "prepare-pipelines: %zu listed pipelines: %u built (%u mesh), %u failed, %u the same as one "
|
|
"built before, %u without a shader this game version or its shader cache has, %u vertex pipelines left to "
|
|
"their first draw (NX2NAVI_PREPARE_PIPELINES=%s), %u mesh pipelines this GPU cannot draw; on %d threads in "
|
|
"%.0f ms; pipeline cache %s", n, built, mesh, failed, known, dropped, other,
|
|
!strcmp(mode, "all") ? "all" : all ? "mesh, every record without the graphics pipeline library" : "mesh", no_mesh,
|
|
nt ? nt : 1, now_ms() - t0,
|
|
saved ? "saved" : "not saved (builds kept running, or its file could not be written)");
|
|
printf("%s\n", msg);
|
|
fflush(stdout);
|
|
rt_log("vk_pipeline: %s", msg);
|
|
fb_log("after the listed pipelines");
|
|
return 0;
|
|
}
|
|
|
|
// nx2navi.exe --prepare-pipelines: the list's pipelines alone, for a folder whose shaders are prepared already.
|
|
int vk_pipeline_prepare_pipelines(void) {
|
|
prep_mode = 1; // builds into pipeline-cache.bin alone (cache_load)
|
|
if (!ensure_init()) {
|
|
fprintf(stderr, "prepare-pipelines: no Vulkan device\n");
|
|
return 1;
|
|
}
|
|
char dir[PLAT_PATH_MAX * 2];
|
|
if (!cache_dir(dir, sizeof dir)) return 1;
|
|
char path[PLAT_PATH_MAX * 2 + 48];
|
|
snprintf(path, sizeof path, "%s/seed-pipeline-keys.bin", dir);
|
|
if (!plat_file_exists(path)) {
|
|
fprintf(stderr, "prepare-pipelines: no %s (tools/prepare_shaders.py writes it from the pipeline list)\n", path);
|
|
return 1;
|
|
}
|
|
return seed_pipelines(dir);
|
|
}
|
|
|
|
// nx2navi.exe --prepare-shaders (tools/prepare_shaders.py): the device sets the translator's host answers (they are in every
|
|
// cache key), the seed next to nx2navi.exe is translated on every core, then every prepared variant's stage library or
|
|
// compute pipeline is built once into the driver's disk cache (lb_warm_prepared). No game runs.
|
|
int vk_pipeline_prepare_shaders(void) {
|
|
prep_mode = 1; // builds into pipeline-cache.bin alone (cache_load)
|
|
if (!ensure_init()) {
|
|
fprintf(stderr, "prepare-shaders: no Vulkan device\n");
|
|
return 1;
|
|
}
|
|
char dir[PLAT_PATH_MAX * 2];
|
|
if (!cache_dir(dir, sizeof dir)) return 1;
|
|
int cpus = (int)plat_cpu_usable();
|
|
const char *pe = cfg_get("NX2NAVI_PREWARM_XLATE"); // no fixed default: half the CPUs, at most 16, when unset
|
|
int procs = pe && *pe ? atoi(pe) : cpus / 2 > 16 ? 16 : cpus / 2;
|
|
int rc = gfx_shader_prepare_seed(dir, cpus, procs < 1 ? 1 : procs);
|
|
// The listed pipelines before the stage libraries: the pipeline cache is saved with them in it and without the
|
|
// tens of thousands of libraries the next step builds, which would take the file past its size limit.
|
|
if (!rc) seed_pipelines(dir);
|
|
if (!rc) lb_warm_prepared(dir);
|
|
return rc;
|
|
}
|
|
|
|
// nx2navi.exe --warm-shaders: every variant of shader-prepared.bin built into the driver's disk cache again from the shader
|
|
// cache, without translating the seed. The AMD cache is keyed to the exe path (a relink keeps it: 29,587 variants in
|
|
// 1.8 s) and holds a limited amount: preparing another build dir evicted this one's libraries, and a library a draw
|
|
// needed then took 40-220 ms in play instead of 0.2 ms. Seconds when the libraries are still there.
|
|
int vk_pipeline_warm_shaders(void) {
|
|
prep_mode = 1; // builds into pipeline-cache.bin alone (cache_load)
|
|
if (!ensure_init()) {
|
|
fprintf(stderr, "warm-shaders: no Vulkan device\n");
|
|
return 1;
|
|
}
|
|
char dir[PLAT_PATH_MAX * 2];
|
|
if (!cache_dir(dir, sizeof dir)) return 1;
|
|
char prp[PLAT_PATH_MAX * 2 + 32];
|
|
snprintf(prp, sizeof prp, "%s/shader-prepared.bin", dir);
|
|
if (!plat_file_exists(prp)) {
|
|
fprintf(stderr, "warm-shaders: no %s (tools/prepare_shaders.py writes it)\n", prp);
|
|
return 1;
|
|
}
|
|
seed_pipelines(dir); // before the libraries, as in vk_pipeline_prepare_shaders
|
|
lb_warm_prepared(dir);
|
|
return 0;
|
|
}
|
|
|
|
// nx2navi.exe --probe-prewarm: the boot's shader and pipeline stages without the game, from the caches next to
|
|
// nx2navi.exe (pipeline-cache.bin, pipeline-keys.bin, shader-answers.bin, the shader cache): the pipeline cache load,
|
|
// the shader prewarm, the recorded pipelines and the compute pipeline prebuild, each followed by vk_mem_stage_log's
|
|
// line, so the log tells what every stage leaves in device memory and in the process's commit. Then up to
|
|
// PROBE_DISPATCHES compute shaders whose pipeline the boot did not keep are asked for as a dispatch would
|
|
// (gfx_shader_load_input for the shader, then vk_pipeline_compute), which gives what a first dispatch costs on this
|
|
// machine and how many of them the pipeline cache held. Fails when one of those pipelines is not created, when the
|
|
// prebuild kept a pipeline it should not have, when a large compute shader went to the background builder although no
|
|
// run dispatched it, or when the record of dispatched shaders, written to a file of the probe's own and read back,
|
|
// does not hold the probe's dispatches. compute-used.bin itself stays as it is; the pipeline cache is saved as after
|
|
// a boot.
|
|
#define PROBE_DISPATCHES 256
|
|
int vk_pipeline_probe_prewarm(void) {
|
|
vk_pipeline_async_enable();
|
|
CU.quiet = 1;
|
|
vk_pipeline_prewarm(); // brings the device up itself: it has to own the key file before ensure_init runs
|
|
if (!P.ready) {
|
|
printf("probe-prewarm: SKIP (the boot prewarm brought no Vulkan device up: none on this machine, NX2NAVI_GPU=null "
|
|
"or NX2NAVI_PIPE_SYNC=1)\n");
|
|
return 0;
|
|
}
|
|
while (__atomic_load_n(&LB.running, __ATOMIC_ACQUIRE) > 0) plat_sleep_ms(50);
|
|
if (!LB.n) {
|
|
printf("probe-prewarm: SKIP (no recorded compute shaders next to the executable: %s)\n", P.cache_path);
|
|
return 0;
|
|
}
|
|
VkPipeStats st;
|
|
vk_pipeline_stats(&st);
|
|
plat_mutex_lock_shared(&LB.stat);
|
|
uint64_t kept = LB.built[2], cache_only = LB.warmed[2], failed = LB.fail[2], there = LB.have[2];
|
|
uint64_t big_unused = LB.big_unused;
|
|
size_t queued = BG.n;
|
|
plat_mutex_unlock_shared(&LB.stat);
|
|
printf("probe-prewarm: %zu compute shaders, %zu to keep: %llu kept, %llu built into the pipeline cache only, %llu "
|
|
"there already, %llu failed, %llu large ones not built (no run dispatched them), %zu large ones for the "
|
|
"background builder; %llu graphics pipelines\n", LB.n, LB.n_keep, (unsigned long long)kept,
|
|
(unsigned long long)cache_only, (unsigned long long)there, (unsigned long long)failed,
|
|
(unsigned long long)big_unused, queued, (unsigned long long)st.graphics);
|
|
int fails = kept > LB.n_keep; // a dispatch never waits in this mode, so nothing else may have been kept
|
|
if (fails) printf("probe-prewarm: FAIL: %llu pipelines kept, %zu expected at most\n", (unsigned long long)kept, LB.n_keep);
|
|
if (queued > LB.n_keep) // only a shader a run dispatched may be built ahead when it is large
|
|
fails++, printf("probe-prewarm: FAIL: %zu large compute shaders queued for the background builder, %zu kept "
|
|
"shaders at most could be among them\n", queued, LB.n_keep);
|
|
uint32_t asked = 0, got = 0;
|
|
double t_all = 0, t_max = 0, t_load = 0;
|
|
for (size_t i = LB.n_keep; i < LB.n && asked < PROBE_DISPATCHES; i++) {
|
|
if (LB.e[i].stage != GFX_STAGE_CS) continue;
|
|
double l0 = now_ms();
|
|
const GfxShader *sh = gfx_shader_load_input(LB.e[i].iid, 0); // from the disk: the boot released its copy
|
|
double load = now_ms() - l0;
|
|
uint64_t words = 0;
|
|
if (!sh || gfx_shader_stage(sh) != GFX_STAGE_CS) continue;
|
|
gfx_shader_spirv(sh, &words);
|
|
if (LB.big_words && words > LB.big_words) continue; // a first dispatch of one compiles it: not what is timed here
|
|
const VkPipeInfo *info = NULL;
|
|
double t0 = now_ms();
|
|
VkPipeline p = vk_pipeline_compute(sh, &info);
|
|
double ms = now_ms() - t0;
|
|
asked++, got += p != VK_NULL_HANDLE;
|
|
t_all += ms, t_load += load;
|
|
if (ms > t_max) t_max = ms;
|
|
}
|
|
vk_mem_stage_log("the probe's first dispatches");
|
|
plat_mutex_lock_shared(&P.lock);
|
|
uint64_t hit = CF.hit;
|
|
plat_mutex_unlock_shared(&P.lock);
|
|
printf("probe-prewarm: %u first dispatches of pipelines the boot did not keep: %u created, avg %.2f ms, longest %.2f "
|
|
"ms, the pipeline cache held %llu of them; reading their shaders took avg %.2f ms\n", asked, got,
|
|
asked ? t_all / asked : 0.0, t_max, (unsigned long long)hit, asked ? t_load / asked : 0.0);
|
|
if (got != asked) fails++, printf("probe-prewarm: FAIL: %u pipelines were not created\n", asked - got);
|
|
if (CU.opened) {
|
|
char rp[sizeof CU.path + 8];
|
|
snprintf(rp, sizeof rp, "%s.probe", CU.path);
|
|
uint64_t rsize = 0;
|
|
CuHeader rh = {0};
|
|
uint32_t mine = 0;
|
|
int wrote = cu_write(rp);
|
|
uint8_t *rf = wrote ? plat_file_read(rp, &rsize) : NULL;
|
|
long rn = cu_parse(rf, rsize, &rh);
|
|
for (long i = 0; i < rn; i++) {
|
|
CuRec r;
|
|
memcpy(&r, rf + sizeof rh + (size_t)i * sizeof r, sizeof r);
|
|
mine += r.run == CU.run;
|
|
}
|
|
free(rf);
|
|
plat_file_delete(rp);
|
|
printf("probe-prewarm: the record of dispatched compute shaders: %ld entries, %u of this run (run %u)\n", rn, mine,
|
|
CU.run);
|
|
if (!wrote || rn < 0 || rh.run != CU.run || mine != got)
|
|
fails++, printf("probe-prewarm: FAIL: the record does not hold the %u shaders dispatched here\n", got);
|
|
}
|
|
lb_stop();
|
|
vk_pipeline_save();
|
|
printf("probe-prewarm: %s\n", fails ? "FAILED" : "passed");
|
|
return fails ? 1 : 0;
|
|
}
|
|
|
|
// ---- dynamic state
|
|
|
|
void vk_pipeline_set_dynamic(VkCommandBuffer cb, const VkPipeState *s) {
|
|
VkDynShadow sh;
|
|
uint64_t skipped = 0;
|
|
sh.have = 0; // nothing known: every call is recorded
|
|
vk_pipeline_set_dynamic_delta(cb, s, &sh, 0, &skipped);
|
|
}
|
|
|
|
// Every graphics pipeline the replay binds lists the same dynamic states (P.dyn; mesh pipelines leave out topology,
|
|
// primitive restart and vertex input), so a value recorded into a command buffer holds across pipeline binds until it
|
|
// is set again. Each call is recorded only when its value differs from the one *sh says the command buffer has: a new
|
|
// pipeline usually changes the shaders and a field or two, not the twenty states this used to record with every bind.
|
|
int vk_pipeline_set_dynamic_delta(VkCommandBuffer cb, const VkPipeState *s, VkDynShadow *sh, int mesh, uint64_t *skipped) {
|
|
if (!ensure_init()) return 0;
|
|
const poly_st *pl = &s->poly;
|
|
const ds_st *d = &s->ds;
|
|
int core = (sh->have & VK_DYN_CORE) != 0, vtx = !mesh && (sh->have & VK_DYN_VTX), blend = (sh->have & VK_DYN_BLEND) != 0;
|
|
int n = 0, left = 0;
|
|
#define DYN(known, field, val, call) \
|
|
do { \
|
|
if ((known) && sh->field == (val)) left++; \
|
|
else sh->field = (val), call, n++; \
|
|
} while (0)
|
|
// The front face maps unchanged: NX2NAVI sets the upper-left window origin, and the y-flipped viewport (y_flip) puts
|
|
// NDC +y at row 0 as NVN does, so the framebuffer winding Vulkan tests is the one NVN tests
|
|
// (a lower-left origin would need it mirrored).
|
|
int front_cw = pl->front == NVN_FRONT_FACE_CW;
|
|
VkCullModeFlags cull = (VkCullModeFlags)(pl->cull & 3); // NVN_FACE_* has VkCullModeFlagBits' values
|
|
VkFrontFace front = front_cw ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE;
|
|
DYN(core, cull, cull, vkCmdSetCullMode(cb, cull));
|
|
DYN(core, front, front, vkCmdSetFrontFace(cb, front));
|
|
if (!mesh) {
|
|
VkPrimitiveTopology topo = vk_pipeline_topology(s->prim);
|
|
DYN(vtx, topo, topo, vkCmdSetPrimitiveTopology(cb, topo));
|
|
}
|
|
DYN(core, dt, (VkBool32)d->dt, vkCmdSetDepthTestEnable(cb, d->dt));
|
|
DYN(core, dw, (VkBool32)d->dw, vkCmdSetDepthWriteEnable(cb, d->dw));
|
|
DYN(core, dop, compare_op(d->df), vkCmdSetDepthCompareOp(cb, compare_op(d->df)));
|
|
DYN(core, st, (VkBool32)d->st, vkCmdSetStencilTestEnable(cb, d->st));
|
|
for (int f = 0; f < 2; f++) {
|
|
VkStencilOp o[3] = {stencil_op(f ? d->fail_b : d->fail_f), stencil_op(f ? d->zpass_b : d->zpass_f),
|
|
stencil_op(f ? d->zfail_b : d->zfail_f)};
|
|
VkCompareOp c = compare_op(f ? d->sf_b : d->sf_f);
|
|
if (core && sh->scmp[f] == c && !memcmp(sh->sop[f], o, sizeof o)) {
|
|
left++;
|
|
continue;
|
|
}
|
|
vkCmdSetStencilOp(cb, f ? VK_STENCIL_FACE_BACK_BIT : VK_STENCIL_FACE_FRONT_BIT, o[0], o[1], o[2], c);
|
|
sh->scmp[f] = c, memcpy(sh->sop[f], o, sizeof o), n++;
|
|
}
|
|
// NVN_POLYGON_OFFSET_ENABLE_* bit of the polygon mode in use: point 1, line 2, fill 4
|
|
unsigned mode = pl->mode <= NVN_POLYGON_MODE_FILL ? pl->mode : NVN_POLYGON_MODE_FILL;
|
|
VkBool32 bias_en = (pl->offset_en >> mode & 1) != 0, discard = s->raster_discard != 0, restart = s->prim_restart != 0;
|
|
DYN(core, bias_en, bias_en, vkCmdSetDepthBiasEnable(cb, bias_en));
|
|
DYN(core, discard, discard, vkCmdSetRasterizerDiscardEnable(cb, discard));
|
|
if (!mesh) DYN(vtx, restart, restart, vkCmdSetPrimitiveRestartEnable(cb, restart));
|
|
if (!mesh && P.dyn_vi && s->vs) {
|
|
VtxInput v;
|
|
vertex_input(s, &v);
|
|
if (vtx && sh->vi.n_attr == v.n_attr && sh->vi.n_bind == v.n_bind &&
|
|
!memcmp(sh->vi.a, v.a, sizeof v.a[0] * v.n_attr) && !memcmp(sh->vi.b, v.b, sizeof v.b[0] * v.n_bind)) {
|
|
left++;
|
|
} else {
|
|
rec_vertex_input(P.set_vi, cb, v.n_bind, v.b, v.n_attr, v.a);
|
|
sh->vi = v, n++;
|
|
}
|
|
} else if (!mesh) {
|
|
sh->vi.n_attr = UINT32_MAX; // not recorded: unknown to the next draw
|
|
}
|
|
if (P.dyn_clamp) {
|
|
VkBool32 clamp = s->depth_clamp != 0;
|
|
DYN(core, clamp, clamp, rec_bool(P.set_clamp, cb, clamp));
|
|
}
|
|
if (P.dyn_poly) {
|
|
VkPolygonMode pm = polygon_mode(pl->mode);
|
|
DYN(core, poly, pm, rec_polygon_mode(P.set_poly, cb, pm));
|
|
}
|
|
if (P.dyn_a2c) { // the same signature as the depth clamp call
|
|
VkBool32 a2c = s->ms.a2c != 0;
|
|
DYN(core, a2c, a2c, rec_bool((PFN_vkCmdSetDepthClampEnableEXT)P.set_a2c, cb, a2c));
|
|
}
|
|
#undef DYN
|
|
sh->have |= VK_DYN_CORE;
|
|
if (mesh) sh->have &= ~(uint32_t)VK_DYN_VTX;
|
|
else sh->have |= VK_DYN_VTX;
|
|
int nt = target_count(s);
|
|
if (nt) {
|
|
VkBool32 en[8];
|
|
VkColorBlendEquationEXT eq[8];
|
|
VkColorComponentFlags wm[8];
|
|
for (int i = 0; i < nt; i++) {
|
|
const blend_st *b = &s->blend[i];
|
|
en[i] = (s->color.blend_en >> i & 1) && blendable(target_format(s->color_fmt[i], NULL, NULL));
|
|
eq[i] = (VkColorBlendEquationEXT){blend_factor(b->src_rgb), blend_factor(b->dst_rgb), blend_op(b->eq_rgb),
|
|
blend_factor(b->src_a), blend_factor(b->dst_a), blend_op(b->eq_a)};
|
|
wm[i] = target_wmask(s, i);
|
|
}
|
|
// The calls write targets [0, nt); targets past nt keep what the command buffer had.
|
|
int known = blend && (uint32_t)nt <= sh->nt;
|
|
if (P.dyn_blend_en) {
|
|
if (known && !memcmp(sh->en, en, sizeof en[0] * nt)) left++;
|
|
else rec_blend_enable(P.set_blend_en, cb, 0, (uint32_t)nt, en), n++;
|
|
}
|
|
if (P.dyn_blend_eq) {
|
|
if (known && !memcmp(sh->eq, eq, sizeof eq[0] * nt)) left++;
|
|
else rec_blend_equation(P.set_blend_eq, cb, 0, (uint32_t)nt, eq), n++;
|
|
}
|
|
if (P.dyn_wmask) {
|
|
if (known && !memcmp(sh->wm, wm, sizeof wm[0] * nt)) left++;
|
|
else rec_write_mask(P.set_wmask, cb, 0, (uint32_t)nt, wm), n++;
|
|
}
|
|
memcpy(sh->en, en, sizeof en[0] * nt), memcpy(sh->eq, eq, sizeof eq[0] * nt), memcpy(sh->wm, wm, sizeof wm[0] * nt);
|
|
sh->nt = blend && sh->nt > (uint32_t)nt ? sh->nt : (uint32_t)nt;
|
|
sh->have |= VK_DYN_BLEND;
|
|
}
|
|
*skipped += (uint64_t)left;
|
|
return n;
|
|
}
|
|
|
|
// ---- statistics, persistence, teardown
|
|
|
|
void vk_pipeline_stats(VkPipeStats *s) {
|
|
plat_mutex_lock_shared(&P.lock);
|
|
*s = P.stats;
|
|
plat_mutex_unlock_shared(&P.lock);
|
|
}
|
|
|
|
static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache
|
|
|
|
// Copies a pipeline cache's data into its file, with cache_rw held exclusively for the copy only (not the file write):
|
|
// P.play into pipeline-cache-play.bin when there is a play cache, else P.cache into pipeline-cache.bin; and P.cache
|
|
// once, with or without a play cache, after the play file was merged into it at the start (P.fold), which then
|
|
// removes the play file.
|
|
// Only when the data changed: the size of a cache's data is asked first, and a cache whose data has the size its file
|
|
// holds is neither copied nor written. A pipeline cache only gains entries, so the same size is the same data. Before,
|
|
// every save copied and wrote the whole cache whenever a pipeline had been created since the last one, also when the
|
|
// cache had held it: three saves of 358.1 MB in one run that added 0.3 MB (2026-11-11).
|
|
// The lock is only ever tried, never waited for: a waiting exclusive request would hold back every new build
|
|
// (a shared request queues behind it), so a pending autosave could stall the replay. The save at exit (exiting) holds
|
|
// new builds back instead (cache_saving_at_exit) and so only waits for the builds already running. wait_ms: how long to
|
|
// keep trying while builds run (0: one try). Returns 0 when no build-free moment came, -1 when the data could not be
|
|
// copied or its file not written (the log says which), 1 otherwise (saved, or nothing to save).
|
|
static int cache_save_one(int which, uint32_t wait_ms, int exiting) {
|
|
VkPipelineCache cache = which ? P.play : P.cache;
|
|
const char *path = which ? P.play_path : P.cache_path;
|
|
if (!cache) return 1;
|
|
uint64_t until = plat_now_ms() + wait_ms;
|
|
if (exiting) __atomic_store_n(&cache_saving_at_exit, 1, __ATOMIC_RELEASE);
|
|
while (!plat_mutex_trylock(&cache_rw)) {
|
|
if (plat_now_ms() >= until) {
|
|
if (exiting) __atomic_store_n(&cache_saving_at_exit, 0, __ATOMIC_RELEASE);
|
|
return 0;
|
|
}
|
|
plat_sleep_ms(2);
|
|
}
|
|
size_t size = 0;
|
|
void *data = NULL;
|
|
int sized = vkGetPipelineCacheData(P.device, cache, &size, NULL) == VK_SUCCESS && size;
|
|
int same = sized && size == P.saved[which];
|
|
int got = sized && !same && (data = malloc(size)) && vkGetPipelineCacheData(P.device, cache, &size, data) == VK_SUCCESS;
|
|
plat_mutex_unlock(&cache_rw);
|
|
// The held-back builds go on before the log line and the file write: one of them may hold a lock those wait for.
|
|
if (exiting) __atomic_store_n(&cache_saving_at_exit, 0, __ATOMIC_RELEASE);
|
|
static uint64_t skipped[2];
|
|
int failed = !sized || (!same && !got);
|
|
if (same) skipped[which]++;
|
|
if (failed) rt_log("vk_pipeline: %s not saved: no copy of its %.1f MB", path, size / 1048576.0);
|
|
if (got) {
|
|
uint64_t t0 = plat_now_ms();
|
|
if (!plat_file_write_atomic(path, data, size)) {
|
|
rt_log("vk_pipeline: cannot write %s (%.1f MB)", path, size / 1048576.0);
|
|
failed = 1;
|
|
} else {
|
|
uint64_t loaded = which ? P.play_loaded : P.cache_loaded;
|
|
rt_log("vk_pipeline: %s saved: %.1f MB (%+.1f MB since the boot) written in %llu ms; %llu saves before it were "
|
|
"skipped with the data unchanged", path, size / 1048576.0, ((double)size - (double)loaded) / 1048576.0,
|
|
(unsigned long long)(plat_now_ms() - t0), (unsigned long long)skipped[which]);
|
|
P.saved[which] = size, skipped[which] = 0;
|
|
if (!which && P.fold) { // the play file's content is in pipeline-cache.bin now
|
|
P.fold = 0;
|
|
plat_file_delete(P.play_path);
|
|
rt_log("vk_pipeline: %s removed: %s holds its pipelines", P.play_path, P.cache_path);
|
|
}
|
|
if (P.feedback) {
|
|
char when[96];
|
|
snprintf(when, sizeof when, "at a save of %.1f MB (%+.1f MB since the boot)", size / 1048576.0,
|
|
((double)size - (double)loaded) / 1048576.0);
|
|
fb_log(when);
|
|
}
|
|
}
|
|
}
|
|
free(data);
|
|
return failed ? -1 : 1;
|
|
}
|
|
|
|
// Both caches' saves (cache_save_one): 1 when each is done, 0 when one found no build-free moment, -1 when one failed.
|
|
static int cache_save(uint32_t wait_ms, int exiting) {
|
|
if (!P.ready || !P.cache || !P.cache_path[0]) return 1;
|
|
// Not under P.lock: holding P.lock through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped
|
|
// every pipeline lookup of the replay, a freeze at each 20 s autosave.
|
|
plat_mutex_lock(&save_lock);
|
|
if (!P.cache) { // vk_pipeline_shutdown destroyed it
|
|
plat_mutex_unlock(&save_lock);
|
|
return 1;
|
|
}
|
|
int ok = 1, r;
|
|
if (!P.play || P.fold) ok = cache_save_one(0, wait_ms, exiting);
|
|
if (P.play && (r = cache_save_one(1, wait_ms, exiting)) < ok) ok = r; // the worse of the two
|
|
plat_mutex_unlock(&save_lock);
|
|
return ok;
|
|
}
|
|
|
|
// Builds still running after this long at a save that waits (exit, the shader preparation) leave the file as it was.
|
|
#define SAVE_WAIT_MS 5000
|
|
|
|
// Returns cache_save's answer.
|
|
static int save_with(uint32_t wait_ms, int exiting) {
|
|
if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log();
|
|
if (P.ready && P.gpl) {
|
|
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; "
|
|
"%llu of pipelines with a feedback loop; avg %.2f ms, "
|
|
"max %.2f ms, %llu failed), %llu lookups by a draw found a stage library not made in this process yet and "
|
|
"left it to a worker (whether the driver's cache has it shows in the library times below), %llu optimised "
|
|
"pipelines swapped in (%llu queued "
|
|
"low after their fast link), %llu drawing-thread links over 5 ms; %llu waiting draws built their libraries "
|
|
"(avg %.2f ms, max %.2f ms)",
|
|
(unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, (unsigned long long)P.g.flinks,
|
|
(unsigned long long)P.g.fb_links,
|
|
P.g.links ? P.g.link_ms / P.g.links : 0.0,
|
|
P.g.link_max, (unsigned long long)P.g.link_fail, (unsigned long long)P.g.defer,
|
|
(unsigned long long)P.g.swaps, (unsigned long long)P.g.requeued, (unsigned long long)P.g.slow,
|
|
(unsigned long long)P.g.dlinks, P.g.dlinks ? P.g.dlink_ms / P.g.dlinks : 0.0, P.g.dlink_max);
|
|
uint64_t rn = P.g.ready_n[0] + P.g.ready_n[1] + P.g.ready_n[2] + P.g.ready_n[3] + P.g.ready_n[4] + P.g.ready_n[5];
|
|
rt_log("vk_pipeline: gpl: pipelines a draw missed were usable after under 4 ms %llu, 4-8 ms %llu, 8-17 ms %llu, "
|
|
"17-50 ms %llu, 50-150 ms %llu, longer %llu (avg %.1f ms, longest %.0f ms); %llu of them had no fast link; "
|
|
"%llu were mesh pipelines (no stage libraries), %llu were named in pipeline-keys.bin and not prebuilt yet "
|
|
"(the others were new to this install), %llu had a shader translated in this session",
|
|
(unsigned long long)P.g.ready_n[0], (unsigned long long)P.g.ready_n[1], (unsigned long long)P.g.ready_n[2],
|
|
(unsigned long long)P.g.ready_n[3], (unsigned long long)P.g.ready_n[4], (unsigned long long)P.g.ready_n[5],
|
|
rn ? P.g.ready_ms / (double)rn : 0.0, P.g.ready_max, (unsigned long long)P.g.ready_nofast,
|
|
(unsigned long long)P.g.ready_mesh, (unsigned long long)P.g.ready_recorded,
|
|
(unsigned long long)P.g.ready_fresh);
|
|
if (P.g.hints)
|
|
rt_log("vk_pipeline: gpl: looking ahead named %llu pipeline states not known yet and queued %llu stage libraries "
|
|
"for them", (unsigned long long)P.g.hints, (unsigned long long)P.g.hint_libs);
|
|
if (A.first || A.taken)
|
|
rt_log("vk_pipeline: whole builds a draw waited for while they were still queued: %llu moved to the head of "
|
|
"the queue, %llu built by the waiting thread (a wait outside the frame's budget, no worker idle)",
|
|
(unsigned long long)A.first, (unsigned long long)A.taken);
|
|
if (U.waits || U.queued)
|
|
rt_log("vk_pipeline: gpl: library jobs for waiting draws: %llu queued, %llu built by a worker, %llu by the "
|
|
"thread that needed them (no worker free), %llu waits ended at their limit; %llu fast links on the %d "
|
|
"library workers",
|
|
(unsigned long long)U.queued, (unsigned long long)U.by_worker, (unsigned long long)U.stolen,
|
|
(unsigned long long)U.timeouts, (unsigned long long)U.links, U.lib_workers);
|
|
static const char *const kn[4] = {"vertex input", "pre-rasterization", "fragment shader", "fragment output"};
|
|
for (int i = 0; i < 4; i++)
|
|
rt_log("vk_pipeline: gpl: %s libraries %llu (avg %.2f ms, max %.2f ms; %llu of 25 ms or more, %.0f ms in all)",
|
|
kn[i], (unsigned long long)P.g.lib_n[i], P.g.lib_n[i] ? P.g.lib_ms[i] / P.g.lib_n[i] : 0.0, P.g.lib_max[i],
|
|
(unsigned long long)P.g.lib_slow[i], P.g.lib_slow_ms[i]);
|
|
if (P.g.pr_added)
|
|
rt_log("vk_pipeline: gpl: %llu of the pre-rasterization libraries had outputs added for their fragment shaders "
|
|
"(avg %.1f ms): another module than the one the preparation built, so the driver's cache has none of them",
|
|
(unsigned long long)P.g.pr_added, P.g.pr_added_ms / (double)P.g.pr_added);
|
|
if (LB.n)
|
|
rt_log("vk_pipeline: gpl: prebuilt libraries a draw used: pre-rasterization %llu, fragment shader %llu; "
|
|
"compute pipelines built on the replay thread %llu", (unsigned long long)P.g.pre_used[LIB_PR],
|
|
(unsigned long long)P.g.pre_used[LIB_FS], (unsigned long long)LB.cs_draw);
|
|
}
|
|
gfx_shader_record_flush(1);
|
|
if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below
|
|
int saved = cache_save(wait_ms, exiting);
|
|
if (!saved && wait_ms)
|
|
rt_log("vk_pipeline: %s not saved: pipeline builds kept running for %u ms", P.play ? P.play_path : P.cache_path,
|
|
(unsigned)wait_ms);
|
|
return saved;
|
|
}
|
|
|
|
// The save after the shader preparation and at shutdown: it waits up to SAVE_WAIT_MS for a moment with no build running.
|
|
void vk_pipeline_save(void) {
|
|
save_with(SAVE_WAIT_MS, 0);
|
|
}
|
|
|
|
// The save as the process ends (window closed, the game's exit, a timed run's end): new builds wait while it waits up to
|
|
// SAVE_WAIT_MS for the running ones to finish (cache_saving_at_exit).
|
|
void vk_pipeline_save_exit(void) {
|
|
save_with(SAVE_WAIT_MS, 1);
|
|
}
|
|
|
|
// The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it
|
|
// runs: at most every 20 s, and only after new pipelines were built or while a save that did not happen is still owed
|
|
// (vk_save_retry.h). New pipeline keys are appended every 2 s.
|
|
static void autosave_once(void) {
|
|
static uint64_t last_keys;
|
|
static SaveRetry sr;
|
|
if (!P.ready) return;
|
|
uint64_t now = plat_now_ms();
|
|
if (now - last_keys >= 2000) {
|
|
last_keys = now;
|
|
if (*(volatile uint32_t *)&K.pend_n) keys_flush();
|
|
pb_flush();
|
|
cu_flush();
|
|
}
|
|
gfx_shader_record_flush(0);
|
|
if (now - sr.at_ms < SAVE_EVERY_MS) return;
|
|
VkPipeStats s;
|
|
vk_pipeline_stats(&s);
|
|
uint64_t n = s.graphics + s.compute + cache_merges; // a merge comes after its pipeline was counted (bg_main)
|
|
if (!save_due(&sr, n, now, P.fold)) return; // P.fold: the play file merged at the start is to be written
|
|
// Only at a moment with no build running (cache_save): while one runs the save waits for the next tick, 500 ms
|
|
// on, before it logs or flushes anything, so a busy stretch does not repeat the statistics lines every tick.
|
|
if (P.cache && P.cache_path[0]) {
|
|
if (!plat_mutex_trylock(&cache_rw)) return;
|
|
plat_mutex_unlock(&cache_rw);
|
|
}
|
|
if (save_tried(&sr, n, now, save_with(0, 0) > 0))
|
|
rt_log("vk_pipeline: the pipeline cache was not saved in %d tries in a row; the next try is made when a new "
|
|
"pipeline is built, or at exit", SAVE_RETRIES + 1);
|
|
}
|
|
|
|
// On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to
|
|
// the disk each) ran on the present thread, and the render thread waited for it to take the next frame.
|
|
static void autosave_main(void *arg) {
|
|
(void)arg;
|
|
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
|
|
for (;;) {
|
|
plat_sleep_ms(500);
|
|
autosave_once();
|
|
}
|
|
}
|
|
|
|
void vk_pipeline_autosave(void) {
|
|
static int32_t started;
|
|
if (!P.ready || plat_xchg32(&started, 1)) return;
|
|
if (!plat_thread_spawn(autosave_main, NULL, "pipe autosave", 0))
|
|
rt_log("vk_pipeline: cannot start the autosave thread; the cache is saved at exit only");
|
|
}
|
|
|
|
void vk_pipeline_shutdown(void) {
|
|
if (!P.ready) return;
|
|
lb_stop();
|
|
workers_drain(); // jobs point at layouts and Pipe entries freed below
|
|
vk_pipeline_save();
|
|
plat_mutex_lock(&P.lock);
|
|
vkDeviceWaitIdle(P.device);
|
|
for (int i = 0; i < PIPE_BUCKETS; i++) {
|
|
for (Pipe *p = pipes[i], *n; p; p = n) {
|
|
n = p->next;
|
|
if (p->info.pipeline) vkDestroyPipeline(P.device, p->info.pipeline, NULL);
|
|
if (p->fast && p->fast != p->info.pipeline) vkDestroyPipeline(P.device, p->fast, NULL);
|
|
free(p->bindings), free(p);
|
|
}
|
|
pipes[i] = NULL;
|
|
}
|
|
libs_destroy();
|
|
for (int i = 0; i < LAYOUT_BUCKETS; i++) {
|
|
for (Layout *l = layouts[i], *n; l; l = n) {
|
|
n = l->next;
|
|
for (int k = 0; k < 2; k++)
|
|
if (l->tpl[k]) P.tpl_destroy(P.device, l->tpl[k], NULL);
|
|
vkDestroyPipelineLayout(P.device, l->layout, NULL);
|
|
for (int h = 0; h < 2; h++)
|
|
if (l->half[h]) vkDestroyPipelineLayout(P.device, l->half[h], NULL);
|
|
vkDestroyDescriptorSetLayout(P.device, l->set, NULL);
|
|
if (l->set1) vkDestroyDescriptorSetLayout(P.device, l->set1, NULL);
|
|
free(l);
|
|
}
|
|
layouts[i] = NULL;
|
|
}
|
|
plat_mutex_lock(&save_lock); // not while the autosave thread copies it
|
|
if (P.cache) vkDestroyPipelineCache(P.device, P.cache, NULL);
|
|
if (P.play) vkDestroyPipelineCache(P.device, P.play, NULL);
|
|
P.cache = P.play = VK_NULL_HANDLE;
|
|
plat_mutex_unlock(&save_lock);
|
|
memset(&P.stats, 0, sizeof P.stats);
|
|
plat_xchg32(&P.ready, 0);
|
|
plat_mutex_unlock(&P.lock);
|
|
}
|
|
|
|
// ---- self-test: census shaders (probe_census_dir, tools/shader_census.py) through gfx_shader_get into pipelines.
|
|
// Graphics pairs are a fragment shader and a vertex shader of the same source file, spread over the census; compute
|
|
// shaders are spread the same way. Every pipeline is built twice (the second call must hit the cache) and the
|
|
// dynamic state of the first graphics pipeline is recorded into a throwaway command buffer.
|
|
|
|
typedef struct { const char *key, *c1, *src; size_t c1_len, src_len; int stage; uint32_t local[3]; } CensusEntry;
|
|
|
|
static const char *st_field(const char *from, const char *to, const char *name, size_t *len) {
|
|
char pat[32];
|
|
snprintf(pat, sizeof pat, "\"%s\": \"", name);
|
|
const char *p = strstr(from, pat);
|
|
if (!p || p >= to) return NULL;
|
|
p += strlen(pat);
|
|
const char *e = memchr(p, '"', (size_t)(to - p));
|
|
if (!e) return NULL;
|
|
*len = (size_t)(e - p);
|
|
return p;
|
|
}
|
|
|
|
static int census_cmp(const void *x, const void *y) {
|
|
const CensusEntry *a = x, *b = y;
|
|
size_t n = a->src_len < b->src_len ? a->src_len : b->src_len;
|
|
int c = memcmp(a->src, b->src, n);
|
|
if (c) return c;
|
|
if (a->src_len != b->src_len) return a->src_len < b->src_len ? -1 : 1;
|
|
return a->stage - b->stage;
|
|
}
|
|
|
|
static char *census_read(const char *root, size_t *size) {
|
|
char path[PLAT_PATH_MAX * 2 + 64];
|
|
snprintf(path, sizeof path, "%s/jobs.json", root);
|
|
FILE *f = fopen(path, "rb");
|
|
if (!f) return NULL;
|
|
fseek(f, 0, SEEK_END);
|
|
long n = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
char *b = malloc((size_t)n + 1);
|
|
if (b && fread(b, 1, (size_t)n, f) != (size_t)n) free(b), b = NULL;
|
|
fclose(f);
|
|
if (b) b[n] = 0, *size = (size_t)n;
|
|
return b;
|
|
}
|
|
|
|
static int census_parse(const char *jobs, CensusEntry *out, int max) {
|
|
int n = 0;
|
|
for (const char *p = jobs; n < max && (p = strstr(p, "\"job\": {")); p += 8) {
|
|
const char *end = strstr(p, "\"origin\": ");
|
|
if (!end) break;
|
|
size_t kl, sl, cl, rl;
|
|
const char *k = st_field(p, end, "key", &kl), *s = st_field(p, end, "stage", &sl), *c = st_field(p, end, "c1", &cl),
|
|
*r = st_field(p, end, "source", &rl);
|
|
if (!k || !s || !c || !r || kl != 64) continue;
|
|
CensusEntry *e = &out[n++];
|
|
e->key = k, e->c1 = c, e->c1_len = cl / 2, e->src = r, e->src_len = rl;
|
|
e->stage = !strncmp(s, "vs", sl) ? GFX_STAGE_VS : !strncmp(s, "ps", sl) ? GFX_STAGE_FS : GFX_STAGE_CS;
|
|
e->local[0] = e->local[1] = e->local[2] = 1;
|
|
const char *l = strstr(p, "\"local\": [");
|
|
if (l && l < end) {
|
|
l += 10;
|
|
for (int i = 0; i < 3; i++) {
|
|
while (*l == ' ' || *l == '\n' || *l == '\r' || *l == ',') l++;
|
|
e->local[i] = (uint32_t)strtoul(l, (char **)&l, 10);
|
|
}
|
|
}
|
|
}
|
|
return n;
|
|
}
|
|
|
|
static int hexv(char c) { return c <= '9' ? c - '0' : (c | 32) - 'a' + 10; }
|
|
|
|
// The program the game would hand NVN for a census entry, translated with default answers (as gfx_shader_probe.c).
|
|
static const GfxShader *census_shader(const char *root, const CensusEntry *e) {
|
|
char path[PLAT_PATH_MAX * 2 + 128];
|
|
static uint8_t c1[8192];
|
|
uint8_t control[0x880] = {0};
|
|
if (e->c1_len > sizeof c1) return NULL;
|
|
for (size_t b = 0; b < e->c1_len; b++) c1[b] = (uint8_t)(hexv(e->c1[b * 2]) << 4 | hexv(e->c1[b * 2 + 1]));
|
|
snprintf(path, sizeof path, "%s/code/%.64s.code", root, e->key);
|
|
FILE *f = fopen(path, "rb");
|
|
if (!f) return NULL;
|
|
fseek(f, 0, SEEK_END);
|
|
long body = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
size_t skip = e->stage == GFX_STAGE_CS ? 0 : 0x30, total = skip + (size_t)body;
|
|
uint8_t *code = calloc(1, total);
|
|
int ok = code && fread(code + skip, 1, (size_t)body, f) == (size_t)body;
|
|
fclose(f);
|
|
if (!ok) return free(code), NULL;
|
|
uint32_t c1_off = (uint32_t)(total - e->c1_len), length = (uint32_t)total;
|
|
memcpy(control + 0x700, &c1_off, 4), memcpy(control + 0x704, &length, 4);
|
|
GfxShaderAnswers a = {0};
|
|
if (e->stage == GFX_STAGE_CS) for (int k = 0; k < 3; k++) a.local[k] = (uint16_t)e->local[k];
|
|
const GfxShader *s = gfx_shader_get(e->stage, control, code, total, c1, &a);
|
|
free(code);
|
|
return s;
|
|
}
|
|
|
|
// A state for a census pair: one target per fragment output (RGBA8 of the output's type), D32S8 depth with the test on,
|
|
// alpha blending on target 0, one RGBA32 attribute per vertex input, interleaved in stream 0.
|
|
static void selftest_state(VkPipeState *s, const GfxShader *vs, const GfxShader *fs) {
|
|
vk_pipe_state_defaults(s);
|
|
s->vs = vs, s->fs = fs;
|
|
GfxShaderIo io[32];
|
|
uint32_t outs = gfx_shader_outputs(fs, io);
|
|
for (int i = 0; i < 8; i++)
|
|
if (outs >> i & 1)
|
|
s->color_fmt[i] = io[i].type == GFX_TYPE_UINT ? NVN_FORMAT_RGBA8UI : io[i].type == GFX_TYPE_SINT ? NVN_FORMAT_RGBA8I
|
|
: NVN_FORMAT_RGBA8;
|
|
s->depth_fmt = NVN_FORMAT_DEPTH32F_STENCIL8;
|
|
s->ds.dt = 1;
|
|
s->color.blend_en = 1;
|
|
s->blend[0].src_rgb = NVN_BLEND_FUNC_SRC_ALPHA, s->blend[0].dst_rgb = NVN_BLEND_FUNC_ONE_MINUS_SRC_ALPHA;
|
|
s->poly.cull = NVN_FACE_BACK;
|
|
uint32_t ins = gfx_shader_inputs(vs, io), off = 0;
|
|
for (int l = 0; l < 16; l++)
|
|
if (ins >> l & 1) {
|
|
va_st a = {io[l].type == GFX_TYPE_UINT ? NVN_FORMAT_RGBA32UI : io[l].type == GFX_TYPE_SINT ? NVN_FORMAT_RGBA32I
|
|
: NVN_FORMAT_RGBA32F,
|
|
off, 0};
|
|
memcpy(&s->vattr[l], &a, sizeof a);
|
|
off += 16;
|
|
}
|
|
s->vstream[0].stride = off ? off : 16;
|
|
}
|
|
|
|
// Records the dynamic state of a pipeline into a command buffer that is never submitted (validation checks the calls).
|
|
static int selftest_record(const VkPipeState *s, VkPipeline pipe) {
|
|
VkCtx *c = vk_ctx();
|
|
VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .queueFamilyIndex = c->gfx_family};
|
|
VkCommandPool pool;
|
|
if (vkCreateCommandPool(c->device, &pi, NULL, &pool) != VK_SUCCESS) return 0;
|
|
VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = pool,
|
|
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
|
|
VkCommandBuffer cb;
|
|
int ok = vkAllocateCommandBuffers(c->device, &ai, &cb) == VK_SUCCESS;
|
|
if (ok) {
|
|
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
|
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
|
|
vkBeginCommandBuffer(cb, &bi);
|
|
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, pipe);
|
|
vk_pipeline_set_dynamic(cb, s);
|
|
ok = vkEndCommandBuffer(cb) == VK_SUCCESS;
|
|
}
|
|
vkDestroyCommandPool(c->device, pool, NULL);
|
|
return ok;
|
|
}
|
|
|
|
int vk_pipeline_selftest(void) {
|
|
enum { MAXN = 20000, WANT_PAIRS = 64, WANT_CS = 16 };
|
|
char root[PLAT_PATH_MAX * 2];
|
|
if (!vk_init() || !ensure_init()) { printf("vk_pipeline selftest: FAILED: no Vulkan device\n"); return 1; }
|
|
if (!probe_census_dir(root, sizeof root)) { // the census folder, build/<version>/shaders
|
|
printf("vk_pipeline selftest: FAILED: no build/v%d/shaders/jobs.json above the exe\n", rt_game_version);
|
|
return 1;
|
|
}
|
|
size_t size;
|
|
char *jobs = census_read(root, &size);
|
|
CensusEntry *e = malloc(sizeof *e * MAXN);
|
|
int n = jobs && e ? census_parse(jobs, e, MAXN) : 0;
|
|
if (!n) { printf("vk_pipeline selftest: FAILED: no census entries\n"); free(jobs), free(e); return 1; }
|
|
qsort(e, (size_t)n, sizeof *e, census_cmp);
|
|
long err0 = vk_validation_errors_total();
|
|
VkPipeStats st0;
|
|
vk_pipeline_stats(&st0);
|
|
|
|
// candidate pairs: each fragment shader of a source file with one of the file's vertex shaders (round robin)
|
|
int (*pairs)[2] = malloc(sizeof *pairs * (size_t)n), np = 0, *cs = malloc(sizeof *cs * (size_t)n), nc = 0;
|
|
for (int i = 0; i < n;) {
|
|
int j = i, v0 = -1, nv = 0, nf = 0;
|
|
for (; j < n && e[j].src_len == e[i].src_len && !memcmp(e[j].src, e[i].src, e[i].src_len); j++) {
|
|
if (e[j].stage == GFX_STAGE_VS && v0 < 0) v0 = j; // sorted by stage within a file: vertex shaders first
|
|
nv += e[j].stage == GFX_STAGE_VS;
|
|
if (e[j].stage == GFX_STAGE_CS) cs[nc++] = j;
|
|
}
|
|
for (int k = i; k < j && nv; k++)
|
|
if (e[k].stage == GFX_STAGE_FS) pairs[np][0] = v0 + nf++ % nv, pairs[np][1] = k, np++;
|
|
i = j;
|
|
}
|
|
int tried = 0, built = 0, cached = 1, shader_fail = 0, ctried = 0, cbuilt = 0, recorded = 0, varying_fix = 0;
|
|
for (int t = 0; t < WANT_PAIRS && t < np; t++) {
|
|
const int *pr = pairs[(long long)t * np / (np < WANT_PAIRS ? np : WANT_PAIRS)];
|
|
tried++;
|
|
const GfxShader *vs = census_shader(root, &e[pr[0]]), *fs = census_shader(root, &e[pr[1]]);
|
|
if (!vs || !fs) { shader_fail++; continue; }
|
|
if (gfx_shader_inputs(fs, NULL) & ~gfx_shader_outputs(vs, NULL)) varying_fix++;
|
|
VkPipeState s;
|
|
selftest_state(&s, vs, fs);
|
|
GfxShaderAnswers fa = {0};
|
|
vk_pipeline_answers(&s, NULL, &fa);
|
|
const VkPipeInfo *info;
|
|
VkPipeline p = vk_pipeline_graphics(&s, &info);
|
|
if (!p) continue;
|
|
built++;
|
|
const VkPipeInfo *again;
|
|
if (vk_pipeline_graphics(&s, &again) != p || again != info) cached = 0;
|
|
if (!recorded) recorded = selftest_record(&s, p);
|
|
}
|
|
for (int t = 0; t < WANT_CS && t < nc; t++) {
|
|
const CensusEntry *c = &e[cs[(long long)t * nc / (nc < WANT_CS ? nc : WANT_CS)]];
|
|
ctried++;
|
|
const GfxShader *s = census_shader(root, c);
|
|
if (!s) { shader_fail++; continue; }
|
|
const VkPipeInfo *info;
|
|
VkPipeline p = vk_pipeline_compute(s, &info);
|
|
if (!p) continue;
|
|
cbuilt++;
|
|
const VkPipeInfo *again;
|
|
if (vk_pipeline_compute(s, &again) != p || again != info) cached = 0;
|
|
}
|
|
vk_pipeline_save();
|
|
VkPipeStats st;
|
|
vk_pipeline_stats(&st);
|
|
long errors = vk_validation_errors_total() - err0;
|
|
printf("vk_pipeline selftest: graphics %d/%d census pairs (%d needed vertex outputs added), compute %d/%d, "
|
|
"%d shaders failed to translate; layouts %llu (%llu push); cache hits %llu; dynamic state recorded %d; "
|
|
"validation errors %ld\n",
|
|
built, tried, varying_fix, cbuilt, ctried, shader_fail, (unsigned long long)(st.layouts - st0.layouts),
|
|
(unsigned long long)(st.push_layouts - st0.push_layouts), (unsigned long long)(st.hits - st0.hits), recorded,
|
|
errors);
|
|
free(pairs), free(cs), free(e), free(jobs);
|
|
int ok = built > 0 && cbuilt > 0 && cached && recorded && errors == 0;
|
|
printf("vk_pipeline selftest: %s\n", ok ? "passed" : "FAILED");
|
|
return ok ? 0 : 1;
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): the pipeline table's entries with their pipelines and fast links (never
|
|
// evicted), layouts, stage libraries, the job queues and the pipeline-keys.bin set. A walk of the hash chains under P.lock
|
|
// held shared: the workers' inserts and a draw's memo miss wait for it (well under a millisecond).
|
|
void vk_pipeline_sizes(PerfSizes *s) {
|
|
uint64_t ent = 0, built = 0, fast = 0, libn = 0, libb = 0, lq = 0;
|
|
plat_mutex_lock_shared(&P.lock);
|
|
for (int i = 0; i < PIPE_BUCKETS; i++)
|
|
for (const Pipe *p = pipes[i]; p; p = p->next) ent++, built += p->info.pipeline != VK_NULL_HANDLE, fast += p->fast != VK_NULL_HANDLE;
|
|
for (int i = 0; i < LIB_BUCKETS; i++)
|
|
for (const Lib *b = libs[i]; b; b = b->next) libn++, libb += b->pipe != VK_NULL_HANDLE;
|
|
for (const LibJob *j = U.head; j; j = j->next) lq++;
|
|
uint64_t lay = P.stats.layouts, fails = P.stats.failures, jobs = A.inflight, seen = K.seen_n, seen_b = (uint64_t)K.seen_cap * 8;
|
|
plat_mutex_unlock_shared(&P.lock);
|
|
perf_size(s, "pipeline entries", ent, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "with a pipeline", built, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "fast links", fast, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "pipeline failures", fails, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "pipeline layouts", lay, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "stage library entries", libn, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "built stage libraries", libb, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "library jobs queued", lq, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "pipeline jobs queued or running", jobs, 0, PERF_SIZE_NONE);
|
|
perf_size(s, "pipeline keys seen", seen, 0, seen_b);
|
|
}
|