mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-21 10:09:59 -04:00
Keep compiled pipelines as pipeline binaries under the runtime's own keys, and let the one-shot draw wait be set.
NVIDIA 621.01 reuses a pipeline-cache.bin entry only in the executable that wrote it, so every renamed or updated exe compiled all of its pipelines again (23 s boots, 17 % read from a 470 MB file). With NX2NAVI_PIPE_BINARIES=1, where the driver has VK_KHR_pipeline_binary, every pipeline, stage library and compute pipeline is looked up by its driver pipeline key in pipeline-binaries.bin and made from the stored binaries; anything not stored is compiled once with its data captured and appended. Each binary is stored once under its own key: nearly every graphics pipeline carries the same 2 MiB binary, which made a first version of the file 7 GB. pipeline-cache.bin is not used while this is on. The knob is off by default while it is being tested; NX2NAVI_PIPE_BINARIES_MAX_MB caps the file. NX2NAVI_ONESHOT_WAIT_MS sets how long a draw into a target the previous frame did not render waits for a pipeline built for the first time before it is skipped; the default keeps the old 30 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -66,6 +66,10 @@
|
|||||||
# Which draws are dropped instead of waiting for their pipeline: strict (only draws into per-frame passes), any (also
|
# Which draws are dropped instead of waiting for their pipeline: strict (only draws into per-frame passes), any (also
|
||||||
# draws into passes whose target the previous frame rendered) or fullscreen (those and full-screen draws).
|
# draws into passes whose target the previous frame rendered) or fullscreen (those and full-screen draws).
|
||||||
# pipe_skip = fullscreen
|
# pipe_skip = fullscreen
|
||||||
|
# How long (ms) a draw into a target the previous frame did not render waits for a pipeline built for the first time,
|
||||||
|
# then is skipped. The default waits, as such a target (a lookup table, a new area's target) can stay empty if
|
||||||
|
# skipped; a lower value caps the stall.
|
||||||
|
# oneshot_wait_ms = 30000
|
||||||
# Build shader stage libraries in the background ahead of time, and the threads for it (1 to 16; default: a quarter of
|
# Build shader stage libraries in the background ahead of time, and the threads for it (1 to 16; default: a quarter of
|
||||||
# the CPU threads).
|
# the CPU threads).
|
||||||
# lib_prebuild = 1
|
# lib_prebuild = 1
|
||||||
@@ -83,6 +87,12 @@
|
|||||||
# pipe_feedback = 1
|
# pipe_feedback = 1
|
||||||
# pipeline-cache.bin larger than this (MB) starts empty and is rewritten (0: no limit).
|
# pipeline-cache.bin larger than this (MB) starts empty and is rewritten (0: no limit).
|
||||||
# pipeline_cache_max_mb = 512
|
# pipeline_cache_max_mb = 512
|
||||||
|
# 1: where the driver has VK_KHR_pipeline_binary, compiled pipelines are kept in pipeline-binaries.bin under the
|
||||||
|
# runtime's own keys, so a renamed or updated exe starts without compiling them again (pipeline-cache.bin is then
|
||||||
|
# not used). Default 0 (pipeline-cache.bin only) while it is being tested.
|
||||||
|
# pipe_binaries = 0
|
||||||
|
# pipeline-binaries.bin larger than this (MB) starts empty and is rewritten (0: no limit).
|
||||||
|
# pipe_binaries_max_mb = 1024
|
||||||
# Build known pipelines ahead of time.
|
# Build known pipelines ahead of time.
|
||||||
# pipe_prebuild = 1
|
# pipe_prebuild = 1
|
||||||
# 1: the start waits until every known pipeline is built. Default 0: with the graphics pipeline library the start
|
# 1: the start waits until every known pipeline is built. Default 0: with the graphics pipeline library the start
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ static int save_ok = 1; // the last cfg_save_to wrote (o
|
|||||||
// Every name the runtime reads (and the patch sets in generated/), without the prefix. A file key outside this list is
|
// Every name the runtime reads (and the patch sets in generated/), without the prefix. A file key outside this list is
|
||||||
// logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line.
|
// logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line.
|
||||||
static const char *const known[] = {
|
static const char *const known[] = {
|
||||||
"CORES", "TM_DISPATCH", "NVN2_CS_TABLE", "SKIP_SHADERS", "MOVIE_DEBAND", "MOVIE_FPS", "MOVIE_SKIP", "MOVIE_OVERLAY", "MOVIES", "NPLN", "NVN2_PROGRAMS", "NVN_TRAP", "OCARINA_REALTIME", "OCARINA_STEP", "VI_HZ", "VI_VRR", "TIME_DUMP", "SHADER_DUMP", "SHOT_AFTER", "SHOT_DIR", "SHOT_EVERY", // Switch 2 port: movie skip (romfs.c), NVN2 proc traps (nvn_procs.c)
|
"CORES", "TM_DISPATCH", "NVN2_CS_TABLE", "SKIP_SHADERS", "MOVIE_DEBAND", "MOVIE_FPS", "MOVIE_SKIP", "MOVIE_OVERLAY", "MOVIES", "NPLN", "NVN2_PROGRAMS", "NVN_TRAP", "OCARINA_REALTIME", "ONESHOT_WAIT_MS", "OCARINA_STEP", "VI_HZ", "VI_VRR", "TIME_DUMP", "SHADER_DUMP", "SHOT_AFTER", "SHOT_DIR", "SHOT_EVERY", // Switch 2 port: movie skip (romfs.c), NVN2 proc traps (nvn_procs.c)
|
||||||
"AB", "AMIIBO_DIR", "ANISO", "APP_POOL", "ASPECT", "ASPECT_PATCH", "ASSUME_FOCUS", "ASTC", "ASTC_BC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_CHANNELS", "AUDIO_OFF", "AUDIO_PREFILL_MS",
|
"AB", "AMIIBO_DIR", "ANISO", "APP_POOL", "ASPECT", "ASPECT_PATCH", "ASSUME_FOCUS", "ASTC", "ASTC_BC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_CHANNELS", "AUDIO_OFF", "AUDIO_PREFILL_MS",
|
||||||
"BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", "BIND_LSTICK_DOWN", "BIND_LSTICK_LEFT",
|
"BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", "BIND_LSTICK_DOWN", "BIND_LSTICK_LEFT",
|
||||||
"BIND_LSTICK_RIGHT", "BIND_LSTICK_UP", "BIND_MINUS",
|
"BIND_LSTICK_RIGHT", "BIND_LSTICK_UP", "BIND_MINUS",
|
||||||
@@ -50,7 +50,7 @@ static const char *const known[] = {
|
|||||||
"MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",
|
"MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",
|
||||||
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
|
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
|
||||||
"MV_SURVEY",
|
"MV_SURVEY",
|
||||||
"ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC",
|
"ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC",
|
||||||
"PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES",
|
"PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES",
|
||||||
"PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC",
|
"PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC",
|
||||||
"REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS",
|
"REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS",
|
||||||
@@ -197,7 +197,7 @@ static const CfgMeta metas[] = {
|
|||||||
{"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART},
|
{"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART},
|
||||||
{"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART},
|
{"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART},
|
||||||
{"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART},
|
{"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART},
|
||||||
{"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART},
|
{"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, {"PIPE_BINARIES", "0", "0|1", RESTART}, {"ONESHOT_WAIT_MS", "30000", "0..60000", RESTART}, {"PIPE_BINARIES_MAX_MB", "1024", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART},
|
||||||
{"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART},
|
{"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART},
|
||||||
// textures and memory
|
// textures and memory
|
||||||
{"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART},
|
{"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART},
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ typedef struct VkCtx {
|
|||||||
has_hdr_metadata, // VK_EXT_hdr_metadata (vk_hdr.c vk_hdr_metadata)
|
has_hdr_metadata, // VK_EXT_hdr_metadata (vk_hdr.c vk_hdr_metadata)
|
||||||
has_mesh_shader, has_vrs, // VK_EXT_mesh_shader with taskShader and meshShader (NVN2 task/mesh stages)
|
has_mesh_shader, has_vrs, // VK_EXT_mesh_shader with taskShader and meshShader (NVN2 task/mesh stages)
|
||||||
has_maximal_reconvergence, // VK_KHR_shader_maximal_reconvergence (vk_pipeline.c spirv_reconverge)
|
has_maximal_reconvergence, // VK_KHR_shader_maximal_reconvergence (vk_pipeline.c spirv_reconverge)
|
||||||
|
has_pipeline_binary, // VK_KHR_pipeline_binary with VK_KHR_maintenance5 (vk_pipeline.c pipe_create)
|
||||||
has_mesh_queries; // meshShaderQueries, kept while NX2NAVI_GPU_PASSES or NX2NAVI_DUMP is set
|
has_mesh_queries; // meshShaderQueries, kept while NX2NAVI_GPU_PASSES or NX2NAVI_DUMP is set
|
||||||
// Feedback loops (maxwell.md 1.9, vk_exec_draw.c feedback_prepare): a draw samples a target of its open pass that
|
// Feedback loops (maxwell.md 1.9, vk_exec_draw.c feedback_prepare): a draw samples a target of its open pass that
|
||||||
// an earlier draw of the pass wrote. NX2NAVI_FEEDBACK=0: VK_FEEDBACK_OFF (sampled in GENERAL without a barrier, the
|
// an earlier draw of the pass wrote. NX2NAVI_FEEDBACK=0: VK_FEEDBACK_OFF (sampled in GENERAL without a barrier, the
|
||||||
|
|||||||
@@ -238,6 +238,8 @@ typedef struct {
|
|||||||
VkPhysicalDeviceMeshShaderFeaturesEXT mesh;
|
VkPhysicalDeviceMeshShaderFeaturesEXT mesh;
|
||||||
VkPhysicalDeviceFragmentShadingRateFeaturesKHR vrs;
|
VkPhysicalDeviceFragmentShadingRateFeaturesKHR vrs;
|
||||||
VkPhysicalDeviceShaderMaximalReconvergenceFeaturesKHR mrc;
|
VkPhysicalDeviceShaderMaximalReconvergenceFeaturesKHR mrc;
|
||||||
|
VkPhysicalDeviceMaintenance5FeaturesKHR m5;
|
||||||
|
VkPhysicalDevicePipelineBinaryFeaturesKHR pbin;
|
||||||
} FeatureChain;
|
} FeatureChain;
|
||||||
|
|
||||||
static int create_device(void) {
|
static int create_device(void) {
|
||||||
@@ -280,6 +282,17 @@ static int create_device(void) {
|
|||||||
// (vk_pipeline.c spirv_reconverge)
|
// (vk_pipeline.c spirv_reconverge)
|
||||||
OPTIONAL_EXT(has_maximal_reconvergence, VK_KHR_SHADER_MAXIMAL_RECONVERGENCE_EXTENSION_NAME, mrc,
|
OPTIONAL_EXT(has_maximal_reconvergence, VK_KHR_SHADER_MAXIMAL_RECONVERGENCE_EXTENSION_NAME, mrc,
|
||||||
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MAXIMAL_RECONVERGENCE_FEATURES_KHR);
|
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MAXIMAL_RECONVERGENCE_FEATURES_KHR);
|
||||||
|
// Compiled pipelines kept under the runtime's own keys (vk_pipeline.c pipe_create): VK_KHR_pipeline_binary, which
|
||||||
|
// needs VK_KHR_maintenance5 (core only from Vulkan 1.4; the instance asks for 1.3). NX2NAVI_PIPE_BINARIES=0 leaves
|
||||||
|
// both off.
|
||||||
|
if (cfg_bool("NX2NAVI_PIPE_BINARIES") && has_ext(ep, ne, VK_KHR_PIPELINE_BINARY_EXTENSION_NAME) &&
|
||||||
|
has_ext(ep, ne, VK_KHR_MAINTENANCE_5_EXTENSION_NAME)) {
|
||||||
|
g_vk.has_pipeline_binary = 1;
|
||||||
|
ext[n_ext++] = VK_KHR_MAINTENANCE_5_EXTENSION_NAME;
|
||||||
|
ext[n_ext++] = VK_KHR_PIPELINE_BINARY_EXTENSION_NAME;
|
||||||
|
CHAIN(m5, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_FEATURES_KHR);
|
||||||
|
CHAIN(pbin, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_BINARY_FEATURES_KHR);
|
||||||
|
}
|
||||||
// Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering
|
// Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering
|
||||||
int has_fbl = 0, has_lr = 0;
|
int has_fbl = 0, has_lr = 0;
|
||||||
if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) &&
|
if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) &&
|
||||||
@@ -377,6 +390,7 @@ static int create_device(void) {
|
|||||||
rt_log("vk: mesh shaders: %s", g_vk.has_mesh_shader ? "VK_EXT_mesh_shader" : "not supported (task/mesh draws are skipped)");
|
rt_log("vk: mesh shaders: %s", g_vk.has_mesh_shader ? "VK_EXT_mesh_shader" : "not supported (task/mesh draws are skipped)");
|
||||||
} // vkGetDeviceFaultInfoEXT needs the feature, not just the extension
|
} // vkGetDeviceFaultInfoEXT needs the feature, not just the extension
|
||||||
if (g_vk.has_maximal_reconvergence && !c->mrc.shaderMaximalReconvergence) g_vk.has_maximal_reconvergence = 0;
|
if (g_vk.has_maximal_reconvergence && !c->mrc.shaderMaximalReconvergence) g_vk.has_maximal_reconvergence = 0;
|
||||||
|
if (g_vk.has_pipeline_binary && !(c->pbin.pipelineBinaries && c->m5.maintenance5)) g_vk.has_pipeline_binary = 0;
|
||||||
g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress;
|
g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress;
|
||||||
g_vk.has_astc_ldr = h->textureCompressionASTC_LDR;
|
g_vk.has_astc_ldr = h->textureCompressionASTC_LDR;
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -43,7 +43,21 @@
|
|||||||
#define NVN_STAGE_CS 5
|
#define NVN_STAGE_CS 5
|
||||||
#define NVN_STAGE_TASK 6 // NVN2
|
#define NVN_STAGE_TASK 6 // NVN2
|
||||||
#define NVN_STAGE_MESH 7
|
#define NVN_STAGE_MESH 7
|
||||||
#define ONESHOT_WAIT_MS 30000u // prepare_draw: how long a one-shot draw waits for its pipeline before it is skipped
|
// prepare_draw: how long a one-shot draw waits for its pipeline before it is skipped (NX2NAVI_ONESHOT_WAIT_MS, default
|
||||||
|
// 30000). Such a draw renders into a target the previous frame did not (a volume, a LUT, a target made new at an area
|
||||||
|
// change); skipped, the target can stay empty for the session, so the default waits. A lower value trades that risk for
|
||||||
|
// a frame that never stalls longer than the value on a first-time pipeline.
|
||||||
|
static uint32_t oneshot_wait_ms(void) {
|
||||||
|
static int64_t ms = -1;
|
||||||
|
if (ms < 0) {
|
||||||
|
ms = strtoll(cfg_value("NX2NAVI_ONESHOT_WAIT_MS"), NULL, 10);
|
||||||
|
if (ms < 0) ms = 0;
|
||||||
|
if (ms != 30000) rt_log("vk_exec: a one-shot draw waits at most %lld ms for its pipeline, then is skipped "
|
||||||
|
"(NX2NAVI_ONESHOT_WAIT_MS)", (long long)ms);
|
||||||
|
}
|
||||||
|
return (uint32_t)ms;
|
||||||
|
}
|
||||||
|
#define ONESHOT_WAIT_MS oneshot_wait_ms()
|
||||||
|
|
||||||
enum { W_PROGRAM, W_SHADER, W_PIPELINE, W_NO_VS, W_INDEX, W_UB, W_BIND, W_SSBO_ALIGN, W_SSBO_SLOT, W_RESTART, W_INDIRECT, W_MESH, W_COUNT };
|
enum { W_PROGRAM, W_SHADER, W_PIPELINE, W_NO_VS, W_INDEX, W_UB, W_BIND, W_SSBO_ALIGN, W_SSBO_SLOT, W_RESTART, W_INDIRECT, W_MESH, W_COUNT };
|
||||||
|
|
||||||
|
|||||||
@@ -164,9 +164,401 @@ static void cache_path_init(void) {
|
|||||||
snprintf(P.cache_path, sizeof P.cache_path, "%s/pipeline-cache.bin", exe);
|
snprintf(P.cache_path, sizeof P.cache_path, "%s/pipeline-cache.bin", exe);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- pipeline binaries (VK_KHR_pipeline_binary; NX2NAVI_PIPE_BINARIES, default 0 while it is being tested; 1 on)
|
||||||
|
// pipeline-cache.bin does not carry compiled pipelines from one executable to another on NVIDIA 621.01: two
|
||||||
|
// byte-identical exes under different names, every compute module hash-identical, and the second compiled 317 of the
|
||||||
|
// 323 compute pipelines the first had read from the file (2026-11-09). A pipeline binary made from one exe's pipeline
|
||||||
|
// creates the same pipeline in another in 0.5 ms instead of 26 ms. So where the driver has the extension, every
|
||||||
|
// pipeline, stage library and compute pipeline is created through pipe_create: its driver pipeline key
|
||||||
|
// (vkGetPipelineKeyKHR, computed from the create info) is looked up in pipeline-binaries.bin next to nx2navi.exe and
|
||||||
|
// the pipeline is made from the stored binaries; a key not there is compiled with its data captured, and its binaries
|
||||||
|
// are appended to the file (autosave, every 2 s). pipeline-cache.bin is then neither read nor written: the driver's
|
||||||
|
// own disk cache serves an unchanged exe, and the binaries serve a renamed or updated one.
|
||||||
|
// Each binary is stored once under its own binary key: on NVIDIA 621.01 nearly every graphics pipeline has, besides
|
||||||
|
// its own binaries of 20-200 KB, one 2 MiB binary that is byte-identical across pipelines (2,506 copies in a 4.7 GB
|
||||||
|
// first-version file).
|
||||||
|
// File: header {"NX2NPBIN", version, the driver's global key size and bytes} (another global key: another driver or
|
||||||
|
// GPU, so the file starts empty); then records {u32 bytes after this field, u32 type, u32 key size, 32 key bytes,
|
||||||
|
// payload}: type PB_BLOB, a binary under its binary key, payload its data; type PB_PIPE, a pipeline under its pipeline
|
||||||
|
// key, payload {u32 count, count x {u32 key size, 32 key bytes}} naming its binaries. Only an index of the records
|
||||||
|
// stays in memory; a pipeline's records are read from the file when it is created. NX2NAVI_PIPE_BINARIES_MAX_MB
|
||||||
|
// (default 1024, 0 no limit): a larger file starts empty.
|
||||||
|
#define PB_VERSION 2
|
||||||
|
// 64-bit file offsets: long is 32 bits on Windows, and a file over 2 GB read as cut short.
|
||||||
|
#ifdef _WIN32
|
||||||
|
#define pb_seek _fseeki64
|
||||||
|
#define pb_tell _ftelli64
|
||||||
|
#else
|
||||||
|
#define pb_seek fseeko
|
||||||
|
#define pb_tell ftello
|
||||||
|
#endif
|
||||||
|
#define PB_MAX_BIN 8
|
||||||
|
#define PB_KEY VK_MAX_PIPELINE_BINARY_KEY_SIZE_KHR
|
||||||
|
enum { PB_BLOB = 1, PB_PIPE = 2 };
|
||||||
|
#define PB_HEAD (4 + 4 + PB_KEY) // a record's bytes after its size field and before its payload
|
||||||
|
typedef struct { char magic[8]; uint32_t version, gkey_size; uint8_t gkey[PB_KEY]; } PbHeader;
|
||||||
|
typedef struct { uint64_t h, off; uint32_t size; } PbSlot; // off: file offset of the record's size field
|
||||||
|
static struct {
|
||||||
|
int on;
|
||||||
|
PFN_vkCreatePipelineBinariesKHR create;
|
||||||
|
PFN_vkDestroyPipelineBinaryKHR destroy;
|
||||||
|
PFN_vkGetPipelineKeyKHR key;
|
||||||
|
PFN_vkGetPipelineBinaryDataKHR data;
|
||||||
|
PFN_vkReleaseCapturedPipelineDataKHR release;
|
||||||
|
char path[PLAT_PATH_MAX * 2 + 32];
|
||||||
|
PlatMutex lock; // the index, the pending records and rf
|
||||||
|
FILE *rf; // reads records
|
||||||
|
PbSlot *slot;
|
||||||
|
uint32_t cap, n, pipes, blobs;
|
||||||
|
uint8_t *pend; // records not written yet; they follow the file's end in this order
|
||||||
|
size_t pend_n, pend_cap;
|
||||||
|
uint64_t file_size; // bytes on disk
|
||||||
|
int no_capture[3]; // kind (pipe_create) whose create refused the capture flag
|
||||||
|
uint64_t from_file, load_fail, compiled, captured, capture_fail, blob_new, blob_shared;
|
||||||
|
double from_ms, compile_ms;
|
||||||
|
} PB = {.lock = PLAT_MUTEX_INIT};
|
||||||
|
static __thread int pb_last; // 1: the last pipe_create on this thread made its pipeline from binaries
|
||||||
|
|
||||||
|
static uint64_t pb_hash(uint32_t type, const uint8_t *key, uint32_t n) {
|
||||||
|
uint64_t h = fnv(fnv(FNV0, &type, 4), key, n);
|
||||||
|
return h ? h : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static PbSlot *pb_find(uint64_t h) { // caller holds PB.lock
|
||||||
|
if (!PB.cap) return NULL;
|
||||||
|
for (uint32_t j = (uint32_t)h & (PB.cap - 1);; j = (j + 1) & (PB.cap - 1)) {
|
||||||
|
if (!PB.slot[j].h) return NULL;
|
||||||
|
if (PB.slot[j].h == h) return &PB.slot[j];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enters a record (a hash match is taken as the key: 64 bits). 0 when one with the key is there already or out of
|
||||||
|
// memory. Caller holds PB.lock.
|
||||||
|
static int pb_insert(uint64_t h, uint64_t off, uint32_t size) {
|
||||||
|
if ((PB.n + 1) * 2 > PB.cap) {
|
||||||
|
uint32_t cap = PB.cap ? PB.cap * 2 : 4096;
|
||||||
|
PbSlot *t = calloc(cap, sizeof *t);
|
||||||
|
if (!t) return 0;
|
||||||
|
for (uint32_t i = 0; i < PB.cap; i++) {
|
||||||
|
if (!PB.slot[i].h) continue;
|
||||||
|
uint32_t j = (uint32_t)PB.slot[i].h & (cap - 1);
|
||||||
|
while (t[j].h) j = (j + 1) & (cap - 1);
|
||||||
|
t[j] = PB.slot[i];
|
||||||
|
}
|
||||||
|
free(PB.slot);
|
||||||
|
PB.slot = t, PB.cap = cap;
|
||||||
|
}
|
||||||
|
if (pb_find(h)) return 0;
|
||||||
|
uint32_t j = (uint32_t)h & (PB.cap - 1);
|
||||||
|
while (PB.slot[j].h) j = (j + 1) & (PB.cap - 1);
|
||||||
|
PB.slot[j] = (PbSlot){h, off, size};
|
||||||
|
PB.n++;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void pb_rewrite(const PbHeader *hd) { // an empty file with this driver's header
|
||||||
|
FILE *f = fopen(PB.path, "wb");
|
||||||
|
if (f) fwrite(hd, sizeof *hd, 1, f), fclose(f);
|
||||||
|
PB.file_size = f ? sizeof *hd : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Opens pipeline-binaries.bin and indexes its records (cache_load, before P.cache).
|
||||||
|
static void pb_init(VkCtx *c) {
|
||||||
|
if (!c->has_pipeline_binary || !P.cache_path[0]) return;
|
||||||
|
#define PBPROC(field, name) (PB.field = (void *)vk_dpa(c->device, name))
|
||||||
|
if (!PBPROC(create, "vkCreatePipelineBinariesKHR") || !PBPROC(destroy, "vkDestroyPipelineBinaryKHR") ||
|
||||||
|
!PBPROC(key, "vkGetPipelineKeyKHR") || !PBPROC(data, "vkGetPipelineBinaryDataKHR") ||
|
||||||
|
!PBPROC(release, "vkReleaseCapturedPipelineDataKHR")) {
|
||||||
|
rt_log("vk_pipeline: pipeline binaries: the driver lacks a VK_KHR_pipeline_binary function; using pipeline-cache.bin");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
#undef PBPROC
|
||||||
|
PbHeader want = {{'N', 'X', '2', 'N', 'P', 'B', 'I', 'N'}, PB_VERSION, 0, {0}};
|
||||||
|
VkPipelineBinaryKeyKHR gk = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
||||||
|
if (PB.key(c->device, NULL, &gk) != VK_SUCCESS || !gk.keySize || gk.keySize > PB_KEY) {
|
||||||
|
rt_log("vk_pipeline: pipeline binaries: no global key from the driver; using pipeline-cache.bin");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
want.gkey_size = gk.keySize;
|
||||||
|
memcpy(want.gkey, gk.key, gk.keySize);
|
||||||
|
snprintf(PB.path, sizeof PB.path, "%s", P.cache_path);
|
||||||
|
char *cut = plat_last_sep(PB.path);
|
||||||
|
snprintf(cut ? cut + 1 : PB.path, sizeof PB.path - (size_t)(cut ? cut + 1 - PB.path : 0), "pipeline-binaries.bin");
|
||||||
|
uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPE_BINARIES_MAX_MB"), NULL, 10) << 20;
|
||||||
|
FILE *f = fopen(PB.path, "rb");
|
||||||
|
PbHeader hd;
|
||||||
|
const char *why = NULL;
|
||||||
|
uint64_t good = 0, len = 0;
|
||||||
|
if (f) {
|
||||||
|
pb_seek(f, 0, SEEK_END);
|
||||||
|
len = (uint64_t)pb_tell(f);
|
||||||
|
pb_seek(f, 0, SEEK_SET);
|
||||||
|
if (fread(&hd, sizeof hd, 1, f) != 1 || memcmp(&hd, &want, sizeof hd)) why = "written by another driver, GPU or version";
|
||||||
|
else if (cap && len > cap) why = "over NX2NAVI_PIPE_BINARIES_MAX_MB";
|
||||||
|
else {
|
||||||
|
good = sizeof hd;
|
||||||
|
for (;;) { // index: per record its type, key and offset
|
||||||
|
uint32_t head[3];
|
||||||
|
uint8_t key[PB_KEY];
|
||||||
|
if (fread(head, sizeof head, 1, f) != 1 || head[2] > PB_KEY || head[0] < PB_HEAD ||
|
||||||
|
(head[1] != PB_BLOB && head[1] != PB_PIPE) || fread(key, PB_KEY, 1, f) != 1 || good + 4 + head[0] > len)
|
||||||
|
break;
|
||||||
|
if (pb_insert(pb_hash(head[1], key, head[2]), good, head[0])) {
|
||||||
|
if (head[1] == PB_PIPE) PB.pipes++;
|
||||||
|
else PB.blobs++;
|
||||||
|
}
|
||||||
|
good += 4 + (uint64_t)head[0];
|
||||||
|
if (pb_seek(f, (int64_t)good, SEEK_SET)) break;
|
||||||
|
}
|
||||||
|
if (good != len) why = "cut short (a record half written)";
|
||||||
|
}
|
||||||
|
fclose(f);
|
||||||
|
}
|
||||||
|
if (why || !f) { // start empty: the index is dropped and the file rewritten
|
||||||
|
if (f) rt_log("vk_pipeline: pipeline binaries: %s %s (%.0f MB); starting empty", PB.path, why, len / 1048576.0);
|
||||||
|
free(PB.slot), PB.slot = NULL, PB.cap = PB.n = PB.pipes = PB.blobs = 0;
|
||||||
|
pb_rewrite(&want);
|
||||||
|
} else {
|
||||||
|
PB.file_size = len;
|
||||||
|
}
|
||||||
|
PB.rf = fopen(PB.path, "rb");
|
||||||
|
PB.on = PB.rf != NULL && PB.file_size >= sizeof want;
|
||||||
|
rt_log("vk_pipeline: pipeline binaries %s: %s, %u pipelines, %u binaries, %.1f MB (NX2NAVI_PIPE_BINARIES=0 goes back "
|
||||||
|
"to pipeline-cache.bin)", PB.on ? "on" : "off (cannot open the file)", PB.path, PB.pipes, PB.blobs,
|
||||||
|
PB.file_size / 1048576.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The payload of the record of (type, key), malloc'd (*size bytes), or NULL. Caller holds PB.lock.
|
||||||
|
static uint8_t *pb_read(uint32_t type, const uint8_t *key, uint32_t ks, uint32_t *size) {
|
||||||
|
PbSlot *s = pb_find(pb_hash(type, key, ks));
|
||||||
|
if (!s || s->size < PB_HEAD) return NULL;
|
||||||
|
uint32_t n = s->size - PB_HEAD;
|
||||||
|
uint8_t *head = malloc(PB_HEAD + (size_t)n), *out = NULL;
|
||||||
|
if (!head) return NULL;
|
||||||
|
int ok;
|
||||||
|
if (s->off >= PB.file_size) { // still pending, at its place in pend
|
||||||
|
size_t at = (size_t)(s->off - PB.file_size) + 4;
|
||||||
|
ok = at + s->size <= PB.pend_n;
|
||||||
|
if (ok) memcpy(head, PB.pend + at, s->size);
|
||||||
|
} else {
|
||||||
|
ok = !pb_seek(PB.rf, (int64_t)(s->off + 4), SEEK_SET) && fread(head, s->size, 1, PB.rf) == 1;
|
||||||
|
}
|
||||||
|
uint32_t t, k;
|
||||||
|
if (ok) memcpy(&t, head, 4), memcpy(&k, head + 4, 4);
|
||||||
|
if (ok && t == type && k == ks && !memcmp(head + 8, key, ks)) { // the key itself, not only its hash
|
||||||
|
memmove(head, head + PB_HEAD, n);
|
||||||
|
out = head, *size = n;
|
||||||
|
} else {
|
||||||
|
free(head);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The binaries stored for pipeline key k, created in bins (*n of them). 0 when none are stored or they fail.
|
||||||
|
static int pb_load(const VkPipelineBinaryKeyKHR *k, VkPipelineBinaryKHR bins[PB_MAX_BIN], uint32_t *n) {
|
||||||
|
VkPipelineBinaryKeyKHR keys[PB_MAX_BIN];
|
||||||
|
VkPipelineBinaryDataKHR data[PB_MAX_BIN];
|
||||||
|
uint8_t *blob[PB_MAX_BIN] = {0};
|
||||||
|
uint32_t nb = 0, size = 0;
|
||||||
|
plat_mutex_lock(&PB.lock);
|
||||||
|
uint8_t *pipe = pb_read(PB_PIPE, k->key, k->keySize, &size);
|
||||||
|
int ok = pipe && size >= 4;
|
||||||
|
if (ok) memcpy(&nb, pipe, 4);
|
||||||
|
if (!nb || nb > PB_MAX_BIN || size < 4 + nb * (4 + PB_KEY)) ok = 0;
|
||||||
|
for (uint32_t i = 0; ok && i < nb; i++) {
|
||||||
|
const uint8_t *e = pipe + 4 + i * (4 + PB_KEY);
|
||||||
|
uint32_t ks, ds = 0;
|
||||||
|
memcpy(&ks, e, 4);
|
||||||
|
keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR, .keySize = ks};
|
||||||
|
memcpy(keys[i].key, e + 4, PB_KEY);
|
||||||
|
if (ks > PB_KEY || !(blob[i] = pb_read(PB_BLOB, keys[i].key, ks, &ds))) ok = 0;
|
||||||
|
else data[i] = (VkPipelineBinaryDataKHR){.dataSize = ds, .pData = blob[i]};
|
||||||
|
}
|
||||||
|
plat_mutex_unlock(&PB.lock);
|
||||||
|
if (ok) {
|
||||||
|
VkPipelineBinaryKeysAndDataKHR kd = {.binaryCount = nb, .pPipelineBinaryKeys = keys, .pPipelineBinaryData = data};
|
||||||
|
VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pKeysAndDataInfo = &kd};
|
||||||
|
VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR,
|
||||||
|
.pipelineBinaryCount = nb, .pPipelineBinaries = bins};
|
||||||
|
ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS;
|
||||||
|
*n = nb;
|
||||||
|
}
|
||||||
|
for (uint32_t i = 0; i < PB_MAX_BIN; i++) free(blob[i]);
|
||||||
|
free(pipe);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Appends one record to pend and the index. Caller holds PB.lock.
|
||||||
|
static int pb_put(uint32_t type, const uint8_t *key, uint32_t ks, const void *a, size_t an, const void *b, size_t bn) {
|
||||||
|
size_t size = PB_HEAD + an + bn;
|
||||||
|
if (size >= (1u << 31)) return 0;
|
||||||
|
if (PB.pend_n + 4 + size > PB.pend_cap) {
|
||||||
|
size_t cap = PB.pend_cap ? PB.pend_cap * 2 : (size_t)4 << 20;
|
||||||
|
while (cap < PB.pend_n + 4 + size) cap *= 2;
|
||||||
|
uint8_t *t = realloc(PB.pend, cap);
|
||||||
|
if (!t) return 0;
|
||||||
|
PB.pend = t, PB.pend_cap = cap;
|
||||||
|
}
|
||||||
|
if (!pb_insert(pb_hash(type, key, ks), PB.file_size + PB.pend_n, (uint32_t)size)) return 0;
|
||||||
|
uint8_t *o = PB.pend + PB.pend_n;
|
||||||
|
uint32_t s32 = (uint32_t)size;
|
||||||
|
memcpy(o, &s32, 4), memcpy(o + 4, &type, 4), memcpy(o + 8, &ks, 4);
|
||||||
|
memset(o + 12, 0, PB_KEY), memcpy(o + 12, key, ks);
|
||||||
|
if (an) memcpy(o + 4 + PB_HEAD, a, an);
|
||||||
|
if (bn) memcpy(o + 4 + PB_HEAD + an, b, bn);
|
||||||
|
PB.pend_n += 4 + size;
|
||||||
|
if (type == PB_PIPE) PB.pipes++;
|
||||||
|
else PB.blobs++;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void pb_flush(void);
|
||||||
|
|
||||||
|
// Stores the binaries of pipeline p (created with its data captured) under pipeline key k, then frees the data. A
|
||||||
|
// binary whose key is stored already is not stored again.
|
||||||
|
static void pb_capture(VkPipeline p, const VkPipelineBinaryKeyKHR *k) {
|
||||||
|
VkPipelineBinaryCreateInfoKHR ci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_CREATE_INFO_KHR, .pipeline = p};
|
||||||
|
VkPipelineBinaryKHR bins[PB_MAX_BIN];
|
||||||
|
VkPipelineBinaryHandlesInfoKHR hi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_HANDLES_INFO_KHR};
|
||||||
|
uint32_t nb = 0;
|
||||||
|
int ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS && hi.pipelineBinaryCount &&
|
||||||
|
hi.pipelineBinaryCount <= PB_MAX_BIN;
|
||||||
|
if (ok) {
|
||||||
|
hi.pPipelineBinaries = bins;
|
||||||
|
ok = PB.create(P.device, &ci, NULL, &hi) == VK_SUCCESS;
|
||||||
|
nb = ok ? hi.pipelineBinaryCount : 0;
|
||||||
|
}
|
||||||
|
VkPipelineBinaryKeyKHR keys[PB_MAX_BIN];
|
||||||
|
uint8_t list[4 + PB_MAX_BIN * (4 + PB_KEY)] = {0};
|
||||||
|
memcpy(list, &nb, 4);
|
||||||
|
for (uint32_t i = 0; ok && i < nb; i++) { // each binary's key, and its data unless stored already
|
||||||
|
VkPipelineBinaryDataInfoKHR di = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_DATA_INFO_KHR, .pipelineBinary = bins[i]};
|
||||||
|
keys[i] = (VkPipelineBinaryKeyKHR){.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
||||||
|
size_t ds = 0;
|
||||||
|
ok = PB.data(P.device, &di, &keys[i], &ds, NULL) == VK_SUCCESS && keys[i].keySize && keys[i].keySize <= PB_KEY;
|
||||||
|
if (!ok) break;
|
||||||
|
memcpy(list + 4 + i * (4 + PB_KEY), &keys[i].keySize, 4), memcpy(list + 8 + i * (4 + PB_KEY), keys[i].key, keys[i].keySize);
|
||||||
|
plat_mutex_lock(&PB.lock);
|
||||||
|
int have = pb_find(pb_hash(PB_BLOB, keys[i].key, keys[i].keySize)) != NULL;
|
||||||
|
plat_mutex_unlock(&PB.lock);
|
||||||
|
if (have) {
|
||||||
|
PB.blob_shared++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
uint8_t *d = malloc(ds ? ds : 1);
|
||||||
|
ok = d && PB.data(P.device, &di, &keys[i], &ds, d) == VK_SUCCESS;
|
||||||
|
if (ok) {
|
||||||
|
plat_mutex_lock(&PB.lock);
|
||||||
|
if (pb_put(PB_BLOB, keys[i].key, keys[i].keySize, d, ds, NULL, 0)) PB.blob_new++;
|
||||||
|
plat_mutex_unlock(&PB.lock); // a 0 from pb_put: another thread stored the same binary meanwhile
|
||||||
|
}
|
||||||
|
free(d);
|
||||||
|
}
|
||||||
|
for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL);
|
||||||
|
VkReleaseCapturedPipelineDataInfoKHR ri = {.sType = VK_STRUCTURE_TYPE_RELEASE_CAPTURED_PIPELINE_DATA_INFO_KHR, .pipeline = p};
|
||||||
|
PB.release(P.device, &ri, NULL);
|
||||||
|
plat_mutex_lock(&PB.lock);
|
||||||
|
if (ok && pb_put(PB_PIPE, k->key, k->keySize, list, 4 + nb * (4 + PB_KEY), NULL, 0)) PB.captured++;
|
||||||
|
else if (!ok) PB.capture_fail++;
|
||||||
|
int full = PB.pend_n >= ((size_t)64 << 20);
|
||||||
|
plat_mutex_unlock(&PB.lock);
|
||||||
|
if (full) pb_flush(); // a cold boot captures thousands before the autosave thread runs: keep the buffer small
|
||||||
|
}
|
||||||
|
|
||||||
|
// Appends the pending records (the autosave thread every 2 s, and the save at exit).
|
||||||
|
static void pb_flush(void) {
|
||||||
|
if (!PB.on) return;
|
||||||
|
plat_mutex_lock(&PB.lock);
|
||||||
|
if (PB.pend_n) {
|
||||||
|
FILE *f = fopen(PB.path, "ab");
|
||||||
|
int ok = f && fwrite(PB.pend, PB.pend_n, 1, f) == 1;
|
||||||
|
if (f) ok = !fclose(f) && ok;
|
||||||
|
if (ok) PB.file_size += PB.pend_n;
|
||||||
|
else rt_log("vk_pipeline: pipeline binaries: cannot append to %s; this run's binaries stay in memory", PB.path);
|
||||||
|
if (ok) PB.pend_n = 0;
|
||||||
|
}
|
||||||
|
plat_mutex_unlock(&PB.lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void pb_log(const char *when) {
|
||||||
|
if (!PB.on) return;
|
||||||
|
plat_mutex_lock(&PB.lock);
|
||||||
|
rt_log("vk_pipeline: pipeline binaries %s: %llu made from pipeline-binaries.bin (avg %.2f ms), %llu compiled (avg %.1f "
|
||||||
|
"ms), %llu captured, %llu could not be stored, %llu stored ones failed; binaries %llu new, %llu shared; %u pipelines and "
|
||||||
|
"%u binaries in the file, %.1f MB", when,
|
||||||
|
(unsigned long long)PB.from_file, PB.from_file ? PB.from_ms / PB.from_file : 0.0,
|
||||||
|
(unsigned long long)PB.compiled, PB.compiled ? PB.compile_ms / PB.compiled : 0.0,
|
||||||
|
(unsigned long long)PB.captured, (unsigned long long)PB.capture_fail, (unsigned long long)PB.load_fail,
|
||||||
|
(unsigned long long)PB.blob_new, (unsigned long long)PB.blob_shared, PB.pipes, PB.blobs,
|
||||||
|
(PB.file_size + PB.pend_n) / 1048576.0);
|
||||||
|
plat_mutex_unlock(&PB.lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Creates one pipeline: kind 0 a graphics pipeline, 1 a graphics pipeline library, 2 a compute pipeline; ci its
|
||||||
|
// Vk*PipelineCreateInfo. With binaries on: from the stored binaries of its pipeline key when there are any, else
|
||||||
|
// compiled with P.cache and its binaries captured. The create info is restored before returning.
|
||||||
|
typedef struct { VkStructureType sType; const void *pNext; VkPipelineCreateFlags flags; } PipeCiHead;
|
||||||
|
static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) {
|
||||||
|
PipeCiHead *hd = ci;
|
||||||
|
VkResult r;
|
||||||
|
pb_last = 0;
|
||||||
|
#define PIPE_CREATE(cache) (kind == 2 ? vkCreateComputePipelines(P.device, cache, 1, ci, NULL, pipe) \
|
||||||
|
: vkCreateGraphicsPipelines(P.device, cache, 1, ci, NULL, pipe))
|
||||||
|
if (!PB.on) return PIPE_CREATE(P.cache);
|
||||||
|
VkPipelineCreateInfoKHR kci = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_INFO_KHR, .pNext = ci};
|
||||||
|
VkPipelineBinaryKeyKHR key = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_KEY_KHR};
|
||||||
|
int keyed = PB.key(P.device, &kci, &key) == VK_SUCCESS && key.keySize && key.keySize <= PB_KEY;
|
||||||
|
const void *next = hd->pNext;
|
||||||
|
if (keyed) {
|
||||||
|
VkPipelineBinaryKHR bins[PB_MAX_BIN];
|
||||||
|
uint32_t nb = 0;
|
||||||
|
uint64_t t0 = plat_now_ns();
|
||||||
|
if (pb_load(&key, bins, &nb)) {
|
||||||
|
VkPipelineBinaryInfoKHR bi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_BINARY_INFO_KHR, .pNext = next,
|
||||||
|
.binaryCount = nb, .pPipelineBinaries = bins};
|
||||||
|
hd->pNext = &bi;
|
||||||
|
r = PIPE_CREATE(VK_NULL_HANDLE); // a pipeline made from binaries takes no pipeline cache
|
||||||
|
hd->pNext = next;
|
||||||
|
for (uint32_t i = 0; i < nb; i++) PB.destroy(P.device, bins[i], NULL);
|
||||||
|
plat_mutex_lock(&PB.lock);
|
||||||
|
if (r == VK_SUCCESS) PB.from_file++, PB.from_ms += (plat_now_ns() - t0) / 1e6;
|
||||||
|
else PB.load_fail++;
|
||||||
|
plat_mutex_unlock(&PB.lock);
|
||||||
|
if (r == VK_SUCCESS) return pb_last = 1, r;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int capture = keyed && !PB.no_capture[kind];
|
||||||
|
VkPipelineCreateFlags2CreateInfoKHR f2 = {.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR, .pNext = next,
|
||||||
|
.flags = (VkPipelineCreateFlags2KHR)hd->flags | VK_PIPELINE_CREATE_2_CAPTURE_DATA_BIT_KHR};
|
||||||
|
uint64_t t0 = plat_now_ns();
|
||||||
|
if (capture) hd->pNext = &f2;
|
||||||
|
r = PIPE_CREATE(P.cache);
|
||||||
|
hd->pNext = next;
|
||||||
|
if (capture && r < 0) { // the driver refuses the capture for this kind: create without it from now on
|
||||||
|
PB.no_capture[kind] = 1;
|
||||||
|
rt_log("vk_pipeline: pipeline binaries: a %s with its data captured failed (%d); such pipelines are not stored",
|
||||||
|
kind == 2 ? "compute pipeline" : kind ? "pipeline library" : "graphics pipeline", (int)r);
|
||||||
|
capture = 0;
|
||||||
|
r = PIPE_CREATE(P.cache);
|
||||||
|
}
|
||||||
|
#undef PIPE_CREATE
|
||||||
|
if (r == VK_SUCCESS && *pipe) {
|
||||||
|
plat_mutex_lock(&PB.lock);
|
||||||
|
PB.compiled++, PB.compile_ms += (plat_now_ns() - t0) / 1e6;
|
||||||
|
plat_mutex_unlock(&PB.lock);
|
||||||
|
if (capture) pb_capture(*pipe, &key);
|
||||||
|
}
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
// Cache data whose header names another device or driver is dropped before Vulkan sees it.
|
// Cache data whose header names another device or driver is dropped before Vulkan sees it.
|
||||||
static void cache_load(VkCtx *c) {
|
static void cache_load(VkCtx *c) {
|
||||||
cache_path_init();
|
cache_path_init();
|
||||||
|
pb_init(c);
|
||||||
|
if (PB.on) { // the binaries carry compiled pipelines across runs; P.cache stays VK_NULL_HANDLE
|
||||||
|
rt_log("vk_pipeline: %s is not read or written while pipeline binaries are on", P.cache_path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
void *data = NULL;
|
void *data = NULL;
|
||||||
size_t size = 0;
|
size_t size = 0;
|
||||||
FILE *f = P.cache_path[0] ? fopen(P.cache_path, "rb") : NULL;
|
FILE *f = P.cache_path[0] ? fopen(P.cache_path, "rb") : NULL;
|
||||||
@@ -1442,7 +1834,7 @@ static const void *fb_chain(VkPipelineCreationFeedbackCreateInfo *ci, VkPipeline
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void fb_note(int kind, const VkPipelineCreationFeedback *fb) {
|
static void fb_note(int kind, const VkPipelineCreationFeedback *fb) {
|
||||||
if (!P.feedback || !(fb->flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) return;
|
if (!P.feedback || !P.cache || !(fb->flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) return;
|
||||||
int hit = (fb->flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0;
|
int hit = (fb->flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0;
|
||||||
double ms = (double)fb->duration / 1e6;
|
double ms = (double)fb->duration / 1e6;
|
||||||
plat_mutex_lock(&FB.lock);
|
plat_mutex_lock(&FB.lock);
|
||||||
@@ -1454,7 +1846,8 @@ static void fb_note(int kind, const VkPipelineCreationFeedback *fb) {
|
|||||||
|
|
||||||
// One line: per kind, builds that came from pipeline-cache.bin of all builds so far, with the average time of each.
|
// One line: per kind, builds that came from pipeline-cache.bin of all builds so far, with the average time of each.
|
||||||
static void fb_log(const char *when) {
|
static void fb_log(const char *when) {
|
||||||
if (!P.feedback) return;
|
pb_log(when);
|
||||||
|
if (!P.feedback || !P.cache) return;
|
||||||
static const char *const kn[FB_KINDS] = {"pipelines", "libraries", "compute"};
|
static const char *const kn[FB_KINDS] = {"pipelines", "libraries", "compute"};
|
||||||
char line[512];
|
char line[512];
|
||||||
int o = snprintf(line, sizeof line, "vk_pipeline: pipeline-cache.bin hits %s:", when);
|
int o = snprintf(line, sizeof line, "vk_pipeline: pipeline-cache.bin hits %s:", when);
|
||||||
@@ -1718,7 +2111,7 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay
|
|||||||
pi.pNext = fb_chain(&fci, &fb, pi.pNext);
|
pi.pNext = fb_chain(&fci, &fb, pi.pNext);
|
||||||
uint64_t p0 = perf_begin();
|
uint64_t p0 = perf_begin();
|
||||||
crash_note("a graphics pipeline", s->vs, s->fs, s->ts);
|
crash_note("a graphics pipeline", s->vs, s->fs, s->ts);
|
||||||
VkResult r = vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe);
|
VkResult r = pipe_create(0, &pi, &pipe);
|
||||||
plat_crash_context(NULL);
|
plat_crash_context(NULL);
|
||||||
perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it
|
perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it
|
||||||
if (r != VK_SUCCESS) {
|
if (r != VK_SUCCESS) {
|
||||||
@@ -1877,7 +2270,7 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi
|
|||||||
VkPipelineCreationFeedback fb;
|
VkPipelineCreationFeedback fb;
|
||||||
pi.pNext = fb_chain(&fci, &fb, pi.pNext);
|
pi.pNext = fb_chain(&fci, &fb, pi.pNext);
|
||||||
crash_note("a pipeline library", s->vs, s->fs, NULL);
|
crash_note("a pipeline library", s->vs, s->fs, NULL);
|
||||||
if (vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
if (pipe_create(1, &pi, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
||||||
plat_crash_context(NULL);
|
plat_crash_context(NULL);
|
||||||
if (pipe && (kind == LIB_PR || kind == LIB_FS)) fb_note(FB_LIBRARY, &fb); // the shader libraries: the others compile nothing
|
if (pipe && (kind == LIB_PR || kind == LIB_FS)) fb_note(FB_LIBRARY, &fb); // the shader libraries: the others compile nothing
|
||||||
out:
|
out:
|
||||||
@@ -3098,13 +3491,13 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache
|
|||||||
VkPipelineCreationFeedback fb;
|
VkPipelineCreationFeedback fb;
|
||||||
ci.pNext = fb_chain(&fci, &fb, ci.pNext);
|
ci.pNext = fb_chain(&fci, &fb, ci.pNext);
|
||||||
crash_note("a compute pipeline", cs, NULL, NULL);
|
crash_note("a compute pipeline", cs, NULL, NULL);
|
||||||
if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
if (pipe_create(2, &ci, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE;
|
||||||
plat_crash_context(NULL);
|
plat_crash_context(NULL);
|
||||||
if (pipe) fb_note(FB_COMPUTE, &fb);
|
if (pipe) fb_note(FB_COMPUTE, &fb);
|
||||||
if (P.feedback >= 2 && (pipe || !cached_only))
|
if (P.feedback >= 2 && (pipe || !cached_only))
|
||||||
rt_log("vk_pipeline: compute %016llx: %llu words, module %016llx, %s, %.1f ms", (unsigned long long)fnv(FNV0, code, words * 4),
|
rt_log("vk_pipeline: compute %016llx: %llu words, module %016llx, %s, %.1f ms", (unsigned long long)fnv(FNV0, code, words * 4),
|
||||||
(unsigned long long)words, (unsigned long long)mm_hash,
|
(unsigned long long)words, (unsigned long long)mm_hash,
|
||||||
!pipe ? "failed" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled",
|
!pipe ? "failed" : pb_last ? "from pipeline-binaries.bin" : fb.flags & VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT ? "from pipeline-cache.bin" : "compiled",
|
||||||
(double)fb.duration / 1e6);
|
(double)fb.duration / 1e6);
|
||||||
vkDestroyShaderModule(P.device, m, NULL);
|
vkDestroyShaderModule(P.device, m, NULL);
|
||||||
free(over);
|
free(over);
|
||||||
@@ -4114,7 +4507,7 @@ void vk_pipeline_stats(VkPipeStats *s) {
|
|||||||
static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache
|
static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shutdown takes it to destroy the cache
|
||||||
|
|
||||||
void vk_pipeline_save(void) {
|
void vk_pipeline_save(void) {
|
||||||
if (P.ready) keys_flush();
|
if (P.ready) keys_flush(), pb_flush();
|
||||||
if (P.ready && P.gpl) {
|
if (P.ready && P.gpl) {
|
||||||
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws "
|
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers; avg %.2f ms, max %.2f ms, %llu failed), %llu draws "
|
||||||
"lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued "
|
"lookups found shader libraries missing (left to a worker), %llu optimised pipelines swapped in (%llu queued "
|
||||||
@@ -4140,6 +4533,7 @@ void vk_pipeline_save(void) {
|
|||||||
(unsigned long long)P.g.pre_used[LIB_FS], (unsigned long long)LB.cs_draw);
|
(unsigned long long)P.g.pre_used[LIB_FS], (unsigned long long)LB.cs_draw);
|
||||||
}
|
}
|
||||||
gfx_shader_record_flush(1);
|
gfx_shader_record_flush(1);
|
||||||
|
if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below
|
||||||
if (!P.ready || !P.cache || !P.cache_path[0]) return;
|
if (!P.ready || !P.cache || !P.cache_path[0]) return;
|
||||||
// Not under P.lock: the driver synchronises P.cache, and holding P.lock through the copy and the file write (about
|
// Not under P.lock: the driver synchronises P.cache, and holding P.lock through the copy and the file write (about
|
||||||
// 60 ms on the 1.4.3 fight) stopped every pipeline lookup of the replay, a freeze at each 20 s autosave.
|
// 60 ms on the 1.4.3 fight) stopped every pipeline lookup of the replay, a freeze at each 20 s autosave.
|
||||||
@@ -4170,7 +4564,11 @@ static void autosave_once(void) {
|
|||||||
static uint64_t last_n, last_ms, last_keys;
|
static uint64_t last_n, last_ms, last_keys;
|
||||||
if (!P.ready) return;
|
if (!P.ready) return;
|
||||||
uint64_t now = plat_now_ms();
|
uint64_t now = plat_now_ms();
|
||||||
if (*(volatile uint32_t *)&K.pend_n && now - last_keys >= 2000) last_keys = now, keys_flush();
|
if (now - last_keys >= 2000) {
|
||||||
|
last_keys = now;
|
||||||
|
if (*(volatile uint32_t *)&K.pend_n) keys_flush();
|
||||||
|
pb_flush();
|
||||||
|
}
|
||||||
gfx_shader_record_flush(0);
|
gfx_shader_record_flush(0);
|
||||||
if (now - last_ms < 20000) return;
|
if (now - last_ms < 20000) return;
|
||||||
VkPipeStats s;
|
VkPipeStats s;
|
||||||
|
|||||||
Reference in New Issue
Block a user