mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 13:59:56 -04:00
Merge the shader and pipeline batch: FSR 3.1's pipelines through the kept pipeline caches, a pipeline cache save tried again when it did not happen, and a tool that checks a shader cache's translated shaders offline.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -174,6 +174,12 @@ of them.
|
||||
`make_release.py`). A list belongs to the record layout of the runtime that wrote it; another runtime says so and
|
||||
builds nothing from it. `python tests\pipeline_list_test.py` checks the tool on made-up folders.
|
||||
|
||||
What a game folder's shader cache holds can be checked without the game:
|
||||
`python tools\validate_shader_cache.py <game folder>\shader-cache` reads every translated shader from the pack and
|
||||
the loose files, applies the checks the game makes when it loads one (those that fail are translated again at
|
||||
their next use) and runs `spirv-val` on the rest; `--every N` samples, `--out FILE` lists the failed ones. It writes
|
||||
nothing to the cache. `python tests\validate_shader_cache_test.py` checks the tool on a made-up cache.
|
||||
|
||||
7. Run. The game keeps running when its window loses the focus (`assume_focus = 1` in `nx2navi.ini`, the default; the
|
||||
Game tab switches it). With 0 it pauses, as it does on the Switch when Home is pressed.
|
||||
|
||||
@@ -405,6 +411,19 @@ value no machine has free. What the builders do when they are refused (`warm_bui
|
||||
`lw_worker`) shows in a run's log: start the game with `build_free_mb` set above the machine's free commit and read the
|
||||
`vk_pipeline:` lines that name the commit headroom.
|
||||
|
||||
A change to when the pipeline caches are saved in play (`runtime/vk_save_retry.h`, `runtime/vk_pipeline.c`
|
||||
`autosave_once`) passes `python tests\save_retry_check.py`, which builds the header alone: a save for new content, none
|
||||
without, and after a save that did not happen a bounded row of tries at the save interval.
|
||||
|
||||
A change to how FSR 3.1's pipelines reach the pipeline caches (`runtime/vk_ffx.cpp` `vk_ffx_proc_addr`,
|
||||
`runtime/vk_pipeline.c` `vk_pipeline_create_foreign`) passes `python tests\ffx_cache_check.py <game exe>`. It runs
|
||||
`--probe-ffx-cache` (one upscaling context on the device; no window, no game) twice on pipeline cache files of its own
|
||||
and once with `ffx_pipe_cache = 0`: the library's pipelines must come through the runtime's creates, the second run
|
||||
must compile none of them, and with the setting off none may come through. The executable needs
|
||||
`amd_fidelityfx_vk.dll` beside it; without it the probe and the test skip. The graphics driver's own cache can serve
|
||||
the first run too, so to see the files alone do it, run the probe from a copy of the executable under a name the
|
||||
driver has not seen: its first run compiles, its second does not.
|
||||
|
||||
A change to the start's memory warning (`runtime/mem_need.h`, `runtime/memory.c` `mem_start_note`) passes
|
||||
`python tests\mem_need_check.py [<game exe>]`, which builds the header alone and checks when the game's need counts
|
||||
as short. With the executable it also runs `--probe-memneed` (it prints the machine's commit and the answer and
|
||||
|
||||
@@ -180,6 +180,10 @@
|
||||
# pipeline_cache_play = 1
|
||||
# The size (MB) at which pipeline-cache-play.bin is folded into pipeline-cache.bin at a start.
|
||||
# pipeline_cache_play_mb = 64
|
||||
# 1: the pipelines FSR 3.1 makes for upscaling and frame generation are kept in the pipeline cache files, so a later
|
||||
# start does not compile them again. 0: FSR 3.1 creates them without those files, as the graphics driver alone keeps
|
||||
# them.
|
||||
# ffx_pipe_cache = 1
|
||||
# 1: where the driver has VK_KHR_pipeline_binary, compiled pipelines are kept in pipeline-binaries.bin under the
|
||||
# runtime's own keys, so a renamed or updated exe starts without compiling them again (pipeline-cache.bin is then
|
||||
# not used). Default 0 (pipeline-cache.bin only) while it is being tested.
|
||||
|
||||
@@ -46,7 +46,7 @@ static const char *const known[] = {
|
||||
"COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT",
|
||||
"DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLSS_IDLE_FREE", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
|
||||
"DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES",
|
||||
"FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES",
|
||||
"FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREE_CORES",
|
||||
"FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_BLOWOUT", "HDR_CONTRAST",
|
||||
"HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE",
|
||||
"HDR_HUE_CLIP", "HDR_LIVE", "HEAP_LAZY", "HDR_MATCH_SLOPE", "HDR_MIDGRAY",
|
||||
@@ -136,7 +136,7 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"COMPUTE_KEEP", "performance"},
|
||||
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
|
||||
{"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"PIPELINE_CACHE_PLAY", "performance"}, {"PIPELINE_CACHE_PLAY_MB", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"PIPELINE_CACHE_PLAY", "performance"}, {"PIPELINE_CACHE_PLAY_MB", "performance"}, {"FFX_PIPE_CACHE", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
|
||||
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
|
||||
{"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"},
|
||||
@@ -281,6 +281,7 @@ static const CfgMeta metas[] = {
|
||||
{"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "", "0|1", RESTART}, // vk_caps.c vk_caps_gpl: by the driver, off with AMD's for RDNA 2 and older
|
||||
{"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART},
|
||||
{"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART},
|
||||
{"FFX_PIPE_CACHE", "1", "0|1", RESTART}, // vk_ffx.cpp vk_ffx_proc_addr: FSR 3.1's pipelines through the pipeline caches
|
||||
{"PIPELINE_CACHE_PLAY", "1", "0|1", RESTART}, {"PIPELINE_CACHE_PLAY_MB", "64", "1..65536", RESTART}, // vk_pipeline.c "the play cache": what a run compiles goes into a file of its own
|
||||
{"PIPE_BINARIES", "0", "0|1", RESTART}, {"ONESHOT_WAIT_MS", "30000", "0..60000", RESTART}, {"ONESHOT_BUDGET", "1", "0|1", RESTART}, {"PIPE_BINARIES_MAX_MB", "1024", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART},
|
||||
{"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART},
|
||||
|
||||
@@ -444,6 +444,7 @@ int vk_pipeline_warm_shaders(void); // vk_pipeline.c: --w
|
||||
int vk_pipeline_prepare_pipelines(void); // vk_pipeline.c: --prepare-pipelines (the shared pipeline list)
|
||||
int vk_pipeline_probe_prewarm(void); // vk_pipeline.c: --probe-prewarm
|
||||
int vk_pipeline_probe_buildgate(void); // vk_pipeline.c: --probe-buildgate
|
||||
int vk_ffx_probe_cache(void); // vk_ffx.cpp: --probe-ffx-cache
|
||||
// vk_device.c: the drivers' own shader caches beside the exe (MESA_SHADER_CACHE_DIR, on NVIDIA's Linux driver
|
||||
// __GL_SHADER_DISK_CACHE_PATH and _SIZE) in the environment, once; before any thread can load a Vulkan driver
|
||||
void vk_driver_cache_env(void);
|
||||
|
||||
@@ -23,6 +23,8 @@
|
||||
// --probe-nvn, --probe-nvn-tex, --probe-window, --probe-shader, --probe-vkmem, --probe-vkexec graphics probes
|
||||
// --probe-prewarm the boot's pipeline stages from the caches next to the executable, the memory each leaves and
|
||||
// what a compute pipeline's first dispatch costs; skips without recorded shaders, runs no game
|
||||
// --probe-ffx-cache one FSR 3.1 upscaling context and where its pipelines came from (the pipeline caches kept
|
||||
// across runs); skips without amd_fidelityfx_vk.dll, runs no game
|
||||
// --probe-astc the compute ASTC decoder against astc_decode_ldr; needs no game files
|
||||
// --probe-zstd host zstd streams (ov_zstd.c) on frames made here; needs no game files
|
||||
// --probe-input gamepad polls on a virtual gamepad, also while SDL's update takes long; needs no game files
|
||||
@@ -534,7 +536,7 @@ static int usage(const char *argv0) {
|
||||
"--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | "
|
||||
"--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | "
|
||||
"--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | --probe-coop | "
|
||||
"--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-buildgate | --probe-memneed | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0);
|
||||
"--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-buildgate | --probe-ffx-cache | --probe-memneed | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0);
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -696,6 +698,7 @@ int main(int argc, char **argv) {
|
||||
if (!strcmp(mode, "--prepare-pipelines")) return vk_pipeline_prepare_pipelines();
|
||||
if (!strcmp(mode, "--probe-prewarm")) return vk_pipeline_probe_prewarm();
|
||||
if (!strcmp(mode, "--probe-buildgate")) return vk_pipeline_probe_buildgate();
|
||||
if (!strcmp(mode, "--probe-ffx-cache")) return vk_ffx_probe_cache();
|
||||
if (!strcmp(mode, "--probe-memneed")) { mem_start_note(1); return 0; }
|
||||
if (!strcmp(mode, "--boot")) { mem_start_note(0); cfg.boot = 1; rt_boot(); return 0; }
|
||||
return usage(argv[0]);
|
||||
|
||||
@@ -874,5 +874,13 @@ void vk_pipeline_autosave(void); // vk_pipeline_save at most every 20
|
||||
void vk_pipeline_shutdown(void); // waits for background builds, saves, destroys pipelines, layouts and the
|
||||
// cache; before vk_shutdown
|
||||
int vk_pipeline_selftest(void); // one graphics and one compute pipeline from census shaders; 0 on success
|
||||
// Pipelines a library creates on our device without a pipeline cache of its own (FSR 3.1, vk_ffx.cpp), through the
|
||||
// pipeline caches kept across runs; as vkCreateComputePipelines (compute 1) or vkCreateGraphicsPipelines with no cache.
|
||||
VkResult vk_pipeline_create_foreign(VkDevice dev, int compute, uint32_t n, const void *infos,
|
||||
const VkAllocationCallbacks *alloc, VkPipeline *out);
|
||||
// How many it made so far: from_cache came without a compile and compiled went into the play cache (both only with a
|
||||
// play cache, which is what can tell them apart); uncached were created with no cache of ours.
|
||||
typedef struct { uint64_t made, from_cache, compiled, uncached; } VkForeignPipes;
|
||||
void vk_pipeline_foreign_counts(VkForeignPipes *o);
|
||||
|
||||
#include "vk_rec.h" // last: its macros route the vkCmd* calls of every file that includes vk.h
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <windows.h>
|
||||
#endif
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <math.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
@@ -61,6 +62,54 @@ void message(uint32_t type, const wchar_t *msg) {
|
||||
if (told++ < 40) rt_log("ffx: %s %s", type == FFX_API_MESSAGE_TYPE_ERROR ? "error:" : "warning:", s);
|
||||
}
|
||||
|
||||
// The library's pipeline creates (vk_ffx_proc_addr). It passes no pipeline cache, so one that names none goes through
|
||||
// the runtime's (vk_pipeline.c vk_pipeline_create_foreign); one that names a cache is the library's own business.
|
||||
VKAPI_ATTR VkResult VKAPI_CALL compute_pipelines(VkDevice dev, VkPipelineCache cache, uint32_t n,
|
||||
const VkComputePipelineCreateInfo *ci, const VkAllocationCallbacks *a,
|
||||
VkPipeline *out) {
|
||||
if (cache) return vkCreateComputePipelines(dev, cache, n, ci, a, out);
|
||||
return vk_pipeline_create_foreign(dev, 1, n, ci, a, out);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL graphics_pipelines(VkDevice dev, VkPipelineCache cache, uint32_t n,
|
||||
const VkGraphicsPipelineCreateInfo *ci, const VkAllocationCallbacks *a,
|
||||
VkPipeline *out) {
|
||||
if (cache) return vkCreateGraphicsPipelines(dev, cache, n, ci, a, out);
|
||||
return vk_pipeline_create_foreign(dev, 0, n, ci, a, out);
|
||||
}
|
||||
|
||||
int pipe_cache_on(void) { // NX2NAVI_FFX_PIPE_CACHE, read once
|
||||
static std::atomic<int> on{-1};
|
||||
int v = on.load(std::memory_order_relaxed);
|
||||
if (v < 0) on.store(v = cfg_bool("NX2NAVI_FFX_PIPE_CACHE"), std::memory_order_relaxed);
|
||||
return v;
|
||||
}
|
||||
|
||||
// ffxCreateContext with what it made in pipelines: the library compiles every pass of a context inside this call, on
|
||||
// the thread that asked, so the line says what a context cost and where its pipelines came from.
|
||||
ffxReturnCode_t create_counted(ffxContext *ctx, ffxCreateContextDescHeader *desc) {
|
||||
VkForeignPipes p0, p1;
|
||||
vk_pipeline_foreign_counts(&p0);
|
||||
auto t0 = std::chrono::steady_clock::now();
|
||||
ffxReturnCode_t r = A.create(ctx, desc, nullptr);
|
||||
double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
||||
vk_pipeline_foreign_counts(&p1);
|
||||
unsigned long long made = p1.made - p0.made, hit = p1.from_cache - p0.from_cache, comp = p1.compiled - p0.compiled,
|
||||
none = p1.uncached - p0.uncached;
|
||||
if (r != FFX_API_RETURN_OK) return r;
|
||||
if (!pipe_cache_on())
|
||||
rt_log("ffx: context created in %.0f ms; its pipelines are made without the runtime's pipeline caches "
|
||||
"(NX2NAVI_FFX_PIPE_CACHE=0)", ms);
|
||||
else if (!made)
|
||||
rt_log("ffx: context created in %.0f ms; it made no pipeline through the runtime's creates", ms);
|
||||
else
|
||||
rt_log("ffx: context created in %.0f ms; its %llu pipelines: %llu from the pipeline caches without a compile, "
|
||||
"%llu compiled and kept in the play cache for the next start, %llu through the one pipeline cache (what "
|
||||
"it held is not told apart), %llu with no pipeline cache (none is up yet, or pipeline binaries are on)",
|
||||
ms, made, hit, comp, made - hit - comp - none, none);
|
||||
return r;
|
||||
}
|
||||
|
||||
// One upscaling context per render and output size: each render scale the menu sets brings its own, and so do the
|
||||
// game's second scenes. When all are taken, the least recently used one is destroyed for the new size once the GPU is
|
||||
// idle, as vk_dlaa.c free_size does for DLSS. Without that, a seventh size found no context, and vk_dlaa.c turned
|
||||
@@ -119,7 +168,7 @@ Up *context_for(uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) {
|
||||
be.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_BACKEND_VK;
|
||||
be.vkDevice = c->device;
|
||||
be.vkPhysicalDevice = c->phys;
|
||||
be.vkDeviceProcAddr = vkGetDeviceProcAddr;
|
||||
be.vkDeviceProcAddr = vk_ffx_proc_addr;
|
||||
ffxCreateContextDescUpscale d = {};
|
||||
d.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_UPSCALE;
|
||||
d.header.pNext = &be.header;
|
||||
@@ -127,7 +176,7 @@ Up *context_for(uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) {
|
||||
d.maxRenderSize = {w, h};
|
||||
d.maxUpscaleSize = {ow, oh};
|
||||
d.fpMessage = message;
|
||||
ffxReturnCode_t r = A.create(&slot->ctx, &d.header, nullptr);
|
||||
ffxReturnCode_t r = create_counted(&slot->ctx, &d.header);
|
||||
if (r != FFX_API_RETURN_OK) {
|
||||
rt_log("ffx: FSR 3.1 context creation at %ux%u -> %ux%u failed (%u)", w, h, ow, oh, (unsigned)r);
|
||||
slot->ctx = nullptr;
|
||||
@@ -229,7 +278,55 @@ extern "C" int vk_ffx_upscale(VkCommandBuffer cb, const VkFfxImage *color, const
|
||||
}
|
||||
|
||||
extern "C" ffxReturnCode_t vk_ffx_create(ffxContext *ctx, ffxCreateContextDescHeader *desc) {
|
||||
return vk_ffx_load() ? A.create(ctx, desc, nullptr) : FFX_API_RETURN_ERROR;
|
||||
return vk_ffx_load() ? create_counted(ctx, desc) : FFX_API_RETURN_ERROR;
|
||||
}
|
||||
|
||||
// What the library is handed as ffxCreateBackendVKDesc.vkDeviceProcAddr: the device's functions, with the two
|
||||
// pipeline creates replaced by ours (compute_pipelines, graphics_pipelines) unless NX2NAVI_FFX_PIPE_CACHE is 0.
|
||||
extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_ffx_proc_addr(VkDevice dev, const char *name) {
|
||||
if (name && pipe_cache_on()) {
|
||||
if (!strcmp(name, "vkCreateComputePipelines")) return (PFN_vkVoidFunction)compute_pipelines;
|
||||
if (!strcmp(name, "vkCreateGraphicsPipelines")) return (PFN_vkVoidFunction)graphics_pipelines;
|
||||
}
|
||||
return vkGetDeviceProcAddr(dev, name);
|
||||
}
|
||||
|
||||
// --probe-ffx-cache (host.h): one FSR 3.1 upscaling context on the device, without a window or the game, and where
|
||||
// its pipelines came from. The pipeline caches are those beside the executable, or NX2NAVI_PIPELINE_CACHE's, and are
|
||||
// saved as after a run, so a second run of the probe shows what the first one's files give
|
||||
// (tests/ffx_cache_check.py runs it twice on a folder of its own). Fails when the context is not created, or when
|
||||
// the library made no pipeline through the runtime's creates although they are on.
|
||||
extern "C" int vk_ffx_probe_cache(void) {
|
||||
if (!vk_init()) {
|
||||
printf("probe-ffx-cache: SKIP (no Vulkan device)\n");
|
||||
return 0;
|
||||
}
|
||||
if (!vk_ffx_load()) {
|
||||
printf("probe-ffx-cache: SKIP (no amd_fidelityfx_vk.dll beside the executable, or a build without FSR 3.1)\n");
|
||||
return 0;
|
||||
}
|
||||
vk_pipeline_push_fn(); // the first vk_pipeline_* call reads the pipeline caches
|
||||
VkForeignPipes p0, p1;
|
||||
vk_pipeline_foreign_counts(&p0);
|
||||
auto t0 = std::chrono::steady_clock::now();
|
||||
Up *u = context_for(1280, 720, 2560, 1440);
|
||||
double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
||||
vk_pipeline_foreign_counts(&p1);
|
||||
unsigned long long made = p1.made - p0.made, hit = p1.from_cache - p0.from_cache, comp = p1.compiled - p0.compiled,
|
||||
none = p1.uncached - p0.uncached;
|
||||
int on = pipe_cache_on(), fails = 0;
|
||||
printf("probe-ffx-cache: context %s in %.0f ms; pipelines made through the runtime's creates: %llu (from the "
|
||||
"pipeline caches %llu, compiled into the play cache %llu, through the one cache %llu, with no cache %llu); "
|
||||
"NX2NAVI_FFX_PIPE_CACHE %d\n", u ? "created" : "not created", ms, made, hit, comp, made - hit - comp - none,
|
||||
none, on);
|
||||
if (!u) fails++, printf("probe-ffx-cache: FAIL: no context\n");
|
||||
if (u && on && !made) fails++, printf("probe-ffx-cache: FAIL: the library created its pipelines past the runtime\n");
|
||||
if (u && !on && made) fails++, printf("probe-ffx-cache: FAIL: pipelines came through the runtime with the setting 0\n");
|
||||
vk_queue_lock(), vkDeviceWaitIdle(vk_ctx()->device), vk_queue_unlock();
|
||||
vk_ffx_release();
|
||||
vk_pipeline_save();
|
||||
printf("probe-ffx-cache: %s\n", fails ? "FAILED" : "passed");
|
||||
return fails ? 1 : 0;
|
||||
}
|
||||
extern "C" ffxReturnCode_t vk_ffx_destroy(ffxContext *ctx) { return A.ok ? A.destroy(ctx, nullptr) : FFX_API_RETURN_ERROR; }
|
||||
extern "C" ffxReturnCode_t vk_ffx_configure(ffxContext *ctx, const ffxConfigureDescHeader *desc) {
|
||||
|
||||
@@ -22,6 +22,9 @@ int vk_ffx_framegen_available(void);
|
||||
int vk_ffx_upscale(VkCommandBuffer cb, const VkFfxImage *color, const VkFfxImage *depth, const VkFfxImage *mv,
|
||||
const VkFfxImage *out, float jx, float jy, int reset, float dt_ms, float fov_y);
|
||||
void vk_ffx_release(void); // before the device goes away
|
||||
// The device's functions as the library is given them (ffxCreateBackendVKDesc.vkDeviceProcAddr): its pipeline creates
|
||||
// go through the runtime's pipeline caches (vk_pipeline.c vk_pipeline_create_foreign; NX2NAVI_FFX_PIPE_CACHE=0: not).
|
||||
VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_ffx_proc_addr(VkDevice dev, const char *name);
|
||||
|
||||
#ifdef __cplusplus
|
||||
// The API's calls themselves, for vk_fg.cpp (0 = FFX_API_RETURN_OK): the library loaded first (vk_ffx_load).
|
||||
|
||||
@@ -200,7 +200,7 @@ extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt,
|
||||
be.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_BACKEND_VK;
|
||||
be.vkDevice = c->device;
|
||||
be.vkPhysicalDevice = c->phys;
|
||||
be.vkDeviceProcAddr = vkGetDeviceProcAddr;
|
||||
be.vkDeviceProcAddr = vk_ffx_proc_addr;
|
||||
// the HUD-less picture has the swapchain's format (gfx_present.c hudless_record)
|
||||
ffxCreateContextDescFrameGenerationHudless hl = {};
|
||||
hl.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_FRAMEGENERATION_HUDLESS;
|
||||
|
||||
@@ -52,6 +52,7 @@
|
||||
#include "vk_build_gate.h"
|
||||
#include "vk_cache_file.h"
|
||||
#include "vk_gpa.h"
|
||||
#include "vk_save_retry.h"
|
||||
#include "vk_warm_hint.h"
|
||||
|
||||
#define LOG_ONCE(...) do { static volatile int32_t once_; if (!plat_xchg32(&once_, 1)) rt_log(__VA_ARGS__); } while (0)
|
||||
@@ -593,6 +594,82 @@ static VkResult pipe_create_in(int kind, void *ci, VkPipeline *pipe, VkPipelineC
|
||||
return r;
|
||||
}
|
||||
|
||||
// ---- pipelines of a library that draws on our device (FSR 3.1: vk_ffx.cpp vk_ffx_proc_addr)
|
||||
// Such a library creates its pipelines itself and passes no pipeline cache, so nothing of ours kept them: every
|
||||
// context it made (one per render and output size, and one for frame generation) compiled its passes again unless the
|
||||
// driver's own disk cache still had them, which a renamed or updated exe on NVIDIA and a driver without a disk cache
|
||||
// do not. vk_ffx.cpp hands the library these creates in the place of vkCreateComputePipelines and
|
||||
// vkCreateGraphicsPipelines; a create that names no cache goes through ours as pipe_create_in does: asked of P.cache
|
||||
// first without compiling, and compiled into the play cache when neither it nor the driver has the pipeline, so the
|
||||
// file a run writes (pipeline-cache-play.bin, or pipeline-cache.bin without a play cache) holds it for the next start.
|
||||
// Before the first pipeline of ours (P.ready 0) and with pipeline binaries on there is no cache to give: the create
|
||||
// goes to the driver as the library asked. infos: n VkComputePipelineCreateInfo (compute 1) or
|
||||
// VkGraphicsPipelineCreateInfo. Any thread.
|
||||
static struct { volatile uint64_t made, from_cache, compiled, uncached; } FP;
|
||||
|
||||
void vk_pipeline_foreign_counts(VkForeignPipes *o) {
|
||||
o->made = __atomic_load_n(&FP.made, __ATOMIC_RELAXED), o->from_cache = __atomic_load_n(&FP.from_cache, __ATOMIC_RELAXED);
|
||||
o->compiled = __atomic_load_n(&FP.compiled, __ATOMIC_RELAXED), o->uncached = __atomic_load_n(&FP.uncached, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static VkResult foreign_one(VkDevice dev, int compute, VkPipelineCache cache, uint32_t n, const void *ci,
|
||||
const VkAllocationCallbacks *alloc, VkPipeline *out) {
|
||||
return compute ? vkCreateComputePipelines(dev, cache, n, ci, alloc, out)
|
||||
: vkCreateGraphicsPipelines(dev, cache, n, ci, alloc, out);
|
||||
}
|
||||
|
||||
VkResult vk_pipeline_create_foreign(VkDevice dev, int compute, uint32_t n, const void *infos,
|
||||
const VkAllocationCallbacks *alloc, VkPipeline *out) {
|
||||
VkPipelineCache cache = __atomic_load_n(&P.ready, __ATOMIC_ACQUIRE) && dev == P.device ? P.cache : VK_NULL_HANDLE;
|
||||
if (!cache) {
|
||||
VkResult r = foreign_one(dev, compute, VK_NULL_HANDLE, n, infos, alloc, out);
|
||||
if (r == VK_SUCCESS) __atomic_fetch_add(&FP.made, n, __ATOMIC_RELAXED), __atomic_fetch_add(&FP.uncached, n, __ATOMIC_RELAXED);
|
||||
return r;
|
||||
}
|
||||
VkResult r = VK_SUCCESS;
|
||||
plat_mutex_lock_shared(&cache_rw);
|
||||
if (!P.play) { // one cache: whether it held them is not known here; its save looks at the data's size
|
||||
r = foreign_one(dev, compute, cache, n, infos, alloc, out);
|
||||
if (r == VK_SUCCESS) __atomic_fetch_add(&FP.made, n, __ATOMIC_RELAXED), __atomic_fetch_add(&cache_merges, n, __ATOMIC_RELAXED);
|
||||
}
|
||||
for (uint32_t i = 0; P.play && i < n; i++) {
|
||||
// a copy of the create info: the flags are ours to change, the library's struct is not
|
||||
VkComputePipelineCreateInfo cc;
|
||||
VkGraphicsPipelineCreateInfo gc;
|
||||
void *ci;
|
||||
if (compute) cc = ((const VkComputePipelineCreateInfo *)infos)[i], ci = &cc;
|
||||
else gc = ((const VkGraphicsPipelineCreateInfo *)infos)[i], ci = &gc;
|
||||
PipeCiHead *hd = ci;
|
||||
// Flags given in a VkPipelineCreateFlags2CreateInfo replace the struct's own, so the question "without
|
||||
// compiling" cannot be put through hd->flags: such a create goes to the play cache at once.
|
||||
int flags2 = 0;
|
||||
for (const VkBaseInStructure *b = hd->pNext; b && !flags2; b = b->pNext)
|
||||
flags2 = b->sType == VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR;
|
||||
VkPipelineCreateFlags flags = hd->flags;
|
||||
VkResult ri = VK_PIPELINE_COMPILE_REQUIRED;
|
||||
out[i] = VK_NULL_HANDLE;
|
||||
if (!flags2) {
|
||||
hd->flags = flags | VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
|
||||
ri = foreign_one(dev, compute, cache, 1, ci, alloc, &out[i]);
|
||||
hd->flags = flags;
|
||||
if (ri == VK_SUCCESS) __atomic_fetch_add(&FP.from_cache, 1, __ATOMIC_RELAXED);
|
||||
}
|
||||
if (ri == VK_PIPELINE_COMPILE_REQUIRED) {
|
||||
ri = foreign_one(dev, compute, P.play, 1, ci, alloc, &out[i]);
|
||||
if (ri == VK_SUCCESS) {
|
||||
__atomic_fetch_add(&FP.compiled, 1, __ATOMIC_RELAXED);
|
||||
__atomic_fetch_add(&P.play_made, 1, __ATOMIC_RELAXED);
|
||||
__atomic_fetch_add(&cache_merges, 1, __ATOMIC_RELAXED); // autosave_once: the play file is due
|
||||
}
|
||||
}
|
||||
if (ri == VK_SUCCESS) __atomic_fetch_add(&FP.made, 1, __ATOMIC_RELAXED);
|
||||
else if (r == VK_SUCCESS) r = ri; // the first that failed, as a create of several reports one result
|
||||
if (ri != VK_SUCCESS) out[i] = VK_NULL_HANDLE;
|
||||
}
|
||||
plat_mutex_unlock_shared(&cache_rw);
|
||||
return r;
|
||||
}
|
||||
|
||||
static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) {
|
||||
PipeCiHead *hd = ci;
|
||||
VkResult r;
|
||||
@@ -6327,14 +6404,15 @@ static int seed_pipelines(const char *dir) {
|
||||
while (K.left) plat_cond_wait(&done_cv, &P.lock); // those a pool worker took
|
||||
uint32_t built = K.built - built0, failed = K.failed - failed0;
|
||||
plat_mutex_unlock(&P.lock);
|
||||
int saved = cache_save(5000); // as a save at exit: it waits that long for a moment with no build running
|
||||
int saved = cache_save(5000) > 0; // as a save at exit: it waits that long for a moment with no build running
|
||||
plat_file_delete(mark);
|
||||
char msg[640];
|
||||
snprintf(msg, sizeof msg, "prepare-pipelines: %zu listed pipelines: %u built (%u mesh), %u failed, %u the same as one "
|
||||
"built before, %u without a shader this game version or its shader cache has, %u vertex pipelines left to "
|
||||
"their first draw (NX2NAVI_PREPARE_PIPELINES=%s), %u mesh pipelines this GPU cannot draw; on %d threads in "
|
||||
"%.0f ms; pipeline cache %s", n, built, mesh, failed, known, dropped, other, all ? "all" : "mesh", no_mesh,
|
||||
nt ? nt : 1, now_ms() - t0, saved ? "saved" : "not saved (builds kept running)");
|
||||
nt ? nt : 1, now_ms() - t0,
|
||||
saved ? "saved" : "not saved (builds kept running, or its file could not be written)");
|
||||
printf("%s\n", msg);
|
||||
fflush(stdout);
|
||||
rt_log("vk_pipeline: %s", msg);
|
||||
@@ -6642,7 +6720,8 @@ static PlatMutex save_lock = PLAT_MUTEX_INIT; // one cache save at a time; shu
|
||||
// cache had held it: three saves of 358.1 MB in one run that added 0.3 MB (2026-11-11).
|
||||
// The lock is only ever tried, never waited for: a waiting exclusive request would hold back every new build
|
||||
// (a shared request queues behind it), so a pending save could stall the replay. wait_ms: how long to keep trying
|
||||
// while builds run (0: one try). Returns 0 when no build-free moment came, 1 otherwise (saved, or nothing to save).
|
||||
// while builds run (0: one try). Returns 0 when no build-free moment came, -1 when the data could not be copied or
|
||||
// its file not written (the log says which), 1 otherwise (saved, or nothing to save).
|
||||
static int cache_save_one(int which, uint32_t wait_ms) {
|
||||
VkPipelineCache cache = which ? P.play : P.cache;
|
||||
const char *path = which ? P.play_path : P.cache_path;
|
||||
@@ -6659,12 +6738,14 @@ static int cache_save_one(int which, uint32_t wait_ms) {
|
||||
int got = sized && !same && (data = malloc(size)) && vkGetPipelineCacheData(P.device, cache, &size, data) == VK_SUCCESS;
|
||||
plat_mutex_unlock(&cache_rw);
|
||||
static uint64_t skipped[2];
|
||||
int failed = sized && !same && !got;
|
||||
if (same) skipped[which]++;
|
||||
if (sized && !same && !got) rt_log("vk_pipeline: %s not saved: no copy of its %.1f MB", path, size / 1048576.0);
|
||||
if (failed) rt_log("vk_pipeline: %s not saved: no copy of its %.1f MB", path, size / 1048576.0);
|
||||
if (got) {
|
||||
uint64_t t0 = plat_now_ms();
|
||||
if (!plat_file_write_atomic(path, data, size)) {
|
||||
rt_log("vk_pipeline: cannot write %s (%.1f MB)", path, size / 1048576.0);
|
||||
failed = 1;
|
||||
} else {
|
||||
uint64_t loaded = which ? P.play_loaded : P.cache_loaded;
|
||||
rt_log("vk_pipeline: %s saved: %.1f MB (%+.1f MB since the boot) written in %llu ms; %llu saves before it were "
|
||||
@@ -6685,9 +6766,10 @@ static int cache_save_one(int which, uint32_t wait_ms) {
|
||||
}
|
||||
}
|
||||
free(data);
|
||||
return 1;
|
||||
return failed ? -1 : 1;
|
||||
}
|
||||
|
||||
// Both caches' saves (cache_save_one): 1 when each is done, 0 when one found no build-free moment, -1 when one failed.
|
||||
static int cache_save(uint32_t wait_ms) {
|
||||
if (!P.ready || !P.cache || !P.cache_path[0]) return 1;
|
||||
// Not under P.lock: holding P.lock through the copy and the file write (about 60 ms on the 1.4.3 fight) stopped
|
||||
@@ -6697,9 +6779,9 @@ static int cache_save(uint32_t wait_ms) {
|
||||
plat_mutex_unlock(&save_lock);
|
||||
return 1;
|
||||
}
|
||||
int ok = 1;
|
||||
int ok = 1, r;
|
||||
if (!P.play || P.fold) ok = cache_save_one(0, wait_ms);
|
||||
if (P.play) ok = cache_save_one(1, wait_ms) && ok;
|
||||
if (P.play && (r = cache_save_one(1, wait_ms)) < ok) ok = r; // the worse of the two
|
||||
plat_mutex_unlock(&save_lock);
|
||||
return ok;
|
||||
}
|
||||
@@ -6707,7 +6789,8 @@ static int cache_save(uint32_t wait_ms) {
|
||||
// Builds still running after this long at a save that waits (exit, the shader preparation) leave the file as it was.
|
||||
#define SAVE_WAIT_MS 5000
|
||||
|
||||
static void save_with(uint32_t wait_ms) {
|
||||
// Returns cache_save's answer.
|
||||
static int save_with(uint32_t wait_ms) {
|
||||
if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log();
|
||||
if (P.ready && P.gpl) {
|
||||
rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; "
|
||||
@@ -6762,9 +6845,11 @@ static void save_with(uint32_t wait_ms) {
|
||||
}
|
||||
gfx_shader_record_flush(1);
|
||||
if (P.ready && !P.cache) pb_log("at a save"); // with P.cache the line comes with its own save below
|
||||
if (!cache_save(wait_ms) && wait_ms)
|
||||
int saved = cache_save(wait_ms);
|
||||
if (!saved && wait_ms)
|
||||
rt_log("vk_pipeline: %s not saved: pipeline builds kept running for %u ms", P.play ? P.play_path : P.cache_path,
|
||||
(unsigned)wait_ms);
|
||||
return saved;
|
||||
}
|
||||
|
||||
// The save at exit and after the shader preparation: it waits up to SAVE_WAIT_MS for a moment with no build running.
|
||||
@@ -6773,9 +6858,11 @@ void vk_pipeline_save(void) {
|
||||
}
|
||||
|
||||
// The game never shuts the device down (boot.c and the window end the process), so the cache is saved while it
|
||||
// runs: at most every 20 s, and only after new pipelines were built. New pipeline keys are appended every 2 s.
|
||||
// runs: at most every 20 s, and only after new pipelines were built or while a save that did not happen is still owed
|
||||
// (vk_save_retry.h). New pipeline keys are appended every 2 s.
|
||||
static void autosave_once(void) {
|
||||
static uint64_t last_n, last_ms, last_keys;
|
||||
static uint64_t last_keys;
|
||||
static SaveRetry sr;
|
||||
if (!P.ready) return;
|
||||
uint64_t now = plat_now_ms();
|
||||
if (now - last_keys >= 2000) {
|
||||
@@ -6785,19 +6872,20 @@ static void autosave_once(void) {
|
||||
cu_flush();
|
||||
}
|
||||
gfx_shader_record_flush(0);
|
||||
if (now - last_ms < 20000) return;
|
||||
if (now - sr.at_ms < SAVE_EVERY_MS) return;
|
||||
VkPipeStats s;
|
||||
vk_pipeline_stats(&s);
|
||||
uint64_t n = s.graphics + s.compute + cache_merges; // a merge comes after its pipeline was counted (bg_main)
|
||||
if (n == last_n && !P.fold) return; // P.fold: the play file merged at the start is to be written
|
||||
if (!save_due(&sr, n, now, P.fold)) return; // P.fold: the play file merged at the start is to be written
|
||||
// Only at a moment with no build running (cache_save): while one runs the save waits for the next tick, 500 ms
|
||||
// on, before it logs or flushes anything, so a busy stretch does not repeat the statistics lines every tick.
|
||||
if (P.cache && P.cache_path[0]) {
|
||||
if (!plat_mutex_trylock(&cache_rw)) return;
|
||||
plat_mutex_unlock(&cache_rw);
|
||||
}
|
||||
last_n = n, last_ms = now;
|
||||
save_with(0);
|
||||
if (save_tried(&sr, n, now, save_with(0) > 0))
|
||||
rt_log("vk_pipeline: the pipeline cache was not saved in %d tries in a row; the next try is made when a new "
|
||||
"pipeline is built, or at exit", SAVE_RETRIES + 1);
|
||||
}
|
||||
|
||||
// On its own thread: the save (vkGetPipelineCacheData and the file write, about 60 ms) and the key appends (a flush to
|
||||
|
||||
39
runtime/vk_save_retry.h
Normal file
39
runtime/vk_save_retry.h
Normal file
@@ -0,0 +1,39 @@
|
||||
// When the autosave thread saves the pipeline caches (vk_pipeline.c autosave_once), and when it tries again after a
|
||||
// save that did not happen. Own header so tests/save_retry_check.c checks it without Vulkan.
|
||||
//
|
||||
// A save is due when something new entered the caches since the last try. A try that did not save (no copy of the
|
||||
// data for want of memory, a file that could not be written, a build that began between the look and the copy) used to
|
||||
// be the end of it: the next save waited for the next new pipeline, and a run that built none, then ended without its
|
||||
// orderly exit, lost what it had compiled since the last file. Now the try is repeated at the save interval, up to
|
||||
// SAVE_RETRIES times in a row, so a folder that stays unwritable costs that many log lines and no more until a new
|
||||
// pipeline is built. The same count bounds the save that is pending from the start (the play file folded into the
|
||||
// main one, `pending`), which before was tried at every interval for as long as it failed.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define SAVE_EVERY_MS 20000u // between two tries
|
||||
#define SAVE_RETRIES 6 // tries in a row after one that did not save
|
||||
|
||||
typedef struct {
|
||||
uint64_t n; // what had entered the caches at the last try
|
||||
uint64_t at_ms; // when that try was made
|
||||
uint32_t failed; // tries in a row that did not save
|
||||
} SaveRetry;
|
||||
|
||||
// 1 when a save is to be tried now. n: the count of what entered the caches so far (it only grows); pending: a save
|
||||
// is owed whatever n says.
|
||||
static inline int save_due(const SaveRetry *s, uint64_t n, uint64_t now_ms, int pending) {
|
||||
if (now_ms - s->at_ms < SAVE_EVERY_MS) return 0;
|
||||
if (n != s->n) return 1;
|
||||
return (pending || s->failed) && s->failed <= SAVE_RETRIES;
|
||||
}
|
||||
|
||||
// A try was made at now_ms with n in the caches; saved: the files hold the caches' data now. Returns 1 when this was
|
||||
// the last try in a row (the caller says so once): the next is made for new content.
|
||||
static inline int save_tried(SaveRetry *s, uint64_t n, uint64_t now_ms, int saved) {
|
||||
if (n != s->n) s->failed = 0; // new content starts a row of its own
|
||||
s->n = n, s->at_ms = now_ms;
|
||||
s->failed = saved ? 0 : s->failed + 1;
|
||||
return s->failed == SAVE_RETRIES + 1;
|
||||
}
|
||||
83
tests/ffx_cache_check.py
Normal file
83
tests/ffx_cache_check.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""Run the game executable's --probe-ffx-cache twice on pipeline cache files of the test's own
|
||||
(runtime/vk_ffx.cpp vk_ffx_probe_cache: one FSR 3.1 upscaling context, without a window or the game).
|
||||
|
||||
FSR 3.1's library creates its pipelines itself and names no pipeline cache; the runtime hands it pipeline creates that
|
||||
go through the pipeline caches kept across runs (runtime/vk_pipeline.c vk_pipeline_create_foreign). Checked here:
|
||||
|
||||
- the library's pipelines come through those creates (the first run counts them);
|
||||
- a second run, which reads what the first one saved, compiles none of them: each comes from the caches;
|
||||
- with ffx_pipe_cache = 0 none comes through the runtime.
|
||||
|
||||
The executable needs amd_fidelityfx_vk.dll beside it and a Vulkan device; without either the probe says SKIP and so
|
||||
does this test. The pipeline cache files go to build/test/ffx_cache, never beside the executable.
|
||||
|
||||
Usage: ffx_cache_check.py PATH_TO_GAME_EXE
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LINE = re.compile(r'probe-ffx-cache: context (created|not created) in \d+ ms; pipelines made through the runtime\'s '
|
||||
r'creates: (\d+) \(from the pipeline caches (\d+), compiled into the play cache (\d+), through the '
|
||||
r'one cache (\d+), with no cache (\d+)\)')
|
||||
|
||||
|
||||
def run(game, cache, setting):
|
||||
"""One probe run -> (exit code, last probe line, the counts or None)."""
|
||||
env = dict(os.environ)
|
||||
env['NX2NAVI_PIPELINE_CACHE'] = str(cache)
|
||||
env.pop('NX2NAVI_PIPE_BINARIES', None)
|
||||
env['NX2NAVI_FFX_PIPE_CACHE'] = setting
|
||||
r = subprocess.run([game, '--probe-ffx-cache'], env=env, capture_output=True, text=True, errors='replace')
|
||||
lines = [ln for ln in r.stdout.splitlines() if ln.startswith('probe-ffx-cache:')]
|
||||
counts = None
|
||||
for ln in lines:
|
||||
m = LINE.match(ln)
|
||||
if m:
|
||||
counts = dict(zip(('made', 'cached', 'compiled', 'one', 'none'), map(int, m.groups()[1:])))
|
||||
print('\n'.join(lines))
|
||||
return r.returncode, lines[-1] if lines else '', counts
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print(__doc__)
|
||||
return 2
|
||||
game = sys.argv[1]
|
||||
work = ROOT / 'build' / 'test' / 'ffx_cache'
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
work.mkdir(parents=True)
|
||||
cache = work / 'pipeline-cache.bin'
|
||||
rc, last, first = run(game, cache, '1')
|
||||
if 'SKIP' in last:
|
||||
print('ffx_cache_check: SKIP')
|
||||
return 0
|
||||
fails = []
|
||||
if rc or not last.endswith('passed') or not first:
|
||||
fails.append('the first run did not pass')
|
||||
elif not first['made']:
|
||||
fails.append('the first run made no pipeline through the runtime')
|
||||
rc, last, second = run(game, cache, '1')
|
||||
if rc or not last.endswith('passed') or not second:
|
||||
fails.append('the second run did not pass')
|
||||
elif first and (second['made'] != first['made'] or second['compiled']):
|
||||
fails.append('the second run made %d pipelines and compiled %d of them: with the first run\'s files it must '
|
||||
'make the same %d and compile none' % (second['made'], second['compiled'], first['made']))
|
||||
elif first and not first['one'] and not first['none'] and second['cached'] != second['made']:
|
||||
fails.append('with a play cache every pipeline of the second run comes from the caches: %d of %d did'
|
||||
% (second['cached'], second['made']))
|
||||
rc, last, off = run(game, cache, '0')
|
||||
if rc or not last.endswith('passed') or not off or off['made']:
|
||||
fails.append('with ffx_pipe_cache = 0 the run must pass with no pipeline made through the runtime')
|
||||
for f in fails:
|
||||
print('ffx_cache_check: FAIL: ' + f)
|
||||
print('ffx_cache_check: %s' % ('FAILED' if fails else 'passed'))
|
||||
return 1 if fails else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
65
tests/save_retry_check.c
Normal file
65
tests/save_retry_check.c
Normal file
@@ -0,0 +1,65 @@
|
||||
// runtime/vk_save_retry.h: when the autosave thread saves the pipeline caches and when it tries again after a save
|
||||
// that did not happen (runtime/vk_pipeline.c autosave_once). Built and run by tests/save_retry_check.py; prints each
|
||||
// failed check and exits 1 on any.
|
||||
#include <stdio.h>
|
||||
|
||||
#include "vk_save_retry.h"
|
||||
|
||||
static int fails;
|
||||
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
|
||||
|
||||
int main(void) {
|
||||
uint64_t t = 1000000; // ms since the machine started, as the thread's clock gives them
|
||||
|
||||
// Nothing built, nothing owed: no save, however long the run.
|
||||
SaveRetry s = {0};
|
||||
CHECK(!save_due(&s, 0, t, 0), "a save with nothing new in the caches");
|
||||
CHECK(!save_due(&s, 0, t + 3600000, 0), "a save after an hour with nothing new");
|
||||
|
||||
// A new pipeline: a save, and after it none until the next one; never two within the interval.
|
||||
CHECK(save_due(&s, 1, t, 0), "no save for a new pipeline");
|
||||
CHECK(!save_tried(&s, 1, t, 1), "a good save counted as the last try");
|
||||
CHECK(!save_due(&s, 2, t + SAVE_EVERY_MS - 1, 0), "two saves within the interval");
|
||||
CHECK(save_due(&s, 2, t + SAVE_EVERY_MS, 0), "no save for the next pipeline once the interval passed");
|
||||
save_tried(&s, 2, t + SAVE_EVERY_MS, 1);
|
||||
CHECK(!save_due(&s, 2, t + 10 * SAVE_EVERY_MS, 0), "a save again with nothing new after a good one");
|
||||
|
||||
// A save that did not happen is tried again at the interval with nothing new built, SAVE_RETRIES times; the
|
||||
// last of them says so, and then nothing is tried until a new pipeline.
|
||||
s = (SaveRetry){0};
|
||||
uint64_t now = t;
|
||||
CHECK(save_due(&s, 5, now, 0), "no first save");
|
||||
CHECK(!save_tried(&s, 5, now, 0), "the first failed try counted as the last");
|
||||
int tries = 0, last = 0;
|
||||
for (int i = 0; i < 100 && !last; i++) {
|
||||
CHECK(!save_due(&s, 5, now + SAVE_EVERY_MS - 1, 0), "a retry within the interval");
|
||||
now += SAVE_EVERY_MS;
|
||||
if (!save_due(&s, 5, now, 0)) break;
|
||||
tries++;
|
||||
last = save_tried(&s, 5, now, 0);
|
||||
}
|
||||
CHECK(tries == SAVE_RETRIES && last, "%d retries after a failed save, %d wanted, the last one said: %d", tries,
|
||||
SAVE_RETRIES, last);
|
||||
CHECK(!save_due(&s, 5, now + 100 * SAVE_EVERY_MS, 0), "a try after the last of the row with nothing new");
|
||||
CHECK(save_due(&s, 6, now + SAVE_EVERY_MS, 0), "no try for a new pipeline after the row ended");
|
||||
CHECK(!save_tried(&s, 6, now + SAVE_EVERY_MS, 0), "new content did not start a row of its own");
|
||||
CHECK(save_due(&s, 6, now + 2 * SAVE_EVERY_MS, 0), "no retry in the new row");
|
||||
|
||||
// A retry that saves ends the row: nothing more until something new.
|
||||
s = (SaveRetry){0};
|
||||
save_tried(&s, 3, t, 0);
|
||||
CHECK(save_due(&s, 3, t + SAVE_EVERY_MS, 0), "no retry after one failed save");
|
||||
CHECK(!save_tried(&s, 3, t + SAVE_EVERY_MS, 1), "a good retry counted as the last try");
|
||||
CHECK(!save_due(&s, 3, t + 2 * SAVE_EVERY_MS, 0), "a save again after a good retry");
|
||||
|
||||
// A save owed from the start (the play file folded in) is tried with nothing built, and bounded the same way.
|
||||
s = (SaveRetry){0};
|
||||
CHECK(save_due(&s, 0, t, 1), "no save for the one owed from the start");
|
||||
now = t, tries = 1;
|
||||
save_tried(&s, 0, now, 0);
|
||||
while (tries < 100 && save_due(&s, 0, now += SAVE_EVERY_MS, 1)) tries++, save_tried(&s, 0, now, 0);
|
||||
CHECK(tries == SAVE_RETRIES + 1, "%d tries of an owed save that keeps failing, %d wanted", tries, SAVE_RETRIES + 1);
|
||||
|
||||
printf("save_retry_check: %s\n", fails ? "FAILED" : "passed");
|
||||
return fails ? 1 : 0;
|
||||
}
|
||||
40
tests/save_retry_check.py
Normal file
40
tests/save_retry_check.py
Normal file
@@ -0,0 +1,40 @@
|
||||
"""Build and run tests/save_retry_check.c: runtime/vk_save_retry.h, when the autosave thread saves the pipeline caches
|
||||
and when it tries again after a save that did not happen (no copy of the data, a file that cannot be written).
|
||||
|
||||
Usage: save_retry_check.py
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests' / 'diff'))
|
||||
try:
|
||||
from common import MSYS_BIN, compiler # noqa: E402
|
||||
except ImportError: # a Python without the differential tests' packages: the same compiler choice without them
|
||||
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||
|
||||
def compiler():
|
||||
if os.environ.get('CC'):
|
||||
return os.environ['CC']
|
||||
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
|
||||
return str(MSYS_BIN / 'clang.exe')
|
||||
return shutil.which('clang') or 'clang'
|
||||
|
||||
|
||||
def main():
|
||||
work = ROOT / 'build' / 'test' / 'save_retry'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
exe = work / ('save_retry_check.exe' if os.name == 'nt' else 'save_retry_check')
|
||||
env = dict(os.environ)
|
||||
if os.name == 'nt' and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
|
||||
str(ROOT / 'tests' / 'save_retry_check.c'), '-o', str(exe)], env=env)
|
||||
return subprocess.run([str(exe)], env=env).returncode
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
157
tests/validate_shader_cache_test.py
Normal file
157
tests/validate_shader_cache_test.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""tools/validate_shader_cache.py on a fake shader cache: the entries the game would refuse (a cut or zero-filled
|
||||
module, another magic number, a reflection that is not one JSON object naming its stage, a missing half of a pair) are
|
||||
named, packed and loose entries are both read with the loose one standing for its packed copy, --every samples, and,
|
||||
where spirv-val is installed, a whole module that is not valid SPIR-V is refused by it. No game files.
|
||||
|
||||
Run: python tests/validate_shader_cache_test.py
|
||||
"""
|
||||
import hashlib
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import pack_shader_cache as psc # noqa: E402
|
||||
import validate_shader_cache as vsc # noqa: E402
|
||||
|
||||
RES = b'{"stage":"cs","textures":[]}'
|
||||
|
||||
|
||||
def name(seed):
|
||||
return hashlib.sha256(seed.encode()).hexdigest()
|
||||
|
||||
|
||||
def insn(op, *operands):
|
||||
return struct.pack('<%dI' % (1 + len(operands)), (1 + len(operands)) << 16 | op, *operands)
|
||||
|
||||
|
||||
def text(s):
|
||||
b = s.encode() + b'\0'
|
||||
b += b'\0' * (-len(b) % 4)
|
||||
return struct.unpack('<%dI' % (len(b) // 4), b)
|
||||
|
||||
|
||||
def module(memory_model=True):
|
||||
"""A compute shader that does nothing, SPIR-V 1.3: ids 1 main, 2 void, 3 the function type, 4 the label."""
|
||||
body = insn(17, 1) # OpCapability Shader
|
||||
if memory_model:
|
||||
body += insn(14, 0, 1) # OpMemoryModel Logical GLSL450
|
||||
body += insn(15, 5, 1, *text('main')) # OpEntryPoint GLCompute %1 "main"
|
||||
body += insn(16, 1, 17, 1, 1, 1) # OpExecutionMode %1 LocalSize 1 1 1
|
||||
body += insn(19, 2) + insn(33, 3, 2) # OpTypeVoid, OpTypeFunction
|
||||
body += insn(54, 2, 1, 0, 3) + insn(248, 4) + insn(253) + insn(56) # OpFunction, OpLabel, OpReturn, OpFunctionEnd
|
||||
return struct.pack('<5I', vsc.SPIRV_MAGIC, 0x00010300, 0, 5, 0) + body
|
||||
|
||||
|
||||
def cut_module():
|
||||
"""A module whose last instruction says three words and has one: what a write cut short leaves."""
|
||||
return module() + struct.pack('<I', 3 << 16 | 0)
|
||||
|
||||
|
||||
def put(cache, seed, spv=None, res=None):
|
||||
if spv is not None:
|
||||
(cache / (name(seed) + '.spv')).write_bytes(spv)
|
||||
if res is not None:
|
||||
(cache / (name(seed) + '.res.json')).write_bytes(res)
|
||||
return name(seed)
|
||||
|
||||
|
||||
class ValidateTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.cache = Path(self.tmp.name) / 'shader-cache'
|
||||
self.cache.mkdir()
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def check(self, **kw):
|
||||
checked, failures = vsc.validate(self.cache, say=lambda s: None, **kw)
|
||||
return checked, {k: (o, p) for k, o, p in failures}
|
||||
|
||||
def test_game_checks(self):
|
||||
good = module()
|
||||
ok = put(self.cache, 'ok', good, RES)
|
||||
cut = put(self.cache, 'cut', good[:-6], RES) # not a whole number of words
|
||||
short = put(self.cache, 'short', cut_module(), RES) # the last instruction runs past the end
|
||||
zero = put(self.cache, 'zero', good[:40] + b'\0' * (len(good) - 40), RES)
|
||||
magic = put(self.cache, 'magic', b'\x01' + good[1:], RES)
|
||||
nul = put(self.cache, 'nul', good, b'\0' * len(RES))
|
||||
stage = put(self.cache, 'stage', good, b'{"stage":"xx"}')
|
||||
array = put(self.cache, 'array', good, b'[1]')
|
||||
no_res = put(self.cache, 'no-res', good, None)
|
||||
no_spv = put(self.cache, 'no-spv', None, RES)
|
||||
checked, bad = self.check()
|
||||
self.assertEqual(checked, 10)
|
||||
self.assertNotIn(ok, bad)
|
||||
self.assertIn('not a whole SPIR-V module', bad[cut][1])
|
||||
self.assertIn('past the end', bad[short][1])
|
||||
self.assertIn('zero words', bad[zero][1])
|
||||
self.assertIn('magic', bad[magic][1])
|
||||
self.assertIn('NUL', bad[nul][1])
|
||||
self.assertIn('no stage', bad[stage][1])
|
||||
self.assertIn('not one JSON object', bad[array][1])
|
||||
self.assertEqual(bad[no_res][1], 'no .res.json')
|
||||
self.assertEqual(bad[no_spv][1], 'no .spv')
|
||||
self.assertEqual(len(bad), 9)
|
||||
self.assertTrue(all(o == 'loose' for o, _ in bad.values()))
|
||||
|
||||
def test_stages_and_spaces(self):
|
||||
for st in ('vs', 'ps', 'cs', 'ts', 'ms'):
|
||||
self.assertEqual(vsc.reflection_problem(b' \n{ "stage" : "%s" }\n' % st.encode()), '')
|
||||
self.assertNotEqual(vsc.reflection_problem(b'{"stage":"gs"}'), '')
|
||||
self.assertNotEqual(vsc.reflection_problem(b'{}'), '')
|
||||
self.assertEqual(vsc.spirv_problem(module()), '')
|
||||
|
||||
def test_pack_and_loose(self):
|
||||
good = module()
|
||||
packed_ok = put(self.cache, 'packed-ok', good, RES)
|
||||
packed_bad = put(self.cache, 'packed-bad', cut_module(), RES)
|
||||
fixed = put(self.cache, 'fixed', cut_module(), RES)
|
||||
psc.pack(self.cache)
|
||||
self.assertFalse(list(self.cache.glob('*.spv')))
|
||||
put(self.cache, 'fixed', good, None) # a newer translation beside the pack's bad copy
|
||||
loose_bad = put(self.cache, 'loose-bad', good, b'{}')
|
||||
checked, bad = self.check()
|
||||
self.assertEqual(checked, 4)
|
||||
self.assertEqual(set(bad), {packed_bad, loose_bad})
|
||||
self.assertEqual(bad[packed_bad][0], 'pack')
|
||||
self.assertEqual(bad[loose_bad][0], 'loose')
|
||||
self.assertNotIn(fixed, bad)
|
||||
self.assertNotIn(packed_ok, bad)
|
||||
|
||||
def test_every_and_out(self):
|
||||
for i in range(10):
|
||||
put(self.cache, 'n%d' % i, cut_module() if i % 2 else module(), RES)
|
||||
checked, bad = self.check()
|
||||
self.assertEqual((checked, len(bad)), (10, 5))
|
||||
checked, _ = self.check(every=3)
|
||||
self.assertEqual(checked, 4)
|
||||
out = Path(self.tmp.name) / 'failed.txt'
|
||||
self.check(out=out)
|
||||
lines = out.read_text(encoding='utf-8').splitlines()
|
||||
self.assertEqual(len(lines), 5)
|
||||
self.assertTrue(all(ln.split()[0] in bad and ln.split()[1] == 'loose' for ln in lines))
|
||||
|
||||
def test_groups(self):
|
||||
a = vsc.group_of("spirv-val: error: line 12: ID '34[%34]' has not been defined")
|
||||
b = vsc.group_of("spirv-val: error: line 907: ID '5[%main]' has not been defined")
|
||||
self.assertEqual(a, b)
|
||||
|
||||
@unittest.skipUnless(vsc.find_spirv_val(), 'spirv-val is not installed')
|
||||
def test_spirv_val(self):
|
||||
ok = put(self.cache, 'ok', module(), RES)
|
||||
invalid = put(self.cache, 'invalid', module(memory_model=False), RES) # whole, and not valid SPIR-V
|
||||
checked, bad = self.check(val=vsc.find_spirv_val())
|
||||
self.assertEqual(checked, 2)
|
||||
self.assertNotIn(ok, bad)
|
||||
self.assertTrue(bad[invalid][1].startswith('spirv-val: '), bad[invalid][1])
|
||||
_, without = self.check() # the game's checks alone take both
|
||||
self.assertEqual(without, {})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
200
tools/validate_shader_cache.py
Normal file
200
tools/validate_shader_cache.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""Check the translated shaders an install's shader cache holds, offline: every <key>.spv with its <key>.res.json, from
|
||||
<cache>/shader-cache.pack and from the loose files (a loose file stands for its packed copy, as the game reads them).
|
||||
|
||||
The game checks little of a cached shader when it loads one (runtime/gfx_shader.c disk_load_from: the reflection is one
|
||||
JSON object that names its stage, the SPIR-V has its magic and whole instructions) and hands the module to the driver
|
||||
as it is. A module the driver then compiles wrongly, or crashes on, shows as a missing draw or a crash in play, far
|
||||
from its cause. This tool puts the question to the whole cache at once:
|
||||
|
||||
1. the game's own checks, so the count says how many entries the game would translate again;
|
||||
2. spirv-val on each module that passes them (--target-env vulkan1.3, as tools/shader_census.py validates fresh
|
||||
translations), unless --no-val is given or spirv-val is not found.
|
||||
|
||||
The modules are the translator's output; the pipeline code rewrites a copy of each before the driver sees it
|
||||
(runtime/vk_pipeline.c make_module), and that copy is not what is checked here. tools/shader_census.py --translate
|
||||
validates translations made now from the game files; this validates what a folder already holds, which is also what
|
||||
earlier translators wrote and what an interrupted write left.
|
||||
|
||||
python tools\\validate_shader_cache.py <install folder>\\shader-cache [--every N] [--jobs N] [--no-val] [--out FILE]
|
||||
|
||||
--every N checks every N-th shader (by key order), for a quick look at a large cache; --out writes one line per failed
|
||||
shader (key, origin, what failed). Exit code 0 when every checked entry passed, 1 otherwise, 2 for a folder that is not
|
||||
a shader cache. The game may be running: nothing is written to the cache.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import pack_shader_cache as psc # noqa: E402
|
||||
|
||||
SPV, RES = psc.KINDS['.spv'], psc.KINDS['.res.json']
|
||||
SPIRV_MAGIC = 0x07230203
|
||||
STAGES = (b'"vs"', b'"ps"', b'"cs"', b'"ts"', b'"ms"') # gfx_shader.c reflection_valid
|
||||
TARGET_ENV = 'vulkan1.3'
|
||||
|
||||
|
||||
def find_spirv_val():
|
||||
"""The path of spirv-val, or None: MSYS2 UCRT64's on Windows, else the one on PATH."""
|
||||
msys = Path(r'C:\msys64\ucrt64\bin\spirv-val.exe')
|
||||
if os.name == 'nt' and msys.is_file():
|
||||
return str(msys)
|
||||
return shutil.which('spirv-val')
|
||||
|
||||
|
||||
def spirv_problem(data):
|
||||
"""What gfx_shader.c disk_load_from refuses in a module ('' when it takes it): size, magic, and every instruction
|
||||
after the five header words having a word count that ends inside the module (spirv_complete)."""
|
||||
if data is None:
|
||||
return 'no .spv'
|
||||
if len(data) < 20 or len(data) % 4:
|
||||
return 'the .spv is %d bytes: not a whole SPIR-V module' % len(data)
|
||||
words = len(data) // 4
|
||||
if struct.unpack_from('<I', data, 0)[0] != SPIRV_MAGIC:
|
||||
return 'the .spv has another magic number'
|
||||
i = 5
|
||||
while i < words:
|
||||
n = struct.unpack_from('<I', data, 4 * i)[0] >> 16
|
||||
if not n:
|
||||
return 'an instruction of zero words at word %d (a zero-filled stretch)' % i
|
||||
i += n
|
||||
return '' if i == words else 'the last instruction runs past the end of the module'
|
||||
|
||||
|
||||
def reflection_problem(data):
|
||||
"""What gfx_shader.c reflection_valid refuses in a .res.json ('' when it takes it)."""
|
||||
if data is None:
|
||||
return 'no .res.json'
|
||||
if b'\0' in data:
|
||||
return 'the .res.json holds NUL bytes (an interrupted write)'
|
||||
body = data.strip()
|
||||
if len(body) < 2 or body[:1] != b'{' or body[-1:] != b'}':
|
||||
return 'the .res.json is not one JSON object'
|
||||
at = body.find(b'"stage"')
|
||||
quote = body.find(b'"', at + 7) if at >= 0 else -1
|
||||
if quote < 0 or not body.startswith(STAGES, quote):
|
||||
return 'the .res.json names no stage the game knows'
|
||||
return ''
|
||||
|
||||
|
||||
class Cache:
|
||||
"""The .spv and .res.json entries of a shader cache folder: loose files, then the pack."""
|
||||
|
||||
def __init__(self, folder):
|
||||
self.folder = Path(folder)
|
||||
self.loose = {k: p for k, p in psc.loose_files(self.folder).items() if k[0] in (SPV, RES)}
|
||||
self.pack_path = self.folder / psc.PACK
|
||||
self.packed = {k: v for k, v in psc.read_pack(self.pack_path).items() if k[0] in (SPV, RES)}
|
||||
|
||||
def keys(self):
|
||||
"""Every key with a .spv or a .res.json, sorted."""
|
||||
return sorted({name for _, name in self.loose} | {name for _, name in self.packed})
|
||||
|
||||
def read(self, kind, name, pack_file):
|
||||
"""(data or None, 'loose' | 'pack' | '') of one entry."""
|
||||
path = self.loose.get((kind, name))
|
||||
if path is not None:
|
||||
return psc.read_loose(path), 'loose'
|
||||
at = self.packed.get((kind, name))
|
||||
if at is None or pack_file is None:
|
||||
return None, ''
|
||||
return os.pread(pack_file, at[1], at[0]) if hasattr(os, 'pread') else pread(pack_file, at[1], at[0]), 'pack'
|
||||
|
||||
|
||||
def pread(fd, size, offset):
|
||||
"""os.pread where the platform has none (Windows): callers hold one descriptor per thread."""
|
||||
os.lseek(fd, offset, os.SEEK_SET)
|
||||
return os.read(fd, size)
|
||||
|
||||
|
||||
def check_one(cache, name, val, pack_file):
|
||||
"""(key hex, origin, problem): problem '' when the entry passed every check asked for."""
|
||||
spv, origin = cache.read(SPV, name, pack_file)
|
||||
res, res_origin = cache.read(RES, name, pack_file)
|
||||
origin = origin or res_origin
|
||||
problem = reflection_problem(res) or spirv_problem(spv)
|
||||
if not problem and val:
|
||||
r = subprocess.run([val, '--target-env', TARGET_ENV, '-'], input=spv, capture_output=True)
|
||||
if r.returncode:
|
||||
text = (r.stderr or r.stdout).decode('utf-8', 'replace').strip().splitlines()
|
||||
problem = 'spirv-val: ' + (text[0] if text else 'exit code %d' % r.returncode)
|
||||
return name.hex(), origin, problem
|
||||
|
||||
|
||||
def group_of(problem):
|
||||
"""A problem with its numbers and ids taken out, so the same fault in many shaders counts as one group."""
|
||||
return re.sub(r"'\d+\[%[^\]]*\]'|\b\d+\b", 'N', re.sub(r'^spirv-val: error: line \d+: ', 'spirv-val: ', problem))
|
||||
|
||||
|
||||
def validate(folder, every=1, jobs=0, val=None, out=None, say=print):
|
||||
"""Checks the cache at folder; returns (checked, failures) with failures a list of (key, origin, problem)."""
|
||||
cache = Cache(folder)
|
||||
names = cache.keys()[::max(1, every)]
|
||||
jobs = jobs or min(32, (os.cpu_count() or 4))
|
||||
failures, done, t0, last = [], 0, time.monotonic(), time.monotonic()
|
||||
|
||||
def work(chunk):
|
||||
# one descriptor of the pack per task: positioned reads without a shared file position
|
||||
fd = os.open(cache.pack_path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) if cache.packed else None
|
||||
try:
|
||||
return [check_one(cache, n, val, fd) for n in chunk]
|
||||
finally:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
|
||||
chunks = [names[i:i + 64] for i in range(0, len(names), 64)]
|
||||
with ThreadPoolExecutor(max_workers=jobs) as ex:
|
||||
for results in ex.map(work, chunks):
|
||||
done += len(results)
|
||||
failures += [r for r in results if r[2]]
|
||||
if time.monotonic() - last >= 10:
|
||||
last = time.monotonic()
|
||||
say(f' {done} of {len(names)} shaders checked, {len(failures)} failed, {last - t0:.0f} s')
|
||||
if out:
|
||||
Path(out).write_text(''.join(f'{k} {o or "-"} {p}\n' for k, o, p in failures), encoding='utf-8')
|
||||
return len(names), failures
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
|
||||
ap.add_argument('cache', help='the shader cache folder (<install folder>/shader-cache)')
|
||||
ap.add_argument('--every', type=int, default=1, help='check every N-th shader only')
|
||||
ap.add_argument('--jobs', type=int, default=0, help='threads (default: the CPUs, at most 32)')
|
||||
ap.add_argument('--no-val', action='store_true', help="the game's own checks only, without spirv-val")
|
||||
ap.add_argument('--out', help='write one line per failed shader to this file')
|
||||
args = ap.parse_args()
|
||||
folder = Path(args.cache)
|
||||
if not folder.is_dir():
|
||||
print(f'{folder}: not a folder', file=sys.stderr)
|
||||
return 2
|
||||
val = None if args.no_val else find_spirv_val()
|
||||
if not args.no_val and not val:
|
||||
print("spirv-val not found (MSYS2: pacman -S mingw-w64-ucrt-x86_64-spirv-tools): the game's own checks only")
|
||||
t0 = time.monotonic()
|
||||
checked, failures = validate(folder, args.every, args.jobs, val, args.out)
|
||||
if not checked:
|
||||
print(f'{folder}: no translated shaders (no {psc.PACK} and no loose .spv files)', file=sys.stderr)
|
||||
return 2
|
||||
refused = sum(1 for f in failures if not f[2].startswith('spirv-val: '))
|
||||
print(f'{checked} shaders checked in {time.monotonic() - t0:.0f} s: {checked - len(failures)} passed, {refused} '
|
||||
f"the game would translate again (its own checks), {len(failures) - refused} refused by spirv-val"
|
||||
+ ('' if val else ' (not run)'))
|
||||
groups = Counter(group_of(f[2]) for f in failures)
|
||||
for text, n in groups.most_common(20):
|
||||
first = next(f for f in failures if group_of(f[2]) == text)
|
||||
print(f' {n:6d} {text} (first: {first[0][:16]}, {first[1] or "no file"})')
|
||||
if len(groups) > 20:
|
||||
print(f' ... and {len(groups) - 20} more kinds; --out FILE lists every failed shader')
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user