Merge the crash and memory fixes: a clean stop on a lost device, a commit gate for background pipeline builds, a start-up memory warning, and a retry for a texture whose decode had no scratch memory.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 04:49:50 -05:00
28 changed files with 986 additions and 62 deletions

View File

@@ -150,7 +150,7 @@ static int gpu_decode_ar(VkCtx *c, const uint8_t *src, size_t src_bytes, const V
vmaDestroyBuffer(c->vma, rb, ra), vmaDestroyBuffer(c->vma, ib, ia);
vk_defer_image(im.img, im.alloc);
vk_upload_flush(1);
return ok;
return ok > 0;
}
// The two regions as tight blocks: n0 blocks of level 0, then n1 of level 1.
@@ -240,7 +240,7 @@ static int gpu_bc(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t
vmaDestroyBuffer(c->vma, ib, ia);
vk_defer_image(im.img, im.alloc);
vk_upload_flush(1);
return ok;
return ok > 0;
}
// One BC3 block to 16 RGBA8 texels, as the format defines it (565 endpoints widened by bit replication; the colour

View File

@@ -51,7 +51,7 @@ static const char *const known[] = {
"HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE",
"HDR_HUE_CLIP", "HDR_LIVE", "HEAP_LAZY", "HDR_MATCH_SLOPE", "HDR_MIDGRAY",
"HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_TONE_SLOPE", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
"BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "OQ_BATCH", "GATE_WAIT_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
"BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "BUILD_FREE_MB", "BUILD_RESERVE_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "OQ_BATCH", "GATE_WAIT_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MEM_NEED_MB", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
"MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOUSE_TRACE", "MOVIE_HOST", "MOVIE_PACE",
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
"MV_SURVEY",
@@ -74,7 +74,7 @@ static const char *const known[] = {
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL",
"RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VA_REPORT",
"VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC",
"VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT",
"COOP", "COOP_NAME", "COOP_SERVER", "COOP_PORT", "COOP_ROOM", "COOP_AUTO", "COOP_DROP", // co-op (coop.c); the last three have no menu row
"COOP_PROGRESS", "COOP_ITEMS", "COOP_SONGS", "COOP_QUEST", "COOP_KEYS", "COOP_RUPEES", "COOP_AMMO", // the room's rules (coop_game.c)
"COOP_TEST_SET", // test: sets flags by hash at given frames (coop_game.c); no menu row
@@ -126,6 +126,7 @@ static const struct { const char *key, *section; } sections[] = {
{"DYN_RES", "performance"}, {"DYN_RES_TARGET", "performance"}, {"DYN_RES_STEP", "performance"},
{"RECORDER_CORE", "performance"}, {"RECORDER_PRIO", "performance"}, {"BIND_REUSE", "performance"},
{"DYN_DELTA", "performance"}, {"STG_ARENA", "performance"}, {"SUBMIT_BATCH", "performance"},
{"BUILD_FREE_MB", "performance"}, {"BUILD_RESERVE_MB", "performance"}, {"MEM_NEED_MB", "diagnostics"},
{"ASTC_GPU_DESWIZZLE", "performance"}, {"ASTC_SCRATCH_KEEP", "performance"}, {"ASTC_ARENA", "performance"}, {"POOL_CREATE_BUDGET", "performance"},
{"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"},
{"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"},
@@ -142,7 +143,7 @@ static const struct { const char *key, *section; } sections[] = {
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"},
};
// The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a
@@ -281,6 +282,8 @@ static const CfgMeta metas[] = {
{"LIB_PREBUILD_BIG_BG", "1", "0|1", RESTART}, {"PIPE_FEEDBACK", "1", "0|1|2", RESTART},
{"PIPELINE_CACHE_MAX_MB", "512", "0..65536", RESTART}, {"PIPE_BINARIES", "0", "0|1", RESTART}, {"ONESHOT_WAIT_MS", "30000", "0..60000", RESTART}, {"ONESHOT_BUDGET", "1", "0|1", RESTART}, {"PIPE_BINARIES_MAX_MB", "1024", "0..65536", RESTART}, {"BIG_BUILD_FREE_MB", "8192", "0..262144", RESTART},
{"BIG_BUILD_MAX_MB", "4096", "0..262144", RESTART},
{"BUILD_FREE_MB", "4096", "0..1048576", RESTART}, {"BUILD_RESERVE_MB", "64", "0..65536", RESTART}, // vk_pipeline.c gate_enter
{"MEM_NEED_MB", "24576", "0..1048576", RESTART}, // memory.c mem_start_note; the default is mem_need.h MEM_NEED_DEFAULT_MB
// all|<count>: the compute pipelines the boot keeps, those dispatched most recently (vk_pipeline.c, compute-used.bin)
{"COMPUTE_KEEP", "512", NULL, RESTART},
{"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again
@@ -311,6 +314,7 @@ static const CfgMeta metas[] = {
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha
{"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only
{"VA_REPORT", "0", "0|1|2", RESTART}, // vk_vaddr.c GPU address history for a lost device's report; 2: call stacks
{"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end
{"INPUT_RECORD", "", NULL, RESTART},
};

View File

@@ -59,14 +59,13 @@ void rt_fatal_notes(void) {
__attribute__((weak)) void fs_save_crash_wait(void) {} // fs_host.c; builds without it (the unit checks) have no saves
void rt_fatal(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
// The fatal line, the report's notes and what has to be flushed, then the exit with `code`.
static void fatal_end(int code, const char *fmt, va_list ap) __attribute__((noreturn));
static void fatal_end(int code, const char *fmt, va_list ap) {
rt_log_sync(); // main.c: the lines queued for the log thread, then these, written here
rt_log_raw("FATAL: ", 7);
rt_vlogf(fmt, ap);
rt_log_raw("\n", 1);
va_end(ap);
// A failed allocation (malloc, realloc, VirtualAlloc) means the commit limit was reached: this line tells a leak of
// ours (process private bytes near the limit) from a machine short of page file.
double proc, used, limit;
@@ -81,7 +80,19 @@ void rt_fatal(const char *fmt, ...) {
rtss_limit_restore();
fflush(stdout);
fflush(stderr);
_Exit(70);
_Exit(code);
}
void rt_fatal(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
fatal_end(RT_FATAL_EXIT, fmt, ap);
}
void rt_fatal_code(int code, const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
fatal_end(code, fmt, ap);
}
static const RtModule *module_at(uint64_t pc) {

View File

@@ -51,6 +51,12 @@ RtDispatchStats rt_dispatch_stats(void);
// Runtime fatal error: message to stderr, then exit. Used by the dispatcher and the stubs.
void rt_fatal(const char *fmt, ...) __attribute__((noreturn, format(printf, 1, 2)));
// The same end with an exit code of its own, for a failure that a script or the launcher tells from the others by it
// (vk_device.c vk_device_lost_check ends with RT_LOST_EXIT). rt_fatal ends with RT_FATAL_EXIT; 71 is the crash report's
// (main.c crash_handler).
#define RT_FATAL_EXIT 70
#define RT_LOST_EXIT 72
void rt_fatal_code(int code, const char *fmt, ...) __attribute__((noreturn, format(printf, 2, 3)));
// Lines of context for a fatal error's report, which rt_fatal and the crash report (main.c crash_handler) write after
// their own: fn logs one with rt_log. It runs on the failing thread, which may hold any lock, so it reads only what
// needs none. At most four; adding one twice keeps one. rt_fatal_notes runs them, once per process.

View File

@@ -319,7 +319,7 @@ static void flip_locked(void) {
VkResult r = vk_dlssg_active() ? vk_dlssg_present(vk_ctx()->gfx_queue, &pi)
: vk_fg_active() ? vk_fg_present(&pi) : vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi);
vk_queue_unlock();
vk_device_fault_report(r);
vk_device_lost_check(r, "the present");
if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1);
}
@@ -996,7 +996,7 @@ static void ng_present(VkSwapchainKHR sc, uint32_t idx) {
vk_queue_lock();
VkResult r = vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi);
vk_queue_unlock();
vk_device_fault_report(r);
vk_device_lost_check(r, "the present of a generated frame");
if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1);
__atomic_add_fetch(&ng.presents, 1, __ATOMIC_RELAXED);
}
@@ -1061,7 +1061,7 @@ static int ng_show(VkImage img, int set, int slot, uint64_t n) {
vk_queue_lock();
r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
if (r != VK_SUCCESS) { vk_device_fault_report(r); return 0; }
if (r != VK_SUCCESS) { vk_device_lost_check(r, "the submit of a generated frame"); return 0; }
ng_present(ng.job.sc, j);
return 1;
}
@@ -1331,6 +1331,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w
static VkFormat last = VK_FORMAT_UNDEFINED;
if (src && fmt != last) last = fmt, rt_log("gfx: the game presents %dx%d images of format %d", w, h, (int)fmt);
}
if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): nothing touches the device
gfx_window_start();
VkCtx *c = vk_ctx();
plat_rmutex_lock(&g_pl);
@@ -1359,8 +1360,8 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w
: vk_fg_active() ? vk_fg_acquire(g_sc, 1000000000ull, g_acquire[slot], &idx)
: vkAcquireNextImageKHR(c->device, g_sc, 1000000000ull, g_acquire[slot], VK_NULL_HANDLE, &idx);
}
vk_device_fault_report(r);
// lost, not ready: skip the frame. Acquires that keep timing out (1 s each) mean a swapchain nothing will free an
vk_device_lost_check(r, "the acquire of a swapchain image");
// not ready: skip the frame. Acquires that keep timing out (1 s each) mean a swapchain nothing will free an
// image of again: three in a row rebuild it, where the picture used to stop for good.
static uint32_t n_timeouts;
if (r == VK_TIMEOUT || r == VK_NOT_READY) {
@@ -1554,7 +1555,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w
g_flip_idx = idx;
if (!defer) flip_locked();
} else {
vk_device_fault_report(r);
vk_device_lost_check(r, "the present's submit");
rt_log("gfx: present submit failed (%d)", (int)r);
// The fence was reset after the acquire and the acquire's semaphore is signalled: left so, this slot's fence
// wait would time out (5 s) at every later present and the picture stop for good. An empty batch takes the
@@ -1594,6 +1595,7 @@ void vk_present_blit(VkImage src, VkImageLayout layout, VkFormat fmt, int w, int
void gfx_present_boot(void) { present_image(VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED, VK_FORMAT_UNDEFINED, 1920, 1080, NULL, 0); }
void vk_present_flip(void) {
if (vk_device_is_lost()) return;
plat_rmutex_lock(&g_pl);
flip_locked();
plat_rmutex_unlock(&g_pl);

View File

@@ -191,6 +191,8 @@ typedef struct { uint64_t base, size; uint32_t state, attr, perm, ipc, device, p
void mem_init(void); // reserve every region; on a collision re-run the process
// (up to 20 times, NX2NAVI_REEXEC), then fatal
const RtGuestLayout *mem_layout(void);
int mem_start_note(int probe); // a game's start: a warning line when the free commit is
// below the game's need (mem_need.h); 1 when it is
void mem_guest_census(uint64_t frame); // NX2NAVI_TRACE=perf: committed bytes per guest region
void mem_lazy_heap_report(uint64_t frame); // NX2NAVI_TRACE=perf: the lazy heap's faults since the last line
uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr); // 0x01
@@ -428,6 +430,7 @@ int probe_discord(void); // discord_rpc.c
int probe_loader(void), probe_kernel(void), probe_ipc(void), probe_services(void), probe_fs(void);
int probe_gfx_abi(void); // gfx_abi_probe.c
int probe_vkdevice(void); // vk_device.c
int vk_sync_probe_devlost(void); // vk_sync.c: --probe-devlost
int probe_nvn(void), probe_nvn_tex(void); // nvn_probe.c, nvn_tex_probe.c
int probe_window(void); // gfx_window_probe.c
int probe_shader(void); // gfx_shader.c
@@ -440,6 +443,7 @@ int vk_pipeline_prepare_shaders(void); // vk_pipeline.c: --p
int vk_pipeline_warm_shaders(void); // vk_pipeline.c: --warm-shaders (tools/build_game.py)
int vk_pipeline_prepare_pipelines(void); // vk_pipeline.c: --prepare-pipelines (the shared pipeline list)
int vk_pipeline_probe_prewarm(void); // vk_pipeline.c: --probe-prewarm
int vk_pipeline_probe_buildgate(void); // vk_pipeline.c: --probe-buildgate
// vk_device.c: the drivers' own shader caches beside the exe (MESA_SHADER_CACHE_DIR, on NVIDIA's Linux driver
// __GL_SHADER_DISK_CACHE_PATH and _SIZE) in the environment, once; before any thread can load a Vulkan driver
void vk_driver_cache_env(void);

View File

@@ -534,7 +534,7 @@ static int usage(const char *argv0) {
"--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | "
"--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | "
"--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | --probe-coop | "
"--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0);
"--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-buildgate | --probe-memneed | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0);
return 2;
}
@@ -649,6 +649,7 @@ int main(int argc, char **argv) {
if (cfg.seconds && mode) return usage(argv[0]);
log_open(!mode);
if (!mode) {
mem_start_note(0); // memory.c: a warning when the machine has less memory free than the game takes
cfg.boot = cfg.seconds > 0; // --seconds starts the game like --boot, with its checkpoints
vk_driver_cache_env(); // before the threads below and rt_boot's, any of which may load the Vulkan driver
discord_init(); // the Rich Presence thread (discord_rpc.c); it reads nx2navi.ini itself and idles while off
@@ -685,6 +686,7 @@ int main(int argc, char **argv) {
if (!strcmp(mode, "--probe-zstd")) return probe_zstd();
if (!strcmp(mode, "--probe-input")) return probe_gamepad();
if (!strcmp(mode, "--probe-vkdevice")) return probe_vkdevice();
if (!strcmp(mode, "--probe-devlost")) return vk_sync_probe_devlost();
if (!strcmp(mode, "--probe-movie")) { // the AV1 decoder tells the graphics back end of each frame it writes: none here
cfg.gpu = "null";
return probe_movie();
@@ -693,6 +695,8 @@ int main(int argc, char **argv) {
if (!strcmp(mode, "--warm-shaders")) return vk_pipeline_warm_shaders();
if (!strcmp(mode, "--prepare-pipelines")) return vk_pipeline_prepare_pipelines();
if (!strcmp(mode, "--probe-prewarm")) return vk_pipeline_probe_prewarm();
if (!strcmp(mode, "--boot")) { cfg.boot = 1; rt_boot(); return 0; }
if (!strcmp(mode, "--probe-buildgate")) return vk_pipeline_probe_buildgate();
if (!strcmp(mode, "--probe-memneed")) { mem_start_note(1); return 0; }
if (!strcmp(mode, "--boot")) { mem_start_note(0); cfg.boot = 1; rt_boot(); return 0; }
return usage(argv[0]);
}

25
runtime/mem_need.h Normal file
View File

@@ -0,0 +1,25 @@
// Whether the machine has the memory the game needs, asked once at the start (memory.c mem_start_note). Own header so
// tests/mem_need_check.c checks it alone.
//
// What counts is commit: memory the system has promised to programs, out of RAM plus page file (on Linux the tightest
// of the limits platform.h plat_commit_info reads). An allocation fails when the commit in use reaches the limit,
// however much RAM is free, and on Windows the video driver charges the game's video memory to it too. Measured in
// play (RTX 5090, 3840x2160, three sessions of 19,000 to 50,000 frames on 2026-11-11): this process held 23,910 to
// 24,429 MB of commit, flat over the session, and the system stood at 65,777 to 65,914 of 73,154 MB with the other
// programs' 41,500 MB. A machine that cannot give the game that much pages, drops textures to make room (vk_texture.c
// commit_is_low) or loses the device.
#pragma once
// NX2NAVI_MEM_NEED_MB's default: the commit the game took in the sessions above, as 24 GiB.
#define MEM_NEED_DEFAULT_MB 24576.0
// The commit in MB the game can still get: what is free, and what this process already holds of its need.
static inline double mem_need_room(double proc_mb, double used_mb, double limit_mb) {
return limit_mb - used_mb + (proc_mb > 0 ? proc_mb : 0);
}
// 1 when the game's need does not fit. need_mb 0 or less: not asked. A limit below 0 is an unknown one: not short.
static inline int mem_need_short(double proc_mb, double used_mb, double limit_mb, double need_mb) {
if (need_mb <= 0 || limit_mb < 0) return 0;
return mem_need_room(proc_mb, used_mb, limit_mb) < need_mb;
}

View File

@@ -14,6 +14,7 @@
#include "cfg.h"
#include "host.h"
#include "layout.h"
#include "mem_need.h"
#include "platform/platform.h"
#define PAGE 0x1000ull
@@ -951,6 +952,29 @@ void mem_init(void) {
const RtGuestLayout *mem_layout(void) { return &L; }
// The start of a game (main.c, before the boot): one line when the machine's free commit is below what the game takes
// in play (mem_need.h; NX2NAVI_MEM_NEED_MB, 0 no check), so a log that ends in a failed allocation, dropped textures
// or a lost device says at its top that the machine was short before the game took anything. 1 when it was.
// probe: --probe-memneed, which prints the numbers either way.
int mem_start_note(int probe) {
double proc, used, limit, need = strtod(cfg_value("NX2NAVI_MEM_NEED_MB"), NULL);
plat_commit_info(&proc, &used, &limit);
int is_short = mem_need_short(proc, used, limit, need);
if (probe)
printf("probe-memneed: system commit %.0f of %.0f MB in use, this process %.0f MB, the game needs %.0f MB "
"(NX2NAVI_MEM_NEED_MB): %s\n", used, limit, proc, need,
need <= 0 ? "not checked" : limit < 0 ? "no limit known" : is_short ? "short" : "enough");
if (!is_short) return 0;
char note[400] = "";
plat_commit_note(note, sizeof note);
rt_log("mem: WARNING: this machine has %.0f MB of memory left for the game and the game takes about %.0f MB in play "
"(system commit: %.0f of %.0f MB in use; the limit is RAM plus page file%s%s). Short of memory the game "
"stutters, loses textures or stops with an out-of-memory or lost-device error. Closing other programs or a "
"larger page file gives it room (mem_need_mb = 0 in nx2navi.ini turns this warning off)",
mem_need_room(proc, used, limit), need, used, limit, note[0] ? "; " : "", note);
return 1;
}
// A range inside one watched reservation (stack and heap block, or alias region) only.
static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, int reset) {
if (!max || !size) return 0;

View File

@@ -857,6 +857,47 @@ static int detail_case(pool_t *pool, uint64_t off, uint64_t room, uint32_t size,
return 0;
}
// An ASTC upload whose GPU decode gets no scratch memory (vk_astc.c scratch_get, as when the video memory is used up;
// forced here through vk_astc_scratch_fail) does not count as done: the texture stays "not uploaded", is left alone for
// the frames of the wait, and the first lookup after them uploads it, after which the image holds the storage's blocks.
// Skipped (0) where ASTC is not decoded on the GPU: there is no scratch buffer then.
static int detail_retry(pool_t *pool, uint64_t off, uint64_t room) {
if (!vk_astc_gpu()) return 0;
tex_t t = make_tex(pool, off, 0x80, 128, 128, 1, 24); // ASTC 8x8
GCHECK(t.L.size <= room && off + room <= pool->size, "the retry case's storage does not fit the probe's pool");
if (detail_fill(&t, 1, 0)) return 1;
uint8_t want[4];
detail_color(0, 0, 0, want);
uint64_t r0 = vk_texture_upload_retries();
uint32_t ups = 0;
int ch, pw;
vk_astc_scratch_fail(1);
const VkTexImage *img = vk_texture_get(&t);
vk_astc_scratch_fail(0);
GCHECK(img && img->mirrored, "the retry case's texture image");
if (img->format != VK_FORMAT_R8G8B8A8_UNORM && img->format != VK_FORMAT_BC3_UNORM_BLOCK) return 0; // native ASTC
vk_texture_debug(img, &ups, &ch, &pw);
GCHECK(vk_texture_upload_retries() == r0 + 1 && ups == 1,
"an ASTC upload whose decode had no memory was not left for another try (%llu tries noted, %u uploads)",
(unsigned long long)(vk_texture_upload_retries() - r0), ups);
img = vk_texture_get(&t); // inside the wait: left alone
vk_texture_debug(img, &ups, &ch, &pw);
GCHECK(ups == 1, "the upload was tried again inside its wait (%u uploads)", ups);
__atomic_fetch_add(&gfx_counters.frames, 30, __ATOMIC_RELAXED); // vk_texture.c UP_RETRY_FRAMES
img = vk_texture_get(&t);
vk_texture_debug(img, &ups, &ch, &pw);
GCHECK(ups == 2 && vk_texture_upload_retries() == r0 + 1, "the upload was not tried again after its wait (%u uploads)", ups);
GCHECK(detail_level_is(img, 0, 0, 128, 128, want), "after the second try the image does not hold the storage's blocks");
img = vk_texture_get(&t); // uploaded now: nothing more
vk_texture_debug(img, &ups, &ch, &pw);
GCHECK(ups == 2, "an uploaded texture was uploaded again (%u uploads)", ups);
g_gpu_checks += 4;
vk_texture_release(&t);
printf("probe-vkmem: an ASTC upload whose decode had no scratch memory is tried again after its wait and then holds "
"its blocks\n");
return 0;
}
static int test_detail(pool_t *pool) {
const char *v = cfg_value("NX2NAVI_TEXTURE_DETAIL");
int most = !strcmp(v, "full") ? 0 : !strcmp(v, "reduced") ? 1 : !strcmp(v, "low") ? 2 : vk_texture_detail_auto();
@@ -864,7 +905,8 @@ static int test_detail(pool_t *pool) {
// ones the rule leaves whole: 128x128, and 512x512 without smaller levels
if (detail_case(pool, 0, 0x80000, 1024, 6, 1, 20, most > 1 ? 2 : (uint32_t)most) ||
detail_case(pool, 0x80000, 0x40000, 256, 4, 3, 21, most ? 1 : 0) ||
detail_case(pool, 0xc0000, 0x10000, 128, 4, 1, 22, 0) || detail_case(pool, 0xd0000, 0x20000, 512, 1, 1, 23, 0))
detail_case(pool, 0xc0000, 0x10000, 128, 4, 1, 22, 0) || detail_case(pool, 0xd0000, 0x20000, 512, 1, 1, 23, 0) ||
detail_retry(pool, 0xf0000, 0x10000))
return 1;
// the sampler of a texture made without levels: an open LOD clamp is the slot's sampler, a closed one moves down
samp_par sp;

View File

@@ -80,13 +80,25 @@ VkCtx *vk_ctx(void); // NULL until vk_init() succeeded
int vk_init(void); // idempotent; 1 on success, 0 on failure (reason logged with rt_log)
void vk_shutdown(void);
int vk_selftest(void); // 0 on success
// Call with the result of a failed queue submit, present or wait before stopping on it: VK_ERROR_DEVICE_LOST logs
// what VK_EXT_device_fault reports (description, faulting addresses with their precision, vendor codes) once per
// process, after the GPU breadcrumbs (vk_crumbs_report). Any other result returns at once.
void vk_device_fault_report(VkResult r);
// Call with the result of every queue submit, present, acquire or wait. Any result but VK_ERROR_DEVICE_LOST returns at
// once, and the caller goes on with its own handling. A lost device ends the run here, in one place: the first thread
// to arrive sets vk_lost, logs the GPU breadcrumbs (vk_crumbs_report) and what VK_EXT_device_fault reports
// (description, faulting addresses with their precision, vendor codes), then ends the process as rt_fatal does (the
// log written, a save commit under way finished) with exit code RT_LOST_EXIT (dispatch.h). `where` names the call for
// the fatal line. Every later thread stops in this call and never returns to touch the device; the process ends
// within LOST_EXIT_MS (vk_device.c) even when the first thread hangs on a lock one of them holds.
void vk_device_lost_check(VkResult r, const char *where);
// 1 from the moment a thread saw the device lost. The paths that would touch the device (the replay and its submit,
// the uploads, the present, the completion thread's callbacks) read it where they start and return without doing so.
extern volatile int32_t vk_lost;
static inline int vk_device_is_lost(void) { return __atomic_load_n(&vk_lost, __ATOMIC_RELAXED); }
// Asks the device whether it is lost with a call that has a result and needs no lock (the status of a fence nothing
// signals), and ends the run through vk_device_lost_check when it is. The completion thread calls it before it runs
// callbacks: a lost device may report the submissions' timeline as reached (vk_sync.c completion_main).
void vk_device_lost_poll(void);
// ---- GPU address history (vk_vaddr.c, NX2NAVI_VA_REPORT=1, 2 with call stacks): VK_EXT_device_address_binding_report's
// reports, kept so vk_device_fault_report can name the objects that held a faulting address.
// reports, kept so the lost device report (vk_device.c device_fault_report) can name the objects that held a faulting address.
int vk_vaddr_wanted(void);
void vk_vaddr_start(VkInstance instance); // create_instance, after vkCreateInstance
void vk_vaddr_device(int supported); // create_device: the driver reports address bindings
@@ -157,7 +169,7 @@ const VkPassMark *vk_pass_mark_record(uint32_t id); // the ring record of mark
// The report's text for a started and a finished mark, three lines without a trailing newline: which marks (source:
// the extension they came from), then each with its ring record. vk_pass_marks_report logs it; the probe checks it.
void vk_pass_marks_format(const char *source, uint32_t started, uint32_t finished, char *o, size_t size);
void vk_pass_marks_report(void); // vk_device_fault_report: the AMD words and the NVIDIA checkpoints, formatted
void vk_pass_marks_report(void); // vk_device.c device_fault_report: the AMD words and the NVIDIA checkpoints, formatted
// ---- window and present (gfx_present.c). Host threads only.
// vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled
@@ -448,9 +460,12 @@ typedef struct { VkDeviceSize src; uint32_t w, h, slices, level, layer; VkOffset
// into img (VK_IMAGE_LAYOUT_GENERAL). The caller records its usual barriers around an image copy before and after the
// call. readback (optional, host-visible, TRANSFER_DST): also gets the decoded texels, the regions one after the other.
// bc: img is BC3 (TexVkFormat.astc_bc); the texels are encoded to BC3 blocks before the copy, and readback is ignored.
// 0 when the decoder is unavailable or the scratch buffer cannot be allocated (nothing recorded).
// 1 when recorded. 0 when the decoder is unavailable or cannot take these regions, and -1 when its scratch buffer
// cannot be allocated: nothing is recorded in either case, and after -1 the same call can succeed once memory is free.
int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, const VkAstcRegion *rg, int n, uint32_t bw,
uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback);
void vk_astc_scratch_fail(int n); // a test's hook: the next n scratch buffer requests fail (tex_probe.c)
uint64_t vk_texture_upload_retries(void); // uploads left to be tried again: their decode had no memory (vk_texture.c upload)
VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh); // source bytes the region reads
void vk_astc_stats(uint64_t *dispatches, uint64_t *texels); // since start
uint64_t vk_astc_scratch_bytes(void); // the decoder's shared scratch now

View File

@@ -466,7 +466,12 @@ static int scratch_keep(void) {
// A scratch of at least need bytes for recording into cb; *reused: earlier work (in cb, or with the keep rule in an
// earlier batch) may still use it. 0 on failure.
// A test's hook (vk.h): the next n requests for the scratch buffer fail, as when the video memory is used up.
static volatile int32_t scratch_fail_n;
void vk_astc_scratch_fail(int n) { plat_xchg32(&scratch_fail_n, n); }
static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int *reused) {
if (plat_cas32(&scratch_fail_n, 0, 0) > 0 && plat_add32(&scratch_fail_n, -1) >= 0) return VK_NULL_HANDLE;
int keep = scratch_keep();
plat_mutex_lock(&g_lock);
uint64_t now = kern_now_ns();
@@ -573,9 +578,9 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons
VkBuffer sb = scratch_get(c, cb, cap_in + cap_out + cap_bc, &reused);
if (!sb) {
if (plat_inc32(&n_fail) <= 16)
rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded",
rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded for now",
(unsigned long long)(cap_in + cap_out + cap_bc));
return 0;
return -1; // no memory at this moment: the caller tries again later (vk_texture.c upload)
}
plat_mutex_lock(&g_lock);
int need_tab = !G.tab_uploaded;

37
runtime/vk_build_gate.h Normal file
View File

@@ -0,0 +1,37 @@
// The memory gate of the background pipeline builds (vk_pipeline.c gate_enter): whether a build may start, given the
// system's commit and what the builds already under way were promised. Own header so tests/build_gate_check.c checks
// it without Vulkan.
//
// A pipeline build runs the driver's shader compiler in this process, and what the compiler takes while it works is
// committed memory like any other. A gate that only reads the commit in use sees none of it for the builds that have
// just started: sixteen builders that each find "enough room" start together and take sixteen shares of the same
// room. So every build that starts reserves an estimate of what its compile takes, the gate counts the reservations
// as if they were in use already, and a build that does not fit is refused: it does not start, and its caller leaves
// it to a later moment or to the draw that first needs the pipeline. A machine at its commit limit fails allocations
// (Windows) or pages until the driver gives the device up; a pipeline built later costs one hitch.
#pragma once
typedef struct {
double free_mb; // commit headroom in MB that stays free of background builds; 0 or less: nothing is refused
double reserved_mb; // the estimates of the builds under way
} BuildGate;
// 1 when a build estimated at est_mb may start with used_mb of limit_mb of the system's commit in use. A limit below
// 0 is an unknown one (platform.h plat_commit_info): nothing is refused.
static inline int build_gate_admits(const BuildGate *g, double used_mb, double limit_mb, double est_mb) {
if (g->free_mb <= 0 || limit_mb < 0) return 1;
return limit_mb - used_mb - g->reserved_mb >= g->free_mb + est_mb;
}
// Reserves est_mb for a build that starts now; 0 when it is refused, and then nothing is reserved.
static inline int build_gate_enter(BuildGate *g, double used_mb, double limit_mb, double est_mb) {
if (!build_gate_admits(g, used_mb, limit_mb, est_mb)) return 0;
g->reserved_mb += est_mb;
return 1;
}
// The build that reserved est_mb is over.
static inline void build_gate_leave(BuildGate *g, double est_mb) {
g->reserved_mb -= est_mb;
if (g->reserved_mb < 0) g->reserved_mb = 0;
}

View File

@@ -25,6 +25,10 @@ static uint32_t g_gfx_queues = 1; // queues the graphics family offers (pick_d
static VkCtx g_vk;
static int g_ready;
static PlatMutex g_lock = PLAT_MUTEX_INIT;
// The fence vk_device_lost_poll asks about. Nothing submits with it, so its status is "not ready" for as long as the
// device lives.
static VkFence lost_fence;
static PlatMutex lost_fence_lock = PLAT_MUTEX_INIT;
static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type,
const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) {
@@ -550,6 +554,8 @@ static void destroy_all(void) {
if (g_vk.device) vk_astc_release();
if (g_vk.device) vk_crumbs_release();
if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL);
if (lost_fence) vkDestroyFence(g_vk.device, lost_fence, NULL); // vk_device_lost_poll
lost_fence = VK_NULL_HANDLE;
vk_headroom_stop();
if (g_vk.vma) vmaDestroyAllocator(g_vk.vma);
if (g_vk.device) vkDestroyDevice(g_vk.device, NULL);
@@ -599,15 +605,9 @@ static const char *fault_address_type(VkDeviceFaultAddressTypeEXT t) {
}
}
// The query runs once per process. Threads that hit the loss at the same time wait on the lock until it is logged, so
// none of them reaches its rt_fatal and ends the process before the report is written.
void vk_device_fault_report(VkResult r) {
static PlatMutex lock = PLAT_MUTEX_INIT;
static int done;
if (r != VK_ERROR_DEVICE_LOST) return;
plat_mutex_lock(&lock);
if (done) { plat_mutex_unlock(&lock); return; }
done = 1;
// The report of a lost device: the breadcrumbs, the pass marks, the video memory and the driver's fault information.
// Once per process, on the thread that saw the loss first (vk_device_lost_check).
static void device_fault_report(void) {
vk_crumbs_report();
vk_pass_marks_report();
vk_mem_lost_report();
@@ -619,7 +619,6 @@ void vk_device_fault_report(VkResult r) {
rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point"
: "not enabled");
vk_vaddr_dump(); // the history still names what held a page another report (the kernel's log) gives
plat_mutex_unlock(&lock);
return;
}
VkDeviceFaultCountsEXT n = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT};
@@ -648,7 +647,53 @@ void vk_device_fault_report(VkResult r) {
vk_vaddr_dump();
free(ai);
free(vi);
plat_mutex_unlock(&lock);
}
// ---- the end of a run whose device is lost (vk.h vk_device_lost_check)
// A lost device stays lost: every later call on it fails or, worse, answers as if it had worked, and a driver that is
// called on from several threads after the loss can fault inside itself. So the first thread that sees the loss ends
// the run, and nothing else touches the device meanwhile: the paths that would read vk_lost and return, and a thread
// that gets the loss as a result of its own stops here for good.
// LOST_EXIT_MS bounds the end. The first thread writes the report and waits for a save commit (fs_host.c
// fs_save_crash_wait, 2 s at most); should it hang on a lock that a stopped thread holds, the watchdog ends the
// process with the same code, and the lines already written are in the log (rt_fatal_code wrote them itself).
#define LOST_EXIT_MS 10000
volatile int32_t vk_lost;
static void lost_wait_exit(void *arg) {
(void)arg;
for (int t = 0; t < LOST_EXIT_MS; t += 50) plat_sleep_ms(50);
_Exit(RT_LOST_EXIT);
}
void vk_device_lost_check(VkResult r, const char *where) {
if (r != VK_ERROR_DEVICE_LOST) return;
if (plat_xchg32(&vk_lost, 1)) lost_wait_exit(NULL); // another thread is ending the run: this one stops here
plat_thread_spawn(lost_wait_exit, NULL, "vk lost exit", 0);
device_fault_report();
rt_fatal_code(RT_LOST_EXIT, "vk: the GPU device was lost (seen at: %s). The run ends here; the lines above name what "
"the GPU was doing", where);
}
void vk_device_lost_poll(void) {
VkCtx *c = vk_ctx();
if (!c || vk_device_is_lost()) return;
VkFence f = __atomic_load_n(&lost_fence, __ATOMIC_ACQUIRE);
if (!f) {
plat_mutex_lock(&lost_fence_lock);
if (!lost_fence) {
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
VkFence n = VK_NULL_HANDLE;
VkResult r = vkCreateFence(c->device, &fi, NULL, &n);
if (r == VK_SUCCESS) __atomic_store_n(&lost_fence, n, __ATOMIC_RELEASE);
else vk_device_lost_check(r, "the fence that asks whether the device lives");
}
f = lost_fence;
plat_mutex_unlock(&lost_fence_lock);
if (!f) return;
}
vk_device_lost_check(vkGetFenceStatus(c->device, f), "a poll of the device");
}
int vk_selftest(void) {

View File

@@ -1664,6 +1664,7 @@ static void rec_claim(void) {
}
int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) {
if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): nothing is recorded
uint64_t in_ns = kern_now_ns();
perf_lock_timing_here(); // NX2NAVI_LOCK_TIME
if (prof_replay_tid != plat_thread_id()) perf_replay_thread(); // the profiler follows the thread replaying now

View File

@@ -390,7 +390,7 @@ static uint8_t *draw_prefix(uint8_t *q, prog_t *prog, const pool_t *pool, tex_t
// ---- pass marks part (vk_crumbs.c): one submission with a pass of 2 draws into the draw part's target, a dispatch and
// a pass of 3 draws into a second target. Once its fence is done, the buffer marks must name the last pass as both
// started and finished, the ring records must hold what the three were, and vk_pass_marks_format (the text
// vk_device_fault_report logs) must name it. Without VK_AMD_buffer_marker or with NX2NAVI_PASS_CRUMBS=0 it is skipped:
// vk_device.c device_fault_report logs) must name it. Without VK_AMD_buffer_marker or with NX2NAVI_PASS_CRUMBS=0 it is skipped:
// NVIDIA's checkpoints can only be read once the device is lost.
static char marks_text[2048];
static int marks_skipped;

View File

@@ -323,7 +323,7 @@ static void oq_read(VkDevice dev, uint32_t first, uint32_t n) {
VkResult r = vkGetQueryPoolResults(dev, oq_pool, first, n, sizeof OQB.v[0] * 2 * n, OQB.v, 2 * sizeof OQB.v[0],
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
if (r != VK_SUCCESS && r != VK_NOT_READY) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the read of occlusion query results");
rt_fatal("vk: occlusion query results failed (%d)", (int)r);
}
OQB.first = first, OQB.n = n;
@@ -365,6 +365,7 @@ static void oq_cb(void *arg) {
if (avail) sum += v;
else {
sum += 1;
vk_device_lost_poll(); // a query its submission never wrote: on a lost device the run ends here
LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[i]);
}
}

View File

@@ -2932,6 +2932,7 @@ static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags
// memory.c's fetch (frz_lock held there, so one at a time): [off, off + size) of the mirror h0 into the imported
// buffer h1, after all the GPU work handed over so far.
static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) {
if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): the bytes stay as they are
VkCtx *c = vk_ctx();
VkBufferCopy reg = {off, off, size};
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
@@ -2952,7 +2953,7 @@ static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) {
vk_queue_unlock();
if (r == VK_SUCCESS) r = vkWaitForFences(c->device, 1, &lz_fence, VK_TRUE, 10000000000ull);
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the submit or the wait that brings GPU writes down to guest memory");
rt_fatal("vk: bringing %llu bytes the GPU wrote down to guest memory failed (%d)", (unsigned long long)size, (int)r);
}
vkResetFences(c->device, 1, &lz_fence);
@@ -4109,6 +4110,7 @@ void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) {
}
uint64_t vk_upload_flush(int wait) {
if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): no submit, no wait
uint64_t f0 = kern_now_ns();
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
@@ -4125,7 +4127,7 @@ uint64_t vk_upload_flush(int wait) {
VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence);
vk_queue_unlock();
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the submit of an upload batch");
rt_fatal("vk: upload submit failed (%d)", (int)r);
}
if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight);
@@ -4138,7 +4140,7 @@ uint64_t vk_upload_flush(int wait) {
// loop would spin for good, holding the upload lock, instead of the run ending with the device's report
VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX);
if (w != VK_SUCCESS) {
vk_device_fault_report(w);
vk_device_lost_check(w, "the wait for an upload batch");
rt_fatal("vk: the wait for an upload failed (%d)", (int)w);
}
retire(c, 0);

View File

@@ -48,6 +48,7 @@
#include "nvn_hmap.h"
#include "platform/platform.h"
#include "tex.h"
#include "vk_build_gate.h"
#include "vk_gpa.h"
#include "vk_warm_hint.h"
@@ -3208,6 +3209,7 @@ static struct {
Job *low, *low_tail; // prebuild
uint32_t inflight; // queued or running, both queues
int low_running; // pool workers running a job taken from the low queue
int gate_shut; // pool workers the memory gate has refused a low job since it last admitted one
PlatCond idle; // with P.lock: no job queued or running
} A = {.idle = PLAT_COND_INIT};
@@ -3250,6 +3252,7 @@ static struct {
int warm; // vk_pipeline_prewarm owns the file (prebuild_start leaves it)
uint32_t left; // prebuild jobs queued or running
uint32_t built, failed, promoted;
uint32_t gate_ended; // boot builders the memory gate ended (warm_builder)
double t0; // now_ms() when the prebuild started
double until; // now_ms() after which the boot builders stop (comment at prewarm_run); 0 none
} K = {.io = PLAT_MUTEX_INIT};
@@ -3533,6 +3536,132 @@ static void lib_worker(void *arg) {
}
}
// ---- the memory gate of the background builds (vk_build_gate.h)
// Every pipeline built ahead of its first use passes here before it starts: the recorded pipelines the boot builds
// (warm_builder), the pool's low queue during play (pipe_worker: the rest of the recorded pipelines and the optimised
// builds behind a fast link), the compute pipelines and stage libraries of the boot (lb_worker), the library warm
// (lw_worker) and the large compute shaders (bg_main). A build a draw or a dispatch waits for never does: it is needed
// now.
// NX2NAVI_BUILD_FREE_MB (default 4096; 0 no check): the system commit headroom that stays free of them.
// NX2NAVI_BUILD_RESERVE_MB (default 64): what one build under way is taken to cost the driver's compiler while it
// runs; a large compute shader reserves GATE_BIG_MB. A build starts only when the headroom holds the part kept free,
// the reservations of the builds under way and its own.
// Measured on the machine this was written on (RTX 5090, 73,154 MB commit limit, warm driver cache): the boot's 3,898
// recorded pipelines on 16 threads raised the process commit by 3,054 MB in 2.2 s, and the 4,066 compute pipelines and
// libraries after them, together with the game's own start, by another 9,824 MB. A device was lost on that machine
// at 56,673 of 65,531 MB system commit right after such a prebuild.
// The commit is read at most every GATE_SAMPLE_MS: sixteen builders finishing a cached pipeline every half millisecond
// would otherwise ask the system thousands of times a second. What the builds took since the last reading is covered
// by their reservations.
#define GATE_BIG_MB 4096.0
#define GATE_SAMPLE_MS 100.0
#define GATE_WAIT_MS 30000.0 // how long a boot or warm builder waits for room before it leaves the rest to their first use
static struct {
PlatMutex lock;
BuildGate g;
double est, used, limit, at;
int init;
int off; // a preparation run (seed_pipelines): nothing is refused, there is no game to keep room for
uint64_t refused;
} GATE = {.lock = PLAT_MUTEX_INIT};
// 1 when a build estimated at est MB may start, and then est is reserved until gate_leave. est 0: an ordinary build
// (NX2NAVI_BUILD_RESERVE_MB), whose estimate *took returns for gate_leave.
static int gate_enter(double est, double *took) {
plat_mutex_lock(&GATE.lock);
if (!GATE.init) {
GATE.init = 1;
GATE.g.free_mb = strtod(cfg_value("NX2NAVI_BUILD_FREE_MB"), NULL);
GATE.est = strtod(cfg_value("NX2NAVI_BUILD_RESERVE_MB"), NULL);
if (GATE.est < 0) GATE.est = 0;
GATE.at = -GATE_SAMPLE_MS;
}
if (GATE.off) GATE.g.free_mb = 0;
double now = now_ms();
if (now - GATE.at >= GATE_SAMPLE_MS) {
double proc;
plat_commit_info(&proc, &GATE.used, &GATE.limit);
GATE.at = now;
}
if (est <= 0) est = GATE.est;
int ok = build_gate_enter(&GATE.g, GATE.used, GATE.limit, est);
if (!ok) GATE.refused++;
plat_mutex_unlock(&GATE.lock);
if (took) *took = est;
return ok;
}
static void gate_leave(double est) {
plat_mutex_lock(&GATE.lock);
build_gate_leave(&GATE.g, est);
plat_mutex_unlock(&GATE.lock);
}
// The gate's numbers for a log line that says why a builder stopped or waits.
static const char *gate_text(char *buf, size_t n) {
plat_mutex_lock(&GATE.lock);
snprintf(buf, n, "system commit %.0f of %.0f MB, %.0f MB reserved for builds under way, %.0f MB kept free "
"(NX2NAVI_BUILD_FREE_MB), %.0f MB a build (NX2NAVI_BUILD_RESERVE_MB)", GATE.used, GATE.limit,
GATE.g.reserved_mb, GATE.g.free_mb, GATE.est);
plat_mutex_unlock(&GATE.lock);
return buf;
}
// A boot or warm builder's wait for room: 1 with est reserved (*took), 0 when *stop was set or GATE_WAIT_MS passed
// without room, which the caller takes as the end of its work. who names the builder for the log's one line a wait.
static int gate_wait(const char *who, volatile int32_t *stop, double *took) {
double w0 = now_ms();
int waited = 0;
while (!gate_enter(0, took)) {
char g[256];
if (*stop) return 0;
if (!waited++) rt_log("vk_pipeline: %s: waiting for commit headroom (%s)", who, gate_text(g, sizeof g));
if (now_ms() - w0 > GATE_WAIT_MS) return 0;
for (int i = 0; i < 20 && !*stop; i++) plat_sleep_ms(100);
}
return 1;
}
// --probe-buildgate (host.h): the gate with this machine's commit and the settings in effect, without a device and
// without a build. It prints what the settings and the system give, asks for sixteen ordinary builds and one large
// one, and checks the answers against the same numbers: the builds admitted are those that fit with the earlier
// ones' reservations, a refused one reserves nothing, and a builder's wait ends at once when its stop flag is set.
// tests/build_gate_check.py runs it with build_free_mb at 0 (everything admitted) and at a value no machine has free
// (everything refused).
int vk_pipeline_probe_buildgate(void) {
double took = 0, big = 0;
int fails = 0, in = 0, want = 0;
int first = gate_enter(0, &took);
double free_mb = GATE.g.free_mb, used = GATE.used, limit = GATE.limit, est = took;
if (first) gate_leave(took);
printf("probe-buildgate: system commit %.0f of %.0f MB, %.0f MB kept free (NX2NAVI_BUILD_FREE_MB), %.0f MB a build "
"(NX2NAVI_BUILD_RESERVE_MB)\n", used, limit, free_mb, est);
double room = limit - used - free_mb;
for (int i = 0; i < 16; i++) {
want += free_mb <= 0 || limit < 0 || room - i * est >= est; // the commit sample stays for GATE_SAMPLE_MS
in += gate_enter(0, &took);
}
double reserved = GATE.g.reserved_mb;
int big_in = gate_enter(GATE_BIG_MB, &big);
int big_want = free_mb <= 0 || limit < 0 || room - reserved >= GATE_BIG_MB;
printf("probe-buildgate: %d of 16 ordinary builds admitted (%d fit), %.0f MB reserved; a large compute build beside "
"them %s\n", in, want, reserved, big_in ? "admitted" : "refused");
if (in != want) fails++, printf("probe-buildgate: FAIL: %d admitted, %d fit\n", in, want);
if (reserved != in * est) fails++, printf("probe-buildgate: FAIL: %.0f MB reserved for %d builds\n", reserved, in);
if (big_in != big_want) fails++, printf("probe-buildgate: FAIL: the large build was %s\n", big_in ? "admitted" : "refused");
if (big_in) gate_leave(big);
for (int i = 0; i < in; i++) gate_leave(est);
if (GATE.g.reserved_mb != 0) fails++, printf("probe-buildgate: FAIL: %.0f MB still reserved\n", GATE.g.reserved_mb);
volatile int32_t stop = 1;
double t0 = now_ms();
int got = gate_wait("probe", &stop, &took);
if (got) gate_leave(took);
if (got != first || now_ms() - t0 > 1000)
fails++, printf("probe-buildgate: FAIL: a stopped builder's wait gave %d after %.0f ms\n", got, now_ms() - t0);
printf("probe-buildgate: an ordinary build is %s now; %s\n", first ? "admitted" : "refused", fails ? "FAILED" : "passed");
return fails ? 1 : 0;
}
static void pipe_worker(void *arg) {
int me = (int)(intptr_t)arg;
plat_mutex_lock(&P.lock);
@@ -3554,6 +3683,22 @@ static void pipe_worker(void *arg) {
}
Job *j = A.head;
int lowrun = !j;
double held = 0; // the low job's reservation at the memory gate
if (!j && !gate_enter(0, &held)) {
// No room for a background build now (gate_enter). The job stays queued: a draw that needs its pipeline
// moves it to the draw queue (promote), which wakes this wait; otherwise the gate is asked again in 2 s.
if (!A.gate_shut++) {
char g[256];
rt_log("vk_pipeline: background pipeline builds wait for commit headroom (%s); a pipeline a draw needs "
"is still built at once", gate_text(g, sizeof g));
}
plat_cond_wait_ns(&work_cv, &P.lock, 2000000000ull);
continue;
}
if (!j && A.gate_shut) {
A.gate_shut = 0;
rt_log("vk_pipeline: background pipeline builds go on: the commit headroom is back");
}
if (j) {
if (!(A.head = j->next)) A.tail = NULL;
} else {
@@ -3585,6 +3730,7 @@ static void pipe_worker(void *arg) {
else A.low_tail = j;
A.low = j;
A.low_running -= lowrun;
if (lowrun) gate_leave(held);
continue;
} else if (f) {
vkDestroyPipeline(P.device, f, NULL);
@@ -3612,6 +3758,7 @@ static void pipe_worker(void *arg) {
if (!--K.left && !K.loading) prebuild_report();
}
A.low_running -= lowrun;
if (lowrun) gate_leave(held);
if (!--A.inflight) plat_cond_broadcast(&A.idle);
plat_cond_broadcast(&done_cv);
if (pipe && (!j->prebuild || j->demanded)) perf_count(PERF_CNT_PIPE_ASYNC); // a draw waited for it
@@ -4014,11 +4161,23 @@ double vk_pipeline_building_ms(const VkPipeState *s) {
static int id_cmp(const void *x, const void *y) { return memcmp(x, y, 32); }
// Builds queued prebuild jobs until the low queue is empty or K.until passes, beside the pipe_worker pool (same
// bookkeeping). Jobs left queued at K.until stay for the pool.
// bookkeeping). Jobs left queued at K.until stay for the pool. So do those the memory gate refuses (gate_enter): a
// builder it refuses ends, which leaves fewer builders compiling side by side while the others' reservations stand,
// and none when the machine has no room; the pool builds what is left during play, when the gate lets it.
static void warm_builder(void *arg) {
(void)arg;
plat_mutex_lock(&P.lock);
while (A.low && !(K.until && now_ms() >= K.until)) {
double held;
if (!gate_enter(0, &held)) {
if (!K.gate_ended++) {
char g[256];
rt_log("vk_pipeline: prewarm: the boot builds fewer recorded pipelines side by side, or none: the commit "
"headroom is short (%s). The rest are built in the background when there is room, or at their "
"first draw", gate_text(g, sizeof g));
}
break;
}
Job *j = A.low;
if ((A.low = j->next)) A.low->prev = NULL;
else A.low_tail = NULL;
@@ -4043,6 +4202,7 @@ static void warm_builder(void *arg) {
if (!--A.inflight) plat_cond_broadcast(&A.idle);
plat_cond_broadcast(&done_cv);
free(j);
gate_leave(held);
}
plat_mutex_unlock(&P.lock);
return;
@@ -4500,6 +4660,7 @@ static struct {
size_t n_keep; // e[0..n_keep): compute shaders whose pipeline stays (cu_keep)
volatile int64_t next;
volatile int32_t stop, running;
volatile int32_t short_mem; // the memory gate refused a build for GATE_WAIT_MS: the workers ended
int threads;
PlatThread *th[LB_MAX_THREADS];
uint8_t clamp, poly, minus_w, samples; // library state (GKey fields)
@@ -5103,12 +5264,15 @@ static void bg_queue(const GfxShader *cs, const uint8_t iid[32], uint64_t words)
// A build of one of these shaders took the process up by 1-3 GB on an RTX 5090 and four at once by 16-19 GB on an
// RTX 4090; started on a machine near its commit limit it can end the game. While the headroom is short the shader
// waits, and if the game dispatches it first it is built there, as without the background builder.
static int bg_room(double *used, double *limit) {
// The builds under way elsewhere count through their reservations at the memory gate (gate_enter), where this build
// then reserves GATE_BIG_MB of its own for as long as it runs, so the other background builds leave it its room.
static int bg_room(double *used, double *limit, double *held) {
static double need = -1;
if (need < 0) need = strtod(cfg_value("NX2NAVI_BIG_BUILD_FREE_MB"), NULL);
double proc;
plat_commit_info(&proc, used, limit);
return need <= 0 || *limit < 0 || *limit - *used >= need;
if (!(need <= 0 || *limit < 0 || *limit - *used >= need)) return 0;
return gate_enter(GATE_BIG_MB, held);
}
// NX2NAVI_BIG_BUILD_MAX_MB (default 4096; 0 no cap): how far the builder may raise the process commit before it stops
@@ -5188,8 +5352,9 @@ static void bg_main(void *arg) {
break;
}
int waited = 0, no_room = 0;
double w0 = now_ms();
while (!LB.stop && !bg_room(&used, &limit)) { // not under P.lock: a dispatch builds it meanwhile
double w0 = now_ms(), held = -1; // held: this build's reservation at the memory gate, -1 none
while (!LB.stop && !bg_room(&used, &limit, &held)) { // not under P.lock: a dispatch builds it meanwhile
held = -1;
if (!waited++)
rt_log("vk_pipeline: background compute build: waiting for commit headroom (system %.0f of %.0f MB, "
"NX2NAVI_BIG_BUILD_FREE_MB %s)", used, limit, cfg_value("NX2NAVI_BIG_BUILD_FREE_MB"));
@@ -5201,7 +5366,10 @@ static void bg_main(void *arg) {
}
plat_sleep_ms(2000);
}
if (LB.stop || no_room) break;
if (LB.stop || no_room) {
if (held >= 0 && !no_room) gate_leave(held); // stopped right after the gate let it in
break;
}
plat_mutex_lock(&P.lock);
int there = cs_flight_has(cs) || pipe_find(&k, sizeof k, h); // a dispatch got to it first
Layout *l = there ? NULL : compute_layout(cs, &B, &why);
@@ -5209,6 +5377,7 @@ static void bg_main(void *arg) {
if (l) BG.building = cs;
plat_mutex_unlock(&P.lock);
if (!l) {
if (held >= 0) gate_leave(held);
if (there) BG.there++;
else BG.failed++, rt_log("vk_pipeline: background compute build: %s", why ? why : "no layout");
continue;
@@ -5221,6 +5390,7 @@ static void bg_main(void *arg) {
VkPipeline pipe = compute_create(cs, l, 0);
own_cache = VK_NULL_HANDLE;
double ms = now_ms() - t0;
if (held >= 0) gate_leave(held);
// The pipeline stays: only a shader a run dispatched is queued here (lb_compute), and it is entered before
// the merge below, so a dispatch that comes meanwhile finds it.
plat_mutex_lock(&P.lock);
@@ -5432,6 +5602,18 @@ static void lb_worker(void *arg) {
plat_mutex_unlock(&LB.stat);
continue;
}
// The memory gate (gate_wait): without room for another build the worker waits, and after GATE_WAIT_MS it and
// the other workers end. What they leave is built at its first dispatch or draw, as what a run never recorded.
double held;
if (LB.short_mem || !gate_wait("stage library prebuild", &LB.stop, &held)) {
if (!LB.stop && !plat_xchg32(&LB.short_mem, 1)) {
char g[256];
rt_log("vk_pipeline: stage library prebuild: ends here, at %zu of %zu variants: no commit headroom for "
"another build (%s). The others are built when the game first uses them", i, LB.n,
gate_text(g, sizeof g));
}
break;
}
// A compute shader whose pipeline does not stay is read as a copy of this thread's own and released after its
// build: kept in the shader cache's memory, the 3,900 of an installed copy's list were 240 MB of the process's
// commit for shaders most runs never dispatch. A dispatch that does come reads its shader then (0.05 to
@@ -5453,12 +5635,13 @@ static void lb_worker(void *arg) {
else lb_skip_lib();
}
if (owned) gfx_shader_release_private(sh);
gate_leave(held);
plat_mutex_lock(&LB.stat);
LB.done++;
plat_mutex_unlock(&LB.stat);
}
if (!plat_dec32(&LB.running)) {
lb_report(LB.stop ? "stopped" : "done");
lb_report(LB.stop ? "stopped" : LB.short_mem ? "ended for want of commit headroom" : "done");
fb_log("after the stage library prebuild");
vk_mem_stage_log("the compute pipeline prebuild");
bg_start();
@@ -5504,6 +5687,7 @@ static struct {
size_t n;
volatile int64_t next, done;
volatile int32_t running, stop, started;
volatile int32_t short_mem; // the memory gate refused a build for GATE_WAIT_MS: the warm ended for this run
PlatThread *th[LW_MAX_THREADS + 1];
int threads, strikes;
uint32_t vendor; // the GPU's vendor id, for what a cold cache's log line adds (vk_warm_hint.h)
@@ -5539,13 +5723,26 @@ static void lw_worker(void *arg) {
for (;;) {
size_t i = (size_t)plat_inc64(&LW.next) - 1;
if (i >= LW.n || LW.stop) break;
// The memory gate (gate_wait): without room the warm waits, and after GATE_WAIT_MS it ends for this run.
// shader-warm.txt keeps saying "warming", so the next start goes on with it.
double held;
if (LW.short_mem || !gate_wait("library warm", &LW.stop, &held)) {
if (!LW.stop && !plat_xchg32(&LW.short_mem, 1)) {
char g[256];
rt_log("vk_pipeline: library warm: ends for this run at %lld of %zu prepared variants: no commit headroom "
"for another build (%s). The next start goes on with it", (long long)LW.done, LW.n,
gate_text(g, sizeof g));
}
break;
}
lw_build(&LW.e[i]);
gate_leave(held);
int64_t d = plat_inc64(&LW.done);
if (d % 8000 == 0)
rt_log("vk_pipeline: library warm: %lld of %zu prepared variants in %.0f s", (long long)d, LW.n,
(now_ms() - LW.t0) / 1000);
}
if (!plat_dec32(&LW.running) && !LW.stop) {
if (!plat_dec32(&LW.running) && !LW.stop && !LW.short_mem) {
lw_stamp("done", LW.strikes);
rt_log("vk_pipeline: library warm: done, %zu prepared variants in %.0f s on %d idle threads; new pipelines "
"link from the driver's cache from here on", LW.n, (now_ms() - LW.t0) / 1000, LW.threads);
@@ -5938,6 +6135,7 @@ static int seed_pipelines(const char *dir) {
}
K.loading = 0;
K.until = 0; // every one of them: no boot waits for this
GATE.off = 1; // and no game runs beside it: the memory gate (gate_enter) refuses nothing
plat_mutex_unlock(&P.lock);
free(ids), free(sh), free(r);
PlatThread *th[64];

View File

@@ -173,11 +173,30 @@ void vk_sync_cb_stats(char *out, size_t n) {
plat_mutex_unlock(&st_lock);
}
// NX2NAVI_DEVICE_LOST_AT=n (a test of the lost device's end, vk_device.c vk_device_lost_check): the n-th hand-over to
// the queue is not made and counts as one the driver answered with a lost device. 0 or unset: never.
static int lost_at_due(void) {
static int at = -1;
static int32_t calls;
if (at < 0) {
const char *e = cfg_get("NX2NAVI_DEVICE_LOST_AT");
at = e && *e ? atoi(e) : 0;
if (at < 0) at = 0;
if (at) rt_log("vk: queue hand-over %d will be taken as a lost device (NX2NAVI_DEVICE_LOST_AT)", at);
}
return at && ++calls == at; // queue lock held
}
static void hold_flush_locked(VkCtx *c) {
if (!H.n) return;
VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
if (vk_device_is_lost()) { // the run is ending: the held submissions are dropped, nothing reaches the queue
H.n = 0, H.last = 0;
__atomic_store_n(&held_n, 0, __ATOMIC_RELEASE);
return;
}
VkResult r = lost_at_due() ? VK_ERROR_DEVICE_LOST : vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the submit of the replay's command buffers");
rt_fatal("vk: vkQueueSubmit2 of %d held submissions up to value %llu failed (%d)", H.n,
(unsigned long long)H.last, (int)r);
}
@@ -480,7 +499,7 @@ static void completion_main(void *arg) {
.pSemaphores = &c->timeline, .pValues = &want};
VkResult r = vkWaitSemaphores(c->device, &wi, tmo); // 100 ms: stop is checked between waits
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the completion thread's wait for a submission");
rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r);
}
// A lost device is reported here, where it is first seen. Unchecked, the value was taken as it came: the
@@ -489,12 +508,20 @@ static void completion_main(void *arg) {
// checked queue call failed, an upload's, which then read as the cause.
r = vkGetSemaphoreCounterValue(c->device, c->timeline, &reached);
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the completion thread's read of the submissions' timeline");
rt_fatal("vk: the submissions' timeline could not be read (%d; device lost?)", (int)r);
}
// Both calls may succeed on a lost device and give a value its submissions never reached. So before a
// new value counts, the device is asked once more with a call of its own (vk_device.c
// vk_device_lost_poll, which ends the run when it is lost).
if (reached > __atomic_load_n(&done, __ATOMIC_ACQUIRE)) vk_device_lost_poll();
plat_mutex_lock(&st_lock);
if (reached > issued) reached = issued;
}
// Another thread saw the device lost (vk_lost): no value counts from here on and no callback runs. Theirs is
// work that never finished: frees of what the GPU may still hold, records handed out again, counter reports.
// done stays where it is, the waiters stay asleep, and the thread that saw the loss ends the process.
if (vk_device_is_lost()) break;
while (qn && queue[qhead].fence <= reached) {
Pending p = queue[qhead++];
qn--;
@@ -511,7 +538,9 @@ static void completion_main(void *arg) {
if (k == cbt_n && cbt_n < CBT_N) cbt[cbt_n++].fn = p.fn;
if (k < cbt_n) cbt[k].ns += c1 - c0, cbt[k].calls++;
}
if (vk_device_is_lost()) break;
}
if (vk_device_is_lost()) break;
if (reached > done) {
uint64_t ev = __atomic_load_n(&exw_value, __ATOMIC_ACQUIRE);
if (ev && reached >= ev) { // the side effects a waiter on exec_tl went ahead of are through
@@ -671,6 +700,9 @@ uint64_t vk_sync_reserve(void) {
static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCommandBuffer, void *), void *arg) {
vk_sync_start();
// The run is ending (vk_device.c vk_device_lost_check): nothing is recorded or submitted and no value is issued.
// The value returned is the last one issued; a wait for it sleeps until the process ends.
if (vk_device_is_lost()) return __atomic_load_n(&issued, __ATOMIC_ACQUIRE);
VkCtx *c = vk_ctx();
plat_mutex_lock(&sub_lock);
vk_mem_dirty_submit(); // NX2NAVI_TRACE=perf: what a mirror would upload before this submit
@@ -808,6 +840,9 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
// in such a callback too: a wait that returned before it let the game rewrite a pool that an older write-back
// then overwrote (Kokiri Forest: a sky coefficient the game had reset came back, and the world flickered washed
// out).
// The run is ending (vk_device.c vk_device_lost_check): no wait on the lost device, and none for a value that
// will not be reached.
if (vk_device_is_lost()) return __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value;
if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns);
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1;
uint64_t p0 = perf_begin();
@@ -817,7 +852,7 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
.pSemaphores = &exec_tl, .pValues = &value};
VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns);
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the wait for a submission's commands");
rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r);
}
perf_end(PERF_FENCE, p0);
@@ -877,6 +912,84 @@ void vk_sync_shutdown(void) {
plat_mutex_unlock(&start_lock);
}
// ---- --probe-devlost (host.h): the end of a run whose device is lost, without a lost device and without a window.
// The device itself stays healthy, so what the driver answers after a real loss is not covered here.
// 1. A callback queued for the next value runs when the timeline reaches it (the check that step 2 can fail).
// 2. With vk_lost set, as after another thread saw the loss: a callback queued for the next value does not run although
// the timeline reaches it, done stays, and a submit, an upload flush and a wait for a submission's commands return
// at once without issuing a value.
// 3. With vk_lost clear again, four threads report a lost device at the same moment: the process must end with
// RT_LOST_EXIT, one report and one FATAL line in the output (tests/device_lost_check.py counts them). Reaching the
// end of this function is the failure.
static volatile int32_t probe_ran[2];
static void probe_lost_cb(void *arg) { plat_xchg32((volatile int32_t *)arg, 1); }
static void probe_lost_record(VkCommandBuffer cb, void *arg) { (void)cb; plat_xchg32((volatile int32_t *)arg, 1); }
static volatile int32_t probe_go;
static void probe_lost_thread(void *arg) {
static const char *const where[4] = {"probe thread 0", "probe thread 1", "probe thread 2", "probe thread 3"};
while (!__atomic_load_n(&probe_go, __ATOMIC_ACQUIRE)) {}
vk_device_lost_check(VK_ERROR_DEVICE_LOST, where[(intptr_t)arg & 3]);
}
// Makes the timeline reach the next value as a finished submission would: issued moves on, the semaphore is signalled
// from the host, and the completion thread is woken. Returns the value.
static uint64_t probe_lost_advance(VkCtx *c, volatile int32_t *ran) {
plat_mutex_lock(&st_lock);
uint64_t v = issued + 1;
plat_mutex_unlock(&st_lock);
vk_completion_add(v, probe_lost_cb, (void *)ran);
VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline, .value = v};
if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) return 0;
plat_mutex_lock(&st_lock);
__atomic_store_n(&issued, v, __ATOMIC_RELEASE);
plat_cond_broadcast(&cv_work);
plat_mutex_unlock(&st_lock);
return v;
}
int vk_sync_probe_devlost(void) {
if (!vk_init()) {
printf("probe-devlost: SKIP (no Vulkan device on this machine)\n");
return 0;
}
vk_sync_start();
VkCtx *c = vk_ctx();
uint64_t v1 = probe_lost_advance(c, &probe_ran[0]);
for (int i = 0; i < 200 && !plat_cas32(&probe_ran[0], 1, 1); i++) plat_sleep_ms(10);
if (!v1 || !plat_cas32(&probe_ran[0], 1, 1) || !vk_sync_fence_wait(v1, 2000000000ull)) {
printf("probe-devlost: FAILED: a callback did not run on a healthy device (value %llu)\n", (unsigned long long)v1);
return 1;
}
printf("probe-devlost: healthy device: the callback of value %llu ran\n", (unsigned long long)v1);
plat_xchg32(&vk_lost, 1);
uint64_t v2 = probe_lost_advance(c, &probe_ran[1]);
plat_sleep_ms(500); // five of the completion thread's 100 ms waits
int ran = plat_cas32(&probe_ran[1], 1, 1), reached = vk_sync_fence_done() >= v2;
volatile int32_t recorded = 0;
uint64_t before = vk_sync_issued(), t0 = plat_now_ms();
uint64_t got = vk_sync_probe_submit(probe_lost_record, (void *)&recorded);
vk_upload_flush(1);
int waited = vk_sync_exec_wait(v2, 5000000000ull);
uint64_t ms = plat_now_ms() - t0;
int ok = v2 && !ran && !reached && !recorded && got == before && vk_sync_issued() == before && !waited && ms < 1000;
printf("probe-devlost: device marked lost: callback of value %llu %s, done %s, submit %s (issued %llu -> %llu), "
"the paths returned in %llu ms\n", (unsigned long long)v2, ran ? "RAN" : "not run", reached ? "ADVANCED" : "stayed",
recorded ? "RECORDED" : "not recorded", (unsigned long long)before, (unsigned long long)vk_sync_issued(),
(unsigned long long)ms);
if (!ok) { printf("probe-devlost: FAILED\n"); return 1; }
printf("probe-devlost: four threads report the loss now; the process must end with exit code %d\n", RT_LOST_EXIT);
fflush(stdout);
plat_xchg32(&vk_lost, 0);
for (intptr_t i = 0; i < 4; i++) plat_thread_spawn(probe_lost_thread, (void *)i, "probe lost", 0);
plat_sleep_ms(50);
__atomic_store_n(&probe_go, 1, __ATOMIC_RELEASE);
plat_sleep_ms(20000);
printf("probe-devlost: FAILED: still running 20 s after the loss was reported\n");
return 1;
}
// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): command buffer slots (read without sub_lock, which a whole replay holds),
// the completion queue and the submissions held back.
void vk_sync_sizes(PerfSizes *s) {

View File

@@ -57,6 +57,7 @@ typedef struct Tex {
uint64_t retire_frame; // gfx_counters.frames when it was retired
uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it
int uploaded, released;
uint64_t up_retry; // gfx_counters.frames from which an upload that had no memory is tried again (upload)
uint64_t uid; // names the entry in its bucket's released set (tex_gen_flip); 0 until its first release
uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale)
uint32_t back_gap; // most presents between a release and its taking back so far (stale_after)
@@ -2361,8 +2362,20 @@ static int astc_arena(void) {
// Copies the whole storage into the image (every level and layer the image has). An image made without its top levels
// (texture detail, VkTexImage.skip) has the game's level lv + skip at its level lv: the loops below run over the image's
// levels and read the storage at the game's.
// How long an upload whose decode had no memory waits for its next try (upload, up_wait): half a second of frames. A
// try costs the copy of the texture's blocks into staging memory, so not every frame.
#define UP_RETRY_FRAMES 30
static uint64_t n_up_retry;
uint64_t vk_texture_upload_retries(void) { return __atomic_load_n(&n_up_retry, __ATOMIC_RELAXED); }
// 1 while a texture that is not uploaded waits for its next try (e->up_retry; 0 for one never tried).
static int up_wait(const Tex *e) {
return e->up_retry && __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) < e->up_retry;
}
static void upload(Tex *e) {
static volatile int32_t n_bad, n_undecoded, n_split;
int no_mem = 0; // a part's ASTC decode had no memory: the texture is uploaded again later
const tex_t *t = &e->t;
const nvn_texdesc *d = &t->d;
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
@@ -2526,10 +2539,12 @@ static void upload(Tex *e) {
e->pub.image, e->pub.aspect, rbuf);
perf_end(PERF_ASTC, a0);
vk_texstat_time(VK_TS_ASTC, a0);
if (!checked) {
log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was",
(unsigned long long)e->pub.addr, d->format, d->width, d->height);
if (checked <= 0) {
log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was%s",
(unsigned long long)e->pub.addr, d->format, d->width, d->height,
checked < 0 ? " and tried again later (no memory for the decode)" : "");
e->chash_ok = 0; // the image does not hold the bytes hashed above: the next write uploads, equal or not
no_mem |= checked < 0;
}
} else {
for (k = k0; k < k1; k++) rg[k].bufferOffset += soff;
@@ -2544,7 +2559,7 @@ static void upload(Tex *e) {
parts);
if (acheck) { // waits for the GPU
vk_upload_flush(1);
if (checked) {
if (checked > 0) {
vmaInvalidateAllocation(vk_ctx()->vma, ralloc, 0, VK_WHOLE_SIZE);
astc_check(e, raw_copy, ar, (int)n, rmap);
}
@@ -2570,6 +2585,18 @@ static void upload(Tex *e) {
e->wt_n = 0;
e->uploaded = 1;
e->n_up++;
// The decode had no memory (vk_astc_record: its scratch buffer failed while the video memory was used up), so the
// image does not hold this content: a new image holds nothing, an older one what it held before. Counted as
// uploaded, it stayed that way until the game next wrote the storage, which for most textures is never. It counts
// as not uploaded instead, and make_fresh and make_fresh_snap upload it again UP_RETRY_FRAMES frames on, when
// the evictions under memory pressure have had time to make room; again and again until one succeeds.
if (no_mem) {
e->uploaded = 0;
e->up_retry = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + UP_RETRY_FRAMES;
__atomic_add_fetch(&n_up_retry, 1, __ATOMIC_RELAXED);
} else {
e->up_retry = 0;
}
perf_end(PERF_TEX, p0);
vk_texstat_time(VK_TS_UPLOAD, u0);
vk_texstat_add(VK_TS_UPLOAD, 0, src_bytes);
@@ -2623,6 +2650,7 @@ static void make_fresh(Tex *e) {
if (!e->pub.mirrored || e->pub.gpu_owned) return;
uint64_t size = e->t.L.size;
if (!e->uploaded) {
if (up_wait(e)) return;
vk_mem_changed(e->pub.addr, size, 0, &e->wgen); // resets the write bits before the storage is read
upload(e);
return;
@@ -2639,6 +2667,7 @@ static void make_fresh(Tex *e) {
// after it, during this replay) keeps its write-watch bit and is caught by the next submission's snapshot.
static void make_fresh_snap(Tex *e) {
if (!e->pub.mirrored || e->pub.gpu_owned) return;
if (!e->uploaded && up_wait(e)) return;
uint32_t now;
n_checks++;
int ch = vk_mem_changed_snap(e->pub.addr, e->t.L.size, e->uploaded ? e->wgen : 0, submit_epoch, &now);
@@ -3634,7 +3663,7 @@ static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t
free(ar);
}
if (!agpu) vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)g->nz, rg);
else if (!done) log_limited(&n_astc, "WriteTexels of ASTC texture %#llx not decoded on the GPU: left as it was",
else if (done <= 0) log_limited(&n_astc, "WriteTexels of ASTC texture %#llx not decoded on the GPU: left as it was",
(unsigned long long)e->pub.addr);
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
vk_upload_end();