From 534591be049f5be339abcda598d2be2c2e181299 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Wed, 11 Nov 2026 04:35:30 -0500 Subject: [PATCH] End a run whose GPU device is lost in one place: one report, no callbacks of unfinished submissions, and exit code 72. A lost device used to end the process from whichever submit or wait noticed it first, while the present only logged it and went on and the other threads kept calling the driver. The completion thread could take the submissions' timeline as reached and run the callbacks of submissions that never finished. vk_device_lost_check now takes the result of every submit, present, acquire and wait. The first thread to see the loss sets a flag, writes the report and ends the process as a fatal error does (the log written, a save commit under way finished), with an exit code of its own; later threads stop in that call, and a watchdog ends the process after ten seconds should the first thread hang. The replay, its submit, the uploads, the present and the lazy fetch read the flag and return without touching the device. The completion thread asks the device with a call of its own before a new value counts and runs no callback once the flag is set. The executable's --probe-devlost and tests/device_lost_check.py check this without a lost device, and device_lost_at in nx2navi.ini ends a real run the same way. Co-Authored-By: Claude Fable 5.2 --- docs/building.md | 8 +++ nx2navi.ini.example | 4 ++ runtime/cfg.c | 5 +- runtime/dispatch.c | 21 +++++-- runtime/dispatch.h | 6 ++ runtime/gfx_present.c | 14 +++-- runtime/host.h | 1 + runtime/main.c | 3 +- runtime/vk.h | 24 ++++++-- runtime/vk_device.c | 67 ++++++++++++++++---- runtime/vk_exec.c | 1 + runtime/vk_exec_probe.c | 2 +- runtime/vk_exec_xfer.c | 3 +- runtime/vk_memory.c | 8 ++- runtime/vk_sync.c | 123 +++++++++++++++++++++++++++++++++++-- tests/device_lost_check.py | 70 +++++++++++++++++++++ 16 files changed, 319 insertions(+), 41 deletions(-) create mode 100644 tests/device_lost_check.py diff --git a/docs/building.md b/docs/building.md index a1c2befa..a6f7398f 100644 --- a/docs/building.md +++ b/docs/building.md @@ -389,6 +389,14 @@ A change to what the library warm logs when the driver's cache of stage librarie `runtime/vk_pipeline.c` `lw_main`) passes `python tests\warm_hint_check.py`, which builds the header alone and checks the added text by GPU vendor and by why the cache was cold. No game files. +A change to how a run ends when the GPU device is lost (`runtime/vk_device.c` `vk_device_lost_check`, the checks of +`vk_lost` in the replay, upload and present paths, `runtime/vk_sync.c` `completion_main`) passes +`python tests\device_lost_check.py `, which runs the executable's `--probe-devlost`: with the device marked +lost no completion callback runs and the submit, upload and wait paths return at once, and four threads that report a +loss at the same moment end the process with exit code 72, one device report and one FATAL line. It needs a Vulkan +device and opens no window; the device is not really lost, so what a driver answers after a loss is not covered. +`device_lost_at = ` in `nx2navi.ini` ends a real run the same way at its n-th hand-over to the GPU queue. + A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice.h`, `runtime/vk_memory.c` `vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and its reason by platform, by GPU driver and by what the player set. No game files, no GPU. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index a38420ff..6bd16d6c 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -750,6 +750,10 @@ # last one it finished. Unset: on AMD GPUs (VK_AMD_buffer_marker). 1: also on NVIDIA GPUs # (VK_NV_device_diagnostic_checkpoints), which NVIDIA says costs a little. 0: off. # pass_crumbs = 1 +# A test of how the game ends when the video card stops answering (device lost): the hand-over to the GPU queue with +# this number is not made and is taken as one the driver refused with a lost device. The log then holds one report and +# one FATAL line, and the game ends with exit code 72. 0: never. Default: 0. +# device_lost_at = 0 # Once a second, puts the crash report's exception filter back if a GPU driver or an overlay loaded after the start set # its own in its place, and says so in the log. Without it, a crash in such a DLL can end the game with no FATAL line in # the log. 0: off. diff --git a/runtime/cfg.c b/runtime/cfg.c index 3630a3b9..645ce6a8 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -74,7 +74,7 @@ static const char *const known[] = { "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VA_REPORT", - "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", + "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "COOP", "COOP_NAME", "COOP_SERVER", "COOP_PORT", "COOP_ROOM", "COOP_AUTO", "COOP_DROP", // co-op (coop.c); the last three have no menu row "COOP_PROGRESS", "COOP_ITEMS", "COOP_SONGS", "COOP_QUEST", "COOP_KEYS", "COOP_RUPEES", "COOP_AMMO", // the room's rules (coop_game.c) "COOP_TEST_SET", // test: sets flags by hash at given frames (coop_game.c); no menu row @@ -142,7 +142,7 @@ static const struct { const char *key, *section; } sections[] = { {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"}, - {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, + {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -311,6 +311,7 @@ static const CfgMeta metas[] = { {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"VA_REPORT", "0", "0|1|2", RESTART}, // vk_vaddr.c GPU address history for a lost device's report; 2: call stacks + {"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/dispatch.c b/runtime/dispatch.c index 82996542..1c7f827a 100644 --- a/runtime/dispatch.c +++ b/runtime/dispatch.c @@ -59,14 +59,13 @@ void rt_fatal_notes(void) { __attribute__((weak)) void fs_save_crash_wait(void) {} // fs_host.c; builds without it (the unit checks) have no saves -void rt_fatal(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); +// The fatal line, the report's notes and what has to be flushed, then the exit with `code`. +static void fatal_end(int code, const char *fmt, va_list ap) __attribute__((noreturn)); +static void fatal_end(int code, const char *fmt, va_list ap) { rt_log_sync(); // main.c: the lines queued for the log thread, then these, written here rt_log_raw("FATAL: ", 7); rt_vlogf(fmt, ap); rt_log_raw("\n", 1); - va_end(ap); // A failed allocation (malloc, realloc, VirtualAlloc) means the commit limit was reached: this line tells a leak of // ours (process private bytes near the limit) from a machine short of page file. double proc, used, limit; @@ -81,7 +80,19 @@ void rt_fatal(const char *fmt, ...) { rtss_limit_restore(); fflush(stdout); fflush(stderr); - _Exit(70); + _Exit(code); +} + +void rt_fatal(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + fatal_end(RT_FATAL_EXIT, fmt, ap); +} + +void rt_fatal_code(int code, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + fatal_end(code, fmt, ap); } static const RtModule *module_at(uint64_t pc) { diff --git a/runtime/dispatch.h b/runtime/dispatch.h index 519d6ea8..7950e618 100644 --- a/runtime/dispatch.h +++ b/runtime/dispatch.h @@ -51,6 +51,12 @@ RtDispatchStats rt_dispatch_stats(void); // Runtime fatal error: message to stderr, then exit. Used by the dispatcher and the stubs. void rt_fatal(const char *fmt, ...) __attribute__((noreturn, format(printf, 1, 2))); +// The same end with an exit code of its own, for a failure that a script or the launcher tells from the others by it +// (vk_device.c vk_device_lost_check ends with RT_LOST_EXIT). rt_fatal ends with RT_FATAL_EXIT; 71 is the crash report's +// (main.c crash_handler). +#define RT_FATAL_EXIT 70 +#define RT_LOST_EXIT 72 +void rt_fatal_code(int code, const char *fmt, ...) __attribute__((noreturn, format(printf, 2, 3))); // Lines of context for a fatal error's report, which rt_fatal and the crash report (main.c crash_handler) write after // their own: fn logs one with rt_log. It runs on the failing thread, which may hold any lock, so it reads only what // needs none. At most four; adding one twice keeps one. rt_fatal_notes runs them, once per process. diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index 73cb442d..569ea61e 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -319,7 +319,7 @@ static void flip_locked(void) { VkResult r = vk_dlssg_active() ? vk_dlssg_present(vk_ctx()->gfx_queue, &pi) : vk_fg_active() ? vk_fg_present(&pi) : vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi); vk_queue_unlock(); - vk_device_fault_report(r); + vk_device_lost_check(r, "the present"); if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1); } @@ -996,7 +996,7 @@ static void ng_present(VkSwapchainKHR sc, uint32_t idx) { vk_queue_lock(); VkResult r = vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi); vk_queue_unlock(); - vk_device_fault_report(r); + vk_device_lost_check(r, "the present of a generated frame"); if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1); __atomic_add_fetch(&ng.presents, 1, __ATOMIC_RELAXED); } @@ -1061,7 +1061,7 @@ static int ng_show(VkImage img, int set, int slot, uint64_t n) { vk_queue_lock(); r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); - if (r != VK_SUCCESS) { vk_device_fault_report(r); return 0; } + if (r != VK_SUCCESS) { vk_device_lost_check(r, "the submit of a generated frame"); return 0; } ng_present(ng.job.sc, j); return 1; } @@ -1331,6 +1331,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w static VkFormat last = VK_FORMAT_UNDEFINED; if (src && fmt != last) last = fmt, rt_log("gfx: the game presents %dx%d images of format %d", w, h, (int)fmt); } + if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): nothing touches the device gfx_window_start(); VkCtx *c = vk_ctx(); plat_rmutex_lock(&g_pl); @@ -1359,8 +1360,8 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w : vk_fg_active() ? vk_fg_acquire(g_sc, 1000000000ull, g_acquire[slot], &idx) : vkAcquireNextImageKHR(c->device, g_sc, 1000000000ull, g_acquire[slot], VK_NULL_HANDLE, &idx); } - vk_device_fault_report(r); - // lost, not ready: skip the frame. Acquires that keep timing out (1 s each) mean a swapchain nothing will free an + vk_device_lost_check(r, "the acquire of a swapchain image"); + // not ready: skip the frame. Acquires that keep timing out (1 s each) mean a swapchain nothing will free an // image of again: three in a row rebuild it, where the picture used to stop for good. static uint32_t n_timeouts; if (r == VK_TIMEOUT || r == VK_NOT_READY) { @@ -1554,7 +1555,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w g_flip_idx = idx; if (!defer) flip_locked(); } else { - vk_device_fault_report(r); + vk_device_lost_check(r, "the present's submit"); rt_log("gfx: present submit failed (%d)", (int)r); // The fence was reset after the acquire and the acquire's semaphore is signalled: left so, this slot's fence // wait would time out (5 s) at every later present and the picture stop for good. An empty batch takes the @@ -1594,6 +1595,7 @@ void vk_present_blit(VkImage src, VkImageLayout layout, VkFormat fmt, int w, int void gfx_present_boot(void) { present_image(VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED, VK_FORMAT_UNDEFINED, 1920, 1080, NULL, 0); } void vk_present_flip(void) { + if (vk_device_is_lost()) return; plat_rmutex_lock(&g_pl); flip_locked(); plat_rmutex_unlock(&g_pl); diff --git a/runtime/host.h b/runtime/host.h index f0b27bf6..c905b0b2 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -428,6 +428,7 @@ int probe_discord(void); // discord_rpc.c int probe_loader(void), probe_kernel(void), probe_ipc(void), probe_services(void), probe_fs(void); int probe_gfx_abi(void); // gfx_abi_probe.c int probe_vkdevice(void); // vk_device.c +int vk_sync_probe_devlost(void); // vk_sync.c: --probe-devlost int probe_nvn(void), probe_nvn_tex(void); // nvn_probe.c, nvn_tex_probe.c int probe_window(void); // gfx_window_probe.c int probe_shader(void); // gfx_shader.c diff --git a/runtime/main.c b/runtime/main.c index 0722a6c9..d946e539 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -534,7 +534,7 @@ static int usage(const char *argv0) { "--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " "--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | --probe-coop | " - "--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0); + "--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0); return 2; } @@ -685,6 +685,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--probe-zstd")) return probe_zstd(); if (!strcmp(mode, "--probe-input")) return probe_gamepad(); if (!strcmp(mode, "--probe-vkdevice")) return probe_vkdevice(); + if (!strcmp(mode, "--probe-devlost")) return vk_sync_probe_devlost(); if (!strcmp(mode, "--probe-movie")) { // the AV1 decoder tells the graphics back end of each frame it writes: none here cfg.gpu = "null"; return probe_movie(); diff --git a/runtime/vk.h b/runtime/vk.h index 0ba62f80..809681b4 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -80,13 +80,25 @@ VkCtx *vk_ctx(void); // NULL until vk_init() succeeded int vk_init(void); // idempotent; 1 on success, 0 on failure (reason logged with rt_log) void vk_shutdown(void); int vk_selftest(void); // 0 on success -// Call with the result of a failed queue submit, present or wait before stopping on it: VK_ERROR_DEVICE_LOST logs -// what VK_EXT_device_fault reports (description, faulting addresses with their precision, vendor codes) once per -// process, after the GPU breadcrumbs (vk_crumbs_report). Any other result returns at once. -void vk_device_fault_report(VkResult r); +// Call with the result of every queue submit, present, acquire or wait. Any result but VK_ERROR_DEVICE_LOST returns at +// once, and the caller goes on with its own handling. A lost device ends the run here, in one place: the first thread +// to arrive sets vk_lost, logs the GPU breadcrumbs (vk_crumbs_report) and what VK_EXT_device_fault reports +// (description, faulting addresses with their precision, vendor codes), then ends the process as rt_fatal does (the +// log written, a save commit under way finished) with exit code RT_LOST_EXIT (dispatch.h). `where` names the call for +// the fatal line. Every later thread stops in this call and never returns to touch the device; the process ends +// within LOST_EXIT_MS (vk_device.c) even when the first thread hangs on a lock one of them holds. +void vk_device_lost_check(VkResult r, const char *where); +// 1 from the moment a thread saw the device lost. The paths that would touch the device (the replay and its submit, +// the uploads, the present, the completion thread's callbacks) read it where they start and return without doing so. +extern volatile int32_t vk_lost; +static inline int vk_device_is_lost(void) { return __atomic_load_n(&vk_lost, __ATOMIC_RELAXED); } +// Asks the device whether it is lost with a call that has a result and needs no lock (the status of a fence nothing +// signals), and ends the run through vk_device_lost_check when it is. The completion thread calls it before it runs +// callbacks: a lost device may report the submissions' timeline as reached (vk_sync.c completion_main). +void vk_device_lost_poll(void); // ---- GPU address history (vk_vaddr.c, NX2NAVI_VA_REPORT=1, 2 with call stacks): VK_EXT_device_address_binding_report's -// reports, kept so vk_device_fault_report can name the objects that held a faulting address. +// reports, kept so the lost device report (vk_device.c device_fault_report) can name the objects that held a faulting address. int vk_vaddr_wanted(void); void vk_vaddr_start(VkInstance instance); // create_instance, after vkCreateInstance void vk_vaddr_device(int supported); // create_device: the driver reports address bindings @@ -157,7 +169,7 @@ const VkPassMark *vk_pass_mark_record(uint32_t id); // the ring record of mark // The report's text for a started and a finished mark, three lines without a trailing newline: which marks (source: // the extension they came from), then each with its ring record. vk_pass_marks_report logs it; the probe checks it. void vk_pass_marks_format(const char *source, uint32_t started, uint32_t finished, char *o, size_t size); -void vk_pass_marks_report(void); // vk_device_fault_report: the AMD words and the NVIDIA checkpoints, formatted +void vk_pass_marks_report(void); // vk_device.c device_fault_report: the AMD words and the NVIDIA checkpoints, formatted // ---- window and present (gfx_present.c). Host threads only. // vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 6e1384b5..1d0311cc 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -25,6 +25,10 @@ static uint32_t g_gfx_queues = 1; // queues the graphics family offers (pick_d static VkCtx g_vk; static int g_ready; static PlatMutex g_lock = PLAT_MUTEX_INIT; +// The fence vk_device_lost_poll asks about. Nothing submits with it, so its status is "not ready" for as long as the +// device lives. +static VkFence lost_fence; +static PlatMutex lost_fence_lock = PLAT_MUTEX_INIT; static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type, const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) { @@ -550,6 +554,8 @@ static void destroy_all(void) { if (g_vk.device) vk_astc_release(); if (g_vk.device) vk_crumbs_release(); if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL); + if (lost_fence) vkDestroyFence(g_vk.device, lost_fence, NULL); // vk_device_lost_poll + lost_fence = VK_NULL_HANDLE; vk_headroom_stop(); if (g_vk.vma) vmaDestroyAllocator(g_vk.vma); if (g_vk.device) vkDestroyDevice(g_vk.device, NULL); @@ -599,15 +605,9 @@ static const char *fault_address_type(VkDeviceFaultAddressTypeEXT t) { } } -// The query runs once per process. Threads that hit the loss at the same time wait on the lock until it is logged, so -// none of them reaches its rt_fatal and ends the process before the report is written. -void vk_device_fault_report(VkResult r) { - static PlatMutex lock = PLAT_MUTEX_INIT; - static int done; - if (r != VK_ERROR_DEVICE_LOST) return; - plat_mutex_lock(&lock); - if (done) { plat_mutex_unlock(&lock); return; } - done = 1; +// The report of a lost device: the breadcrumbs, the pass marks, the video memory and the driver's fault information. +// Once per process, on the thread that saw the loss first (vk_device_lost_check). +static void device_fault_report(void) { vk_crumbs_report(); vk_pass_marks_report(); vk_mem_lost_report(); @@ -619,7 +619,6 @@ void vk_device_fault_report(VkResult r) { rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point" : "not enabled"); vk_vaddr_dump(); // the history still names what held a page another report (the kernel's log) gives - plat_mutex_unlock(&lock); return; } VkDeviceFaultCountsEXT n = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT}; @@ -648,7 +647,53 @@ void vk_device_fault_report(VkResult r) { vk_vaddr_dump(); free(ai); free(vi); - plat_mutex_unlock(&lock); +} + +// ---- the end of a run whose device is lost (vk.h vk_device_lost_check) +// A lost device stays lost: every later call on it fails or, worse, answers as if it had worked, and a driver that is +// called on from several threads after the loss can fault inside itself. So the first thread that sees the loss ends +// the run, and nothing else touches the device meanwhile: the paths that would read vk_lost and return, and a thread +// that gets the loss as a result of its own stops here for good. +// LOST_EXIT_MS bounds the end. The first thread writes the report and waits for a save commit (fs_host.c +// fs_save_crash_wait, 2 s at most); should it hang on a lock that a stopped thread holds, the watchdog ends the +// process with the same code, and the lines already written are in the log (rt_fatal_code wrote them itself). +#define LOST_EXIT_MS 10000 +volatile int32_t vk_lost; + +static void lost_wait_exit(void *arg) { + (void)arg; + for (int t = 0; t < LOST_EXIT_MS; t += 50) plat_sleep_ms(50); + _Exit(RT_LOST_EXIT); +} + +void vk_device_lost_check(VkResult r, const char *where) { + if (r != VK_ERROR_DEVICE_LOST) return; + if (plat_xchg32(&vk_lost, 1)) lost_wait_exit(NULL); // another thread is ending the run: this one stops here + plat_thread_spawn(lost_wait_exit, NULL, "vk lost exit", 0); + device_fault_report(); + rt_fatal_code(RT_LOST_EXIT, "vk: the GPU device was lost (seen at: %s). The run ends here; the lines above name what " + "the GPU was doing", where); +} + + +void vk_device_lost_poll(void) { + VkCtx *c = vk_ctx(); + if (!c || vk_device_is_lost()) return; + VkFence f = __atomic_load_n(&lost_fence, __ATOMIC_ACQUIRE); + if (!f) { + plat_mutex_lock(&lost_fence_lock); + if (!lost_fence) { + VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence n = VK_NULL_HANDLE; + VkResult r = vkCreateFence(c->device, &fi, NULL, &n); + if (r == VK_SUCCESS) __atomic_store_n(&lost_fence, n, __ATOMIC_RELEASE); + else vk_device_lost_check(r, "the fence that asks whether the device lives"); + } + f = lost_fence; + plat_mutex_unlock(&lost_fence_lock); + if (!f) return; + } + vk_device_lost_check(vkGetFenceStatus(c->device, f), "a poll of the device"); } int vk_selftest(void) { diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 740e2aba..3698abe4 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1664,6 +1664,7 @@ static void rec_claim(void) { } int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { + if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): nothing is recorded uint64_t in_ns = kern_now_ns(); perf_lock_timing_here(); // NX2NAVI_LOCK_TIME if (prof_replay_tid != plat_thread_id()) perf_replay_thread(); // the profiler follows the thread replaying now diff --git a/runtime/vk_exec_probe.c b/runtime/vk_exec_probe.c index c2152406..f560dae0 100644 --- a/runtime/vk_exec_probe.c +++ b/runtime/vk_exec_probe.c @@ -390,7 +390,7 @@ static uint8_t *draw_prefix(uint8_t *q, prog_t *prog, const pool_t *pool, tex_t // ---- pass marks part (vk_crumbs.c): one submission with a pass of 2 draws into the draw part's target, a dispatch and // a pass of 3 draws into a second target. Once its fence is done, the buffer marks must name the last pass as both // started and finished, the ring records must hold what the three were, and vk_pass_marks_format (the text -// vk_device_fault_report logs) must name it. Without VK_AMD_buffer_marker or with NX2NAVI_PASS_CRUMBS=0 it is skipped: +// vk_device.c device_fault_report logs) must name it. Without VK_AMD_buffer_marker or with NX2NAVI_PASS_CRUMBS=0 it is skipped: // NVIDIA's checkpoints can only be read once the device is lost. static char marks_text[2048]; static int marks_skipped; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index a0e83134..713bd344 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -311,7 +311,7 @@ static void oq_read(VkDevice dev, uint32_t first, uint32_t n) { VkResult r = vkGetQueryPoolResults(dev, oq_pool, first, n, sizeof OQB.v[0] * 2 * n, OQB.v, 2 * sizeof OQB.v[0], VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); if (r != VK_SUCCESS && r != VK_NOT_READY) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the read of occlusion query results"); rt_fatal("vk: occlusion query results failed (%d)", (int)r); } OQB.first = first, OQB.n = n; @@ -353,6 +353,7 @@ static void oq_cb(void *arg) { if (avail) sum += v; else { sum += 1; + vk_device_lost_poll(); // a query its submission never wrote: on a lost device the run ends here LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[i]); } } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index e564d58b..519de236 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2932,6 +2932,7 @@ static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags // memory.c's fetch (frz_lock held there, so one at a time): [off, off + size) of the mirror h0 into the imported // buffer h1, after all the GPU work handed over so far. static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) { + if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): the bytes stay as they are VkCtx *c = vk_ctx(); VkBufferCopy reg = {off, off, size}; VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, @@ -2952,7 +2953,7 @@ static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) { vk_queue_unlock(); if (r == VK_SUCCESS) r = vkWaitForFences(c->device, 1, &lz_fence, VK_TRUE, 10000000000ull); if (r != VK_SUCCESS) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the submit or the wait that brings GPU writes down to guest memory"); rt_fatal("vk: bringing %llu bytes the GPU wrote down to guest memory failed (%d)", (unsigned long long)size, (int)r); } vkResetFences(c->device, 1, &lz_fence); @@ -4109,6 +4110,7 @@ void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) { } uint64_t vk_upload_flush(int wait) { + if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): no submit, no wait uint64_t f0 = kern_now_ns(); plat_once(&cmd_once, cmd_init, NULL); plat_rmutex_lock(&cmd_lock); @@ -4125,7 +4127,7 @@ uint64_t vk_upload_flush(int wait) { VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); if (r != VK_SUCCESS) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the submit of an upload batch"); rt_fatal("vk: upload submit failed (%d)", (int)r); } if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight); @@ -4138,7 +4140,7 @@ uint64_t vk_upload_flush(int wait) { // loop would spin for good, holding the upload lock, instead of the run ending with the device's report VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX); if (w != VK_SUCCESS) { - vk_device_fault_report(w); + vk_device_lost_check(w, "the wait for an upload batch"); rt_fatal("vk: the wait for an upload failed (%d)", (int)w); } retire(c, 0); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 6244a181..af5c3cb1 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -114,11 +114,30 @@ int vk_sync_wb_split(void) { return wb_split && exec_tl != VK_NULL_HANDLE; } +// NX2NAVI_DEVICE_LOST_AT=n (a test of the lost device's end, vk_device.c vk_device_lost_check): the n-th hand-over to +// the queue is not made and counts as one the driver answered with a lost device. 0 or unset: never. +static int lost_at_due(void) { + static int at = -1; + static int32_t calls; + if (at < 0) { + const char *e = cfg_get("NX2NAVI_DEVICE_LOST_AT"); + at = e && *e ? atoi(e) : 0; + if (at < 0) at = 0; + if (at) rt_log("vk: queue hand-over %d will be taken as a lost device (NX2NAVI_DEVICE_LOST_AT)", at); + } + return at && ++calls == at; // queue lock held +} + static void hold_flush_locked(VkCtx *c) { if (!H.n) return; - VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); + if (vk_device_is_lost()) { // the run is ending: the held submissions are dropped, nothing reaches the queue + H.n = 0, H.last = 0; + __atomic_store_n(&held_n, 0, __ATOMIC_RELEASE); + return; + } + VkResult r = lost_at_due() ? VK_ERROR_DEVICE_LOST : vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); if (r != VK_SUCCESS) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the submit of the replay's command buffers"); rt_fatal("vk: vkQueueSubmit2 of %d held submissions up to value %llu failed (%d)", H.n, (unsigned long long)H.last, (int)r); } @@ -421,7 +440,7 @@ static void completion_main(void *arg) { .pSemaphores = &c->timeline, .pValues = &want}; VkResult r = vkWaitSemaphores(c->device, &wi, tmo); // 100 ms: stop is checked between waits if (r != VK_SUCCESS && r != VK_TIMEOUT) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the completion thread's wait for a submission"); rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); } // A lost device is reported here, where it is first seen. Unchecked, the value was taken as it came: the @@ -430,19 +449,29 @@ static void completion_main(void *arg) { // checked queue call failed, an upload's, which then read as the cause. r = vkGetSemaphoreCounterValue(c->device, c->timeline, &reached); if (r != VK_SUCCESS) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the completion thread's read of the submissions' timeline"); rt_fatal("vk: the submissions' timeline could not be read (%d; device lost?)", (int)r); } + // Both calls may succeed on a lost device and give a value its submissions never reached. So before a + // new value counts, the device is asked once more with a call of its own (vk_device.c + // vk_device_lost_poll, which ends the run when it is lost). + if (reached > __atomic_load_n(&done, __ATOMIC_ACQUIRE)) vk_device_lost_poll(); plat_mutex_lock(&st_lock); if (reached > issued) reached = issued; } + // Another thread saw the device lost (vk_lost): no value counts from here on and no callback runs. Theirs is + // work that never finished: frees of what the GPU may still hold, records handed out again, counter reports. + // done stays where it is, the waiters stay asleep, and the thread that saw the loss ends the process. + if (vk_device_is_lost()) break; while (qn && queue[qhead].fence <= reached) { Pending p = queue[qhead++]; qn--; plat_mutex_unlock(&st_lock); p.fn(p.arg); plat_mutex_lock(&st_lock); + if (vk_device_is_lost()) break; } + if (vk_device_is_lost()) break; if (reached > done) { // The waiters go first (a fence wait, the frame's first submission, the acquire): freeing what the // finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each @@ -595,6 +624,9 @@ uint64_t vk_sync_reserve(void) { static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCommandBuffer, void *), void *arg) { vk_sync_start(); + // The run is ending (vk_device.c vk_device_lost_check): nothing is recorded or submitted and no value is issued. + // The value returned is the last one issued; a wait for it sleeps until the process ends. + if (vk_device_is_lost()) return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); VkCtx *c = vk_ctx(); plat_mutex_lock(&sub_lock); vk_mem_dirty_submit(); // NX2NAVI_TRACE=perf: what a mirror would upload before this submit @@ -730,6 +762,9 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { // fence callback, after the executed timeline: a wait that returned there let the game rewrite a pool that an // older write-back then overwrote (Kokiri Forest: a sky coefficient the game had reset came back, and the world // flickered washed out). So the wait covers the write-backs there, as without the split. + // The run is ending (vk_device.c vk_device_lost_check): no wait on the lost device, and none for a value that + // will not be reached. + if (vk_device_is_lost()) return __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value; if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns); if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; uint64_t p0 = perf_begin(); @@ -739,7 +774,7 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { .pSemaphores = &exec_tl, .pValues = &value}; VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns); if (r != VK_SUCCESS && r != VK_TIMEOUT) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the wait for a submission's commands"); rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r); } perf_end(PERF_FENCE, p0); @@ -791,6 +826,84 @@ void vk_sync_shutdown(void) { plat_mutex_unlock(&start_lock); } +// ---- --probe-devlost (host.h): the end of a run whose device is lost, without a lost device and without a window. +// The device itself stays healthy, so what the driver answers after a real loss is not covered here. +// 1. A callback queued for the next value runs when the timeline reaches it (the check that step 2 can fail). +// 2. With vk_lost set, as after another thread saw the loss: a callback queued for the next value does not run although +// the timeline reaches it, done stays, and a submit, an upload flush and a wait for a submission's commands return +// at once without issuing a value. +// 3. With vk_lost clear again, four threads report a lost device at the same moment: the process must end with +// RT_LOST_EXIT, one report and one FATAL line in the output (tests/device_lost_check.py counts them). Reaching the +// end of this function is the failure. +static volatile int32_t probe_ran[2]; +static void probe_lost_cb(void *arg) { plat_xchg32((volatile int32_t *)arg, 1); } +static void probe_lost_record(VkCommandBuffer cb, void *arg) { (void)cb; plat_xchg32((volatile int32_t *)arg, 1); } +static volatile int32_t probe_go; +static void probe_lost_thread(void *arg) { + static const char *const where[4] = {"probe thread 0", "probe thread 1", "probe thread 2", "probe thread 3"}; + while (!__atomic_load_n(&probe_go, __ATOMIC_ACQUIRE)) {} + vk_device_lost_check(VK_ERROR_DEVICE_LOST, where[(intptr_t)arg & 3]); +} + +// Makes the timeline reach the next value as a finished submission would: issued moves on, the semaphore is signalled +// from the host, and the completion thread is woken. Returns the value. +static uint64_t probe_lost_advance(VkCtx *c, volatile int32_t *ran) { + plat_mutex_lock(&st_lock); + uint64_t v = issued + 1; + plat_mutex_unlock(&st_lock); + vk_completion_add(v, probe_lost_cb, (void *)ran); + VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline, .value = v}; + if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) return 0; + plat_mutex_lock(&st_lock); + __atomic_store_n(&issued, v, __ATOMIC_RELEASE); + plat_cond_broadcast(&cv_work); + plat_mutex_unlock(&st_lock); + return v; +} + +int vk_sync_probe_devlost(void) { + if (!vk_init()) { + printf("probe-devlost: SKIP (no Vulkan device on this machine)\n"); + return 0; + } + vk_sync_start(); + VkCtx *c = vk_ctx(); + uint64_t v1 = probe_lost_advance(c, &probe_ran[0]); + for (int i = 0; i < 200 && !plat_cas32(&probe_ran[0], 1, 1); i++) plat_sleep_ms(10); + if (!v1 || !plat_cas32(&probe_ran[0], 1, 1) || !vk_sync_fence_wait(v1, 2000000000ull)) { + printf("probe-devlost: FAILED: a callback did not run on a healthy device (value %llu)\n", (unsigned long long)v1); + return 1; + } + printf("probe-devlost: healthy device: the callback of value %llu ran\n", (unsigned long long)v1); + + plat_xchg32(&vk_lost, 1); + uint64_t v2 = probe_lost_advance(c, &probe_ran[1]); + plat_sleep_ms(500); // five of the completion thread's 100 ms waits + int ran = plat_cas32(&probe_ran[1], 1, 1), reached = vk_sync_fence_done() >= v2; + volatile int32_t recorded = 0; + uint64_t before = vk_sync_issued(), t0 = plat_now_ms(); + uint64_t got = vk_sync_probe_submit(probe_lost_record, (void *)&recorded); + vk_upload_flush(1); + int waited = vk_sync_exec_wait(v2, 5000000000ull); + uint64_t ms = plat_now_ms() - t0; + int ok = v2 && !ran && !reached && !recorded && got == before && vk_sync_issued() == before && !waited && ms < 1000; + printf("probe-devlost: device marked lost: callback of value %llu %s, done %s, submit %s (issued %llu -> %llu), " + "the paths returned in %llu ms\n", (unsigned long long)v2, ran ? "RAN" : "not run", reached ? "ADVANCED" : "stayed", + recorded ? "RECORDED" : "not recorded", (unsigned long long)before, (unsigned long long)vk_sync_issued(), + (unsigned long long)ms); + if (!ok) { printf("probe-devlost: FAILED\n"); return 1; } + + printf("probe-devlost: four threads report the loss now; the process must end with exit code %d\n", RT_LOST_EXIT); + fflush(stdout); + plat_xchg32(&vk_lost, 0); + for (intptr_t i = 0; i < 4; i++) plat_thread_spawn(probe_lost_thread, (void *)i, "probe lost", 0); + plat_sleep_ms(50); + __atomic_store_n(&probe_go, 1, __ATOMIC_RELEASE); + plat_sleep_ms(20000); + printf("probe-devlost: FAILED: still running 20 s after the loss was reported\n"); + return 1; +} + // NX2NAVI_TRACE=perf "mem: sizes" (perf.c): command buffer slots (read without sub_lock, which a whole replay holds), // the completion queue and the submissions held back. void vk_sync_sizes(PerfSizes *s) { diff --git a/tests/device_lost_check.py b/tests/device_lost_check.py new file mode 100644 index 00000000..81155239 --- /dev/null +++ b/tests/device_lost_check.py @@ -0,0 +1,70 @@ +"""Run the game executable's --probe-devlost and check how a run ends when the GPU device is lost +(runtime/vk_device.c vk_device_lost_check, runtime/vk_sync.c vk_sync_probe_devlost). + +The probe needs a Vulkan device and opens no window. It does not lose the device: it marks it lost, and then has four +threads report a loss at the same moment. Checked here: + + - on the healthy device a completion callback runs (so the next check can fail); + - with the device marked lost no callback runs, no value counts as done, and the submit, upload and wait paths return + at once; + - the process ends with exit code 72 (dispatch.h RT_LOST_EXIT), with one device report and one FATAL line, well + before the watchdog's 10 seconds. + +What a driver answers after a real loss is not covered. + +Usage: device_lost_check.py PATH_TO_GAME_EXE +""" +import subprocess +import sys +import time + +LOST_EXIT = 72 + + +def main(): + if len(sys.argv) != 2: + print(__doc__) + return 2 + p = subprocess.Popen([sys.argv[1], '--probe-devlost'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, errors='replace') + lines, reported_at = [], None + for line in p.stdout: + lines.append(line.rstrip('\n')) + if 'four threads report the loss now' in line: + reported_at = time.monotonic() + code = p.wait() + ended = time.monotonic() + text = '\n'.join(lines) + if 'probe-devlost: SKIP' in text: + print('device_lost_check: SKIP (no Vulkan device)') + return 0 + fails = [] + + def check(ok, what): + if not ok: + fails.append(what) + + check('probe-devlost: healthy device: the callback of value' in text, 'no callback ran on the healthy device') + marked = [ln for ln in lines if ln.startswith('probe-devlost: device marked lost:')] + check(len(marked) == 1 and 'not run, done stayed, submit not recorded' in marked[0], + 'a path went on after the device was marked lost: %s' % (marked or ['no line'])) + check('probe-devlost: FAILED' not in text, 'the probe reports a failure') + check(code == LOST_EXIT, 'exit code %d, expected %d' % (code, LOST_EXIT)) + fatal = [ln for ln in lines if ln.startswith('FATAL: ')] + check(len(fatal) == 1 and 'the GPU device was lost' in fatal[0], '%d FATAL lines, expected one: %s' % (len(fatal), fatal)) + reports = [ln for ln in lines if 'video memory when the device was lost' in ln] + check(len(reports) == 1, '%d device reports, expected one' % len(reports)) + took = ended - reported_at if reported_at else -1 + check(0 <= took < 5, 'the process ended %.1f s after the loss was reported (the watchdog, not the reporting thread?)' % took) + for f in fails: + print('device_lost_check: FAIL:', f) + if fails: + print(text[-3000:]) + return 1 + print('device_lost_check: passed (exit code %d, one report, one FATAL line, %.2f s from the report to the exit)' + % (code, took)) + return 0 + + +if __name__ == '__main__': + sys.exit(main())