diff --git a/docs/building.md b/docs/building.md index a1c2befa..a6f7398f 100644 --- a/docs/building.md +++ b/docs/building.md @@ -389,6 +389,14 @@ A change to what the library warm logs when the driver's cache of stage librarie `runtime/vk_pipeline.c` `lw_main`) passes `python tests\warm_hint_check.py`, which builds the header alone and checks the added text by GPU vendor and by why the cache was cold. No game files. +A change to how a run ends when the GPU device is lost (`runtime/vk_device.c` `vk_device_lost_check`, the checks of +`vk_lost` in the replay, upload and present paths, `runtime/vk_sync.c` `completion_main`) passes +`python tests\device_lost_check.py `, which runs the executable's `--probe-devlost`: with the device marked +lost no completion callback runs and the submit, upload and wait paths return at once, and four threads that report a +loss at the same moment end the process with exit code 72, one device report and one FATAL line. It needs a Vulkan +device and opens no window; the device is not really lost, so what a driver answers after a loss is not covered. +`device_lost_at = ` in `nx2navi.ini` ends a real run the same way at its n-th hand-over to the GPU queue. + A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice.h`, `runtime/vk_memory.c` `vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and its reason by platform, by GPU driver and by what the player set. No game files, no GPU. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index a38420ff..6bd16d6c 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -750,6 +750,10 @@ # last one it finished. Unset: on AMD GPUs (VK_AMD_buffer_marker). 1: also on NVIDIA GPUs # (VK_NV_device_diagnostic_checkpoints), which NVIDIA says costs a little. 0: off. # pass_crumbs = 1 +# A test of how the game ends when the video card stops answering (device lost): the hand-over to the GPU queue with +# this number is not made and is taken as one the driver refused with a lost device. The log then holds one report and +# one FATAL line, and the game ends with exit code 72. 0: never. Default: 0. +# device_lost_at = 0 # Once a second, puts the crash report's exception filter back if a GPU driver or an overlay loaded after the start set # its own in its place, and says so in the log. Without it, a crash in such a DLL can end the game with no FATAL line in # the log. 0: off. diff --git a/runtime/cfg.c b/runtime/cfg.c index 3630a3b9..645ce6a8 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -74,7 +74,7 @@ static const char *const known[] = { "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VA_REPORT", - "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", + "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT", "COOP", "COOP_NAME", "COOP_SERVER", "COOP_PORT", "COOP_ROOM", "COOP_AUTO", "COOP_DROP", // co-op (coop.c); the last three have no menu row "COOP_PROGRESS", "COOP_ITEMS", "COOP_SONGS", "COOP_QUEST", "COOP_KEYS", "COOP_RUPEES", "COOP_AMMO", // the room's rules (coop_game.c) "COOP_TEST_SET", // test: sets flags by hash at given frames (coop_game.c); no menu row @@ -142,7 +142,7 @@ static const struct { const char *key, *section; } sections[] = { {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"}, - {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, + {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -311,6 +311,7 @@ static const CfgMeta metas[] = { {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"VA_REPORT", "0", "0|1|2", RESTART}, // vk_vaddr.c GPU address history for a lost device's report; 2: call stacks + {"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/dispatch.c b/runtime/dispatch.c index 82996542..1c7f827a 100644 --- a/runtime/dispatch.c +++ b/runtime/dispatch.c @@ -59,14 +59,13 @@ void rt_fatal_notes(void) { __attribute__((weak)) void fs_save_crash_wait(void) {} // fs_host.c; builds without it (the unit checks) have no saves -void rt_fatal(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); +// The fatal line, the report's notes and what has to be flushed, then the exit with `code`. +static void fatal_end(int code, const char *fmt, va_list ap) __attribute__((noreturn)); +static void fatal_end(int code, const char *fmt, va_list ap) { rt_log_sync(); // main.c: the lines queued for the log thread, then these, written here rt_log_raw("FATAL: ", 7); rt_vlogf(fmt, ap); rt_log_raw("\n", 1); - va_end(ap); // A failed allocation (malloc, realloc, VirtualAlloc) means the commit limit was reached: this line tells a leak of // ours (process private bytes near the limit) from a machine short of page file. double proc, used, limit; @@ -81,7 +80,19 @@ void rt_fatal(const char *fmt, ...) { rtss_limit_restore(); fflush(stdout); fflush(stderr); - _Exit(70); + _Exit(code); +} + +void rt_fatal(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + fatal_end(RT_FATAL_EXIT, fmt, ap); +} + +void rt_fatal_code(int code, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + fatal_end(code, fmt, ap); } static const RtModule *module_at(uint64_t pc) { diff --git a/runtime/dispatch.h b/runtime/dispatch.h index 519d6ea8..7950e618 100644 --- a/runtime/dispatch.h +++ b/runtime/dispatch.h @@ -51,6 +51,12 @@ RtDispatchStats rt_dispatch_stats(void); // Runtime fatal error: message to stderr, then exit. Used by the dispatcher and the stubs. void rt_fatal(const char *fmt, ...) __attribute__((noreturn, format(printf, 1, 2))); +// The same end with an exit code of its own, for a failure that a script or the launcher tells from the others by it +// (vk_device.c vk_device_lost_check ends with RT_LOST_EXIT). rt_fatal ends with RT_FATAL_EXIT; 71 is the crash report's +// (main.c crash_handler). +#define RT_FATAL_EXIT 70 +#define RT_LOST_EXIT 72 +void rt_fatal_code(int code, const char *fmt, ...) __attribute__((noreturn, format(printf, 2, 3))); // Lines of context for a fatal error's report, which rt_fatal and the crash report (main.c crash_handler) write after // their own: fn logs one with rt_log. It runs on the failing thread, which may hold any lock, so it reads only what // needs none. At most four; adding one twice keeps one. rt_fatal_notes runs them, once per process. diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index 73cb442d..569ea61e 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -319,7 +319,7 @@ static void flip_locked(void) { VkResult r = vk_dlssg_active() ? vk_dlssg_present(vk_ctx()->gfx_queue, &pi) : vk_fg_active() ? vk_fg_present(&pi) : vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi); vk_queue_unlock(); - vk_device_fault_report(r); + vk_device_lost_check(r, "the present"); if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1); } @@ -996,7 +996,7 @@ static void ng_present(VkSwapchainKHR sc, uint32_t idx) { vk_queue_lock(); VkResult r = vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi); vk_queue_unlock(); - vk_device_fault_report(r); + vk_device_lost_check(r, "the present of a generated frame"); if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1); __atomic_add_fetch(&ng.presents, 1, __ATOMIC_RELAXED); } @@ -1061,7 +1061,7 @@ static int ng_show(VkImage img, int set, int slot, uint64_t n) { vk_queue_lock(); r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE); vk_queue_unlock(); - if (r != VK_SUCCESS) { vk_device_fault_report(r); return 0; } + if (r != VK_SUCCESS) { vk_device_lost_check(r, "the submit of a generated frame"); return 0; } ng_present(ng.job.sc, j); return 1; } @@ -1331,6 +1331,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w static VkFormat last = VK_FORMAT_UNDEFINED; if (src && fmt != last) last = fmt, rt_log("gfx: the game presents %dx%d images of format %d", w, h, (int)fmt); } + if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): nothing touches the device gfx_window_start(); VkCtx *c = vk_ctx(); plat_rmutex_lock(&g_pl); @@ -1359,8 +1360,8 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w : vk_fg_active() ? vk_fg_acquire(g_sc, 1000000000ull, g_acquire[slot], &idx) : vkAcquireNextImageKHR(c->device, g_sc, 1000000000ull, g_acquire[slot], VK_NULL_HANDLE, &idx); } - vk_device_fault_report(r); - // lost, not ready: skip the frame. Acquires that keep timing out (1 s each) mean a swapchain nothing will free an + vk_device_lost_check(r, "the acquire of a swapchain image"); + // not ready: skip the frame. Acquires that keep timing out (1 s each) mean a swapchain nothing will free an // image of again: three in a row rebuild it, where the picture used to stop for good. static uint32_t n_timeouts; if (r == VK_TIMEOUT || r == VK_NOT_READY) { @@ -1554,7 +1555,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w g_flip_idx = idx; if (!defer) flip_locked(); } else { - vk_device_fault_report(r); + vk_device_lost_check(r, "the present's submit"); rt_log("gfx: present submit failed (%d)", (int)r); // The fence was reset after the acquire and the acquire's semaphore is signalled: left so, this slot's fence // wait would time out (5 s) at every later present and the picture stop for good. An empty batch takes the @@ -1594,6 +1595,7 @@ void vk_present_blit(VkImage src, VkImageLayout layout, VkFormat fmt, int w, int void gfx_present_boot(void) { present_image(VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED, VK_FORMAT_UNDEFINED, 1920, 1080, NULL, 0); } void vk_present_flip(void) { + if (vk_device_is_lost()) return; plat_rmutex_lock(&g_pl); flip_locked(); plat_rmutex_unlock(&g_pl); diff --git a/runtime/host.h b/runtime/host.h index f0b27bf6..c905b0b2 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -428,6 +428,7 @@ int probe_discord(void); // discord_rpc.c int probe_loader(void), probe_kernel(void), probe_ipc(void), probe_services(void), probe_fs(void); int probe_gfx_abi(void); // gfx_abi_probe.c int probe_vkdevice(void); // vk_device.c +int vk_sync_probe_devlost(void); // vk_sync.c: --probe-devlost int probe_nvn(void), probe_nvn_tex(void); // nvn_probe.c, nvn_tex_probe.c int probe_window(void); // gfx_window_probe.c int probe_shader(void); // gfx_shader.c diff --git a/runtime/main.c b/runtime/main.c index 0722a6c9..d946e539 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -534,7 +534,7 @@ static int usage(const char *argv0) { "--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " "--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | --probe-coop | " - "--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0); + "--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0); return 2; } @@ -685,6 +685,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--probe-zstd")) return probe_zstd(); if (!strcmp(mode, "--probe-input")) return probe_gamepad(); if (!strcmp(mode, "--probe-vkdevice")) return probe_vkdevice(); + if (!strcmp(mode, "--probe-devlost")) return vk_sync_probe_devlost(); if (!strcmp(mode, "--probe-movie")) { // the AV1 decoder tells the graphics back end of each frame it writes: none here cfg.gpu = "null"; return probe_movie(); diff --git a/runtime/vk.h b/runtime/vk.h index 0ba62f80..809681b4 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -80,13 +80,25 @@ VkCtx *vk_ctx(void); // NULL until vk_init() succeeded int vk_init(void); // idempotent; 1 on success, 0 on failure (reason logged with rt_log) void vk_shutdown(void); int vk_selftest(void); // 0 on success -// Call with the result of a failed queue submit, present or wait before stopping on it: VK_ERROR_DEVICE_LOST logs -// what VK_EXT_device_fault reports (description, faulting addresses with their precision, vendor codes) once per -// process, after the GPU breadcrumbs (vk_crumbs_report). Any other result returns at once. -void vk_device_fault_report(VkResult r); +// Call with the result of every queue submit, present, acquire or wait. Any result but VK_ERROR_DEVICE_LOST returns at +// once, and the caller goes on with its own handling. A lost device ends the run here, in one place: the first thread +// to arrive sets vk_lost, logs the GPU breadcrumbs (vk_crumbs_report) and what VK_EXT_device_fault reports +// (description, faulting addresses with their precision, vendor codes), then ends the process as rt_fatal does (the +// log written, a save commit under way finished) with exit code RT_LOST_EXIT (dispatch.h). `where` names the call for +// the fatal line. Every later thread stops in this call and never returns to touch the device; the process ends +// within LOST_EXIT_MS (vk_device.c) even when the first thread hangs on a lock one of them holds. +void vk_device_lost_check(VkResult r, const char *where); +// 1 from the moment a thread saw the device lost. The paths that would touch the device (the replay and its submit, +// the uploads, the present, the completion thread's callbacks) read it where they start and return without doing so. +extern volatile int32_t vk_lost; +static inline int vk_device_is_lost(void) { return __atomic_load_n(&vk_lost, __ATOMIC_RELAXED); } +// Asks the device whether it is lost with a call that has a result and needs no lock (the status of a fence nothing +// signals), and ends the run through vk_device_lost_check when it is. The completion thread calls it before it runs +// callbacks: a lost device may report the submissions' timeline as reached (vk_sync.c completion_main). +void vk_device_lost_poll(void); // ---- GPU address history (vk_vaddr.c, NX2NAVI_VA_REPORT=1, 2 with call stacks): VK_EXT_device_address_binding_report's -// reports, kept so vk_device_fault_report can name the objects that held a faulting address. +// reports, kept so the lost device report (vk_device.c device_fault_report) can name the objects that held a faulting address. int vk_vaddr_wanted(void); void vk_vaddr_start(VkInstance instance); // create_instance, after vkCreateInstance void vk_vaddr_device(int supported); // create_device: the driver reports address bindings @@ -157,7 +169,7 @@ const VkPassMark *vk_pass_mark_record(uint32_t id); // the ring record of mark // The report's text for a started and a finished mark, three lines without a trailing newline: which marks (source: // the extension they came from), then each with its ring record. vk_pass_marks_report logs it; the probe checks it. void vk_pass_marks_format(const char *source, uint32_t started, uint32_t finished, char *o, size_t size); -void vk_pass_marks_report(void); // vk_device_fault_report: the AMD words and the NVIDIA checkpoints, formatted +void vk_pass_marks_report(void); // vk_device.c device_fault_report: the AMD words and the NVIDIA checkpoints, formatted // ---- window and present (gfx_present.c). Host threads only. // vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 6e1384b5..1d0311cc 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -25,6 +25,10 @@ static uint32_t g_gfx_queues = 1; // queues the graphics family offers (pick_d static VkCtx g_vk; static int g_ready; static PlatMutex g_lock = PLAT_MUTEX_INIT; +// The fence vk_device_lost_poll asks about. Nothing submits with it, so its status is "not ready" for as long as the +// device lives. +static VkFence lost_fence; +static PlatMutex lost_fence_lock = PLAT_MUTEX_INIT; static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type, const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) { @@ -550,6 +554,8 @@ static void destroy_all(void) { if (g_vk.device) vk_astc_release(); if (g_vk.device) vk_crumbs_release(); if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL); + if (lost_fence) vkDestroyFence(g_vk.device, lost_fence, NULL); // vk_device_lost_poll + lost_fence = VK_NULL_HANDLE; vk_headroom_stop(); if (g_vk.vma) vmaDestroyAllocator(g_vk.vma); if (g_vk.device) vkDestroyDevice(g_vk.device, NULL); @@ -599,15 +605,9 @@ static const char *fault_address_type(VkDeviceFaultAddressTypeEXT t) { } } -// The query runs once per process. Threads that hit the loss at the same time wait on the lock until it is logged, so -// none of them reaches its rt_fatal and ends the process before the report is written. -void vk_device_fault_report(VkResult r) { - static PlatMutex lock = PLAT_MUTEX_INIT; - static int done; - if (r != VK_ERROR_DEVICE_LOST) return; - plat_mutex_lock(&lock); - if (done) { plat_mutex_unlock(&lock); return; } - done = 1; +// The report of a lost device: the breadcrumbs, the pass marks, the video memory and the driver's fault information. +// Once per process, on the thread that saw the loss first (vk_device_lost_check). +static void device_fault_report(void) { vk_crumbs_report(); vk_pass_marks_report(); vk_mem_lost_report(); @@ -619,7 +619,6 @@ void vk_device_fault_report(VkResult r) { rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point" : "not enabled"); vk_vaddr_dump(); // the history still names what held a page another report (the kernel's log) gives - plat_mutex_unlock(&lock); return; } VkDeviceFaultCountsEXT n = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT}; @@ -648,7 +647,53 @@ void vk_device_fault_report(VkResult r) { vk_vaddr_dump(); free(ai); free(vi); - plat_mutex_unlock(&lock); +} + +// ---- the end of a run whose device is lost (vk.h vk_device_lost_check) +// A lost device stays lost: every later call on it fails or, worse, answers as if it had worked, and a driver that is +// called on from several threads after the loss can fault inside itself. So the first thread that sees the loss ends +// the run, and nothing else touches the device meanwhile: the paths that would read vk_lost and return, and a thread +// that gets the loss as a result of its own stops here for good. +// LOST_EXIT_MS bounds the end. The first thread writes the report and waits for a save commit (fs_host.c +// fs_save_crash_wait, 2 s at most); should it hang on a lock that a stopped thread holds, the watchdog ends the +// process with the same code, and the lines already written are in the log (rt_fatal_code wrote them itself). +#define LOST_EXIT_MS 10000 +volatile int32_t vk_lost; + +static void lost_wait_exit(void *arg) { + (void)arg; + for (int t = 0; t < LOST_EXIT_MS; t += 50) plat_sleep_ms(50); + _Exit(RT_LOST_EXIT); +} + +void vk_device_lost_check(VkResult r, const char *where) { + if (r != VK_ERROR_DEVICE_LOST) return; + if (plat_xchg32(&vk_lost, 1)) lost_wait_exit(NULL); // another thread is ending the run: this one stops here + plat_thread_spawn(lost_wait_exit, NULL, "vk lost exit", 0); + device_fault_report(); + rt_fatal_code(RT_LOST_EXIT, "vk: the GPU device was lost (seen at: %s). The run ends here; the lines above name what " + "the GPU was doing", where); +} + + +void vk_device_lost_poll(void) { + VkCtx *c = vk_ctx(); + if (!c || vk_device_is_lost()) return; + VkFence f = __atomic_load_n(&lost_fence, __ATOMIC_ACQUIRE); + if (!f) { + plat_mutex_lock(&lost_fence_lock); + if (!lost_fence) { + VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence n = VK_NULL_HANDLE; + VkResult r = vkCreateFence(c->device, &fi, NULL, &n); + if (r == VK_SUCCESS) __atomic_store_n(&lost_fence, n, __ATOMIC_RELEASE); + else vk_device_lost_check(r, "the fence that asks whether the device lives"); + } + f = lost_fence; + plat_mutex_unlock(&lost_fence_lock); + if (!f) return; + } + vk_device_lost_check(vkGetFenceStatus(c->device, f), "a poll of the device"); } int vk_selftest(void) { diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 740e2aba..3698abe4 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1664,6 +1664,7 @@ static void rec_claim(void) { } int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { + if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): nothing is recorded uint64_t in_ns = kern_now_ns(); perf_lock_timing_here(); // NX2NAVI_LOCK_TIME if (prof_replay_tid != plat_thread_id()) perf_replay_thread(); // the profiler follows the thread replaying now diff --git a/runtime/vk_exec_probe.c b/runtime/vk_exec_probe.c index c2152406..f560dae0 100644 --- a/runtime/vk_exec_probe.c +++ b/runtime/vk_exec_probe.c @@ -390,7 +390,7 @@ static uint8_t *draw_prefix(uint8_t *q, prog_t *prog, const pool_t *pool, tex_t // ---- pass marks part (vk_crumbs.c): one submission with a pass of 2 draws into the draw part's target, a dispatch and // a pass of 3 draws into a second target. Once its fence is done, the buffer marks must name the last pass as both // started and finished, the ring records must hold what the three were, and vk_pass_marks_format (the text -// vk_device_fault_report logs) must name it. Without VK_AMD_buffer_marker or with NX2NAVI_PASS_CRUMBS=0 it is skipped: +// vk_device.c device_fault_report logs) must name it. Without VK_AMD_buffer_marker or with NX2NAVI_PASS_CRUMBS=0 it is skipped: // NVIDIA's checkpoints can only be read once the device is lost. static char marks_text[2048]; static int marks_skipped; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index a0e83134..713bd344 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -311,7 +311,7 @@ static void oq_read(VkDevice dev, uint32_t first, uint32_t n) { VkResult r = vkGetQueryPoolResults(dev, oq_pool, first, n, sizeof OQB.v[0] * 2 * n, OQB.v, 2 * sizeof OQB.v[0], VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); if (r != VK_SUCCESS && r != VK_NOT_READY) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the read of occlusion query results"); rt_fatal("vk: occlusion query results failed (%d)", (int)r); } OQB.first = first, OQB.n = n; @@ -353,6 +353,7 @@ static void oq_cb(void *arg) { if (avail) sum += v; else { sum += 1; + vk_device_lost_poll(); // a query its submission never wrote: on a lost device the run ends here LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[i]); } } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index e564d58b..519de236 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2932,6 +2932,7 @@ static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags // memory.c's fetch (frz_lock held there, so one at a time): [off, off + size) of the mirror h0 into the imported // buffer h1, after all the GPU work handed over so far. static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) { + if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): the bytes stay as they are VkCtx *c = vk_ctx(); VkBufferCopy reg = {off, off, size}; VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, @@ -2952,7 +2953,7 @@ static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) { vk_queue_unlock(); if (r == VK_SUCCESS) r = vkWaitForFences(c->device, 1, &lz_fence, VK_TRUE, 10000000000ull); if (r != VK_SUCCESS) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the submit or the wait that brings GPU writes down to guest memory"); rt_fatal("vk: bringing %llu bytes the GPU wrote down to guest memory failed (%d)", (unsigned long long)size, (int)r); } vkResetFences(c->device, 1, &lz_fence); @@ -4109,6 +4110,7 @@ void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) { } uint64_t vk_upload_flush(int wait) { + if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): no submit, no wait uint64_t f0 = kern_now_ns(); plat_once(&cmd_once, cmd_init, NULL); plat_rmutex_lock(&cmd_lock); @@ -4125,7 +4127,7 @@ uint64_t vk_upload_flush(int wait) { VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); if (r != VK_SUCCESS) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the submit of an upload batch"); rt_fatal("vk: upload submit failed (%d)", (int)r); } if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight); @@ -4138,7 +4140,7 @@ uint64_t vk_upload_flush(int wait) { // loop would spin for good, holding the upload lock, instead of the run ending with the device's report VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX); if (w != VK_SUCCESS) { - vk_device_fault_report(w); + vk_device_lost_check(w, "the wait for an upload batch"); rt_fatal("vk: the wait for an upload failed (%d)", (int)w); } retire(c, 0); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 6244a181..af5c3cb1 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -114,11 +114,30 @@ int vk_sync_wb_split(void) { return wb_split && exec_tl != VK_NULL_HANDLE; } +// NX2NAVI_DEVICE_LOST_AT=n (a test of the lost device's end, vk_device.c vk_device_lost_check): the n-th hand-over to +// the queue is not made and counts as one the driver answered with a lost device. 0 or unset: never. +static int lost_at_due(void) { + static int at = -1; + static int32_t calls; + if (at < 0) { + const char *e = cfg_get("NX2NAVI_DEVICE_LOST_AT"); + at = e && *e ? atoi(e) : 0; + if (at < 0) at = 0; + if (at) rt_log("vk: queue hand-over %d will be taken as a lost device (NX2NAVI_DEVICE_LOST_AT)", at); + } + return at && ++calls == at; // queue lock held +} + static void hold_flush_locked(VkCtx *c) { if (!H.n) return; - VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); + if (vk_device_is_lost()) { // the run is ending: the held submissions are dropped, nothing reaches the queue + H.n = 0, H.last = 0; + __atomic_store_n(&held_n, 0, __ATOMIC_RELEASE); + return; + } + VkResult r = lost_at_due() ? VK_ERROR_DEVICE_LOST : vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); if (r != VK_SUCCESS) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the submit of the replay's command buffers"); rt_fatal("vk: vkQueueSubmit2 of %d held submissions up to value %llu failed (%d)", H.n, (unsigned long long)H.last, (int)r); } @@ -421,7 +440,7 @@ static void completion_main(void *arg) { .pSemaphores = &c->timeline, .pValues = &want}; VkResult r = vkWaitSemaphores(c->device, &wi, tmo); // 100 ms: stop is checked between waits if (r != VK_SUCCESS && r != VK_TIMEOUT) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the completion thread's wait for a submission"); rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r); } // A lost device is reported here, where it is first seen. Unchecked, the value was taken as it came: the @@ -430,19 +449,29 @@ static void completion_main(void *arg) { // checked queue call failed, an upload's, which then read as the cause. r = vkGetSemaphoreCounterValue(c->device, c->timeline, &reached); if (r != VK_SUCCESS) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the completion thread's read of the submissions' timeline"); rt_fatal("vk: the submissions' timeline could not be read (%d; device lost?)", (int)r); } + // Both calls may succeed on a lost device and give a value its submissions never reached. So before a + // new value counts, the device is asked once more with a call of its own (vk_device.c + // vk_device_lost_poll, which ends the run when it is lost). + if (reached > __atomic_load_n(&done, __ATOMIC_ACQUIRE)) vk_device_lost_poll(); plat_mutex_lock(&st_lock); if (reached > issued) reached = issued; } + // Another thread saw the device lost (vk_lost): no value counts from here on and no callback runs. Theirs is + // work that never finished: frees of what the GPU may still hold, records handed out again, counter reports. + // done stays where it is, the waiters stay asleep, and the thread that saw the loss ends the process. + if (vk_device_is_lost()) break; while (qn && queue[qhead].fence <= reached) { Pending p = queue[qhead++]; qn--; plat_mutex_unlock(&st_lock); p.fn(p.arg); plat_mutex_lock(&st_lock); + if (vk_device_is_lost()) break; } + if (vk_device_is_lost()) break; if (reached > done) { // The waiters go first (a fence wait, the frame's first submission, the acquire): freeing what the // finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each @@ -595,6 +624,9 @@ uint64_t vk_sync_reserve(void) { static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCommandBuffer, void *), void *arg) { vk_sync_start(); + // The run is ending (vk_device.c vk_device_lost_check): nothing is recorded or submitted and no value is issued. + // The value returned is the last one issued; a wait for it sleeps until the process ends. + if (vk_device_is_lost()) return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); VkCtx *c = vk_ctx(); plat_mutex_lock(&sub_lock); vk_mem_dirty_submit(); // NX2NAVI_TRACE=perf: what a mirror would upload before this submit @@ -730,6 +762,9 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { // fence callback, after the executed timeline: a wait that returned there let the game rewrite a pool that an // older write-back then overwrote (Kokiri Forest: a sky coefficient the game had reset came back, and the world // flickered washed out). So the wait covers the write-backs there, as without the split. + // The run is ending (vk_device.c vk_device_lost_check): no wait on the lost device, and none for a value that + // will not be reached. + if (vk_device_is_lost()) return __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value; if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns); if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; uint64_t p0 = perf_begin(); @@ -739,7 +774,7 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { .pSemaphores = &exec_tl, .pValues = &value}; VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns); if (r != VK_SUCCESS && r != VK_TIMEOUT) { - vk_device_fault_report(r); + vk_device_lost_check(r, "the wait for a submission's commands"); rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r); } perf_end(PERF_FENCE, p0); @@ -791,6 +826,84 @@ void vk_sync_shutdown(void) { plat_mutex_unlock(&start_lock); } +// ---- --probe-devlost (host.h): the end of a run whose device is lost, without a lost device and without a window. +// The device itself stays healthy, so what the driver answers after a real loss is not covered here. +// 1. A callback queued for the next value runs when the timeline reaches it (the check that step 2 can fail). +// 2. With vk_lost set, as after another thread saw the loss: a callback queued for the next value does not run although +// the timeline reaches it, done stays, and a submit, an upload flush and a wait for a submission's commands return +// at once without issuing a value. +// 3. With vk_lost clear again, four threads report a lost device at the same moment: the process must end with +// RT_LOST_EXIT, one report and one FATAL line in the output (tests/device_lost_check.py counts them). Reaching the +// end of this function is the failure. +static volatile int32_t probe_ran[2]; +static void probe_lost_cb(void *arg) { plat_xchg32((volatile int32_t *)arg, 1); } +static void probe_lost_record(VkCommandBuffer cb, void *arg) { (void)cb; plat_xchg32((volatile int32_t *)arg, 1); } +static volatile int32_t probe_go; +static void probe_lost_thread(void *arg) { + static const char *const where[4] = {"probe thread 0", "probe thread 1", "probe thread 2", "probe thread 3"}; + while (!__atomic_load_n(&probe_go, __ATOMIC_ACQUIRE)) {} + vk_device_lost_check(VK_ERROR_DEVICE_LOST, where[(intptr_t)arg & 3]); +} + +// Makes the timeline reach the next value as a finished submission would: issued moves on, the semaphore is signalled +// from the host, and the completion thread is woken. Returns the value. +static uint64_t probe_lost_advance(VkCtx *c, volatile int32_t *ran) { + plat_mutex_lock(&st_lock); + uint64_t v = issued + 1; + plat_mutex_unlock(&st_lock); + vk_completion_add(v, probe_lost_cb, (void *)ran); + VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline, .value = v}; + if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) return 0; + plat_mutex_lock(&st_lock); + __atomic_store_n(&issued, v, __ATOMIC_RELEASE); + plat_cond_broadcast(&cv_work); + plat_mutex_unlock(&st_lock); + return v; +} + +int vk_sync_probe_devlost(void) { + if (!vk_init()) { + printf("probe-devlost: SKIP (no Vulkan device on this machine)\n"); + return 0; + } + vk_sync_start(); + VkCtx *c = vk_ctx(); + uint64_t v1 = probe_lost_advance(c, &probe_ran[0]); + for (int i = 0; i < 200 && !plat_cas32(&probe_ran[0], 1, 1); i++) plat_sleep_ms(10); + if (!v1 || !plat_cas32(&probe_ran[0], 1, 1) || !vk_sync_fence_wait(v1, 2000000000ull)) { + printf("probe-devlost: FAILED: a callback did not run on a healthy device (value %llu)\n", (unsigned long long)v1); + return 1; + } + printf("probe-devlost: healthy device: the callback of value %llu ran\n", (unsigned long long)v1); + + plat_xchg32(&vk_lost, 1); + uint64_t v2 = probe_lost_advance(c, &probe_ran[1]); + plat_sleep_ms(500); // five of the completion thread's 100 ms waits + int ran = plat_cas32(&probe_ran[1], 1, 1), reached = vk_sync_fence_done() >= v2; + volatile int32_t recorded = 0; + uint64_t before = vk_sync_issued(), t0 = plat_now_ms(); + uint64_t got = vk_sync_probe_submit(probe_lost_record, (void *)&recorded); + vk_upload_flush(1); + int waited = vk_sync_exec_wait(v2, 5000000000ull); + uint64_t ms = plat_now_ms() - t0; + int ok = v2 && !ran && !reached && !recorded && got == before && vk_sync_issued() == before && !waited && ms < 1000; + printf("probe-devlost: device marked lost: callback of value %llu %s, done %s, submit %s (issued %llu -> %llu), " + "the paths returned in %llu ms\n", (unsigned long long)v2, ran ? "RAN" : "not run", reached ? "ADVANCED" : "stayed", + recorded ? "RECORDED" : "not recorded", (unsigned long long)before, (unsigned long long)vk_sync_issued(), + (unsigned long long)ms); + if (!ok) { printf("probe-devlost: FAILED\n"); return 1; } + + printf("probe-devlost: four threads report the loss now; the process must end with exit code %d\n", RT_LOST_EXIT); + fflush(stdout); + plat_xchg32(&vk_lost, 0); + for (intptr_t i = 0; i < 4; i++) plat_thread_spawn(probe_lost_thread, (void *)i, "probe lost", 0); + plat_sleep_ms(50); + __atomic_store_n(&probe_go, 1, __ATOMIC_RELEASE); + plat_sleep_ms(20000); + printf("probe-devlost: FAILED: still running 20 s after the loss was reported\n"); + return 1; +} + // NX2NAVI_TRACE=perf "mem: sizes" (perf.c): command buffer slots (read without sub_lock, which a whole replay holds), // the completion queue and the submissions held back. void vk_sync_sizes(PerfSizes *s) { diff --git a/tests/device_lost_check.py b/tests/device_lost_check.py new file mode 100644 index 00000000..81155239 --- /dev/null +++ b/tests/device_lost_check.py @@ -0,0 +1,70 @@ +"""Run the game executable's --probe-devlost and check how a run ends when the GPU device is lost +(runtime/vk_device.c vk_device_lost_check, runtime/vk_sync.c vk_sync_probe_devlost). + +The probe needs a Vulkan device and opens no window. It does not lose the device: it marks it lost, and then has four +threads report a loss at the same moment. Checked here: + + - on the healthy device a completion callback runs (so the next check can fail); + - with the device marked lost no callback runs, no value counts as done, and the submit, upload and wait paths return + at once; + - the process ends with exit code 72 (dispatch.h RT_LOST_EXIT), with one device report and one FATAL line, well + before the watchdog's 10 seconds. + +What a driver answers after a real loss is not covered. + +Usage: device_lost_check.py PATH_TO_GAME_EXE +""" +import subprocess +import sys +import time + +LOST_EXIT = 72 + + +def main(): + if len(sys.argv) != 2: + print(__doc__) + return 2 + p = subprocess.Popen([sys.argv[1], '--probe-devlost'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, errors='replace') + lines, reported_at = [], None + for line in p.stdout: + lines.append(line.rstrip('\n')) + if 'four threads report the loss now' in line: + reported_at = time.monotonic() + code = p.wait() + ended = time.monotonic() + text = '\n'.join(lines) + if 'probe-devlost: SKIP' in text: + print('device_lost_check: SKIP (no Vulkan device)') + return 0 + fails = [] + + def check(ok, what): + if not ok: + fails.append(what) + + check('probe-devlost: healthy device: the callback of value' in text, 'no callback ran on the healthy device') + marked = [ln for ln in lines if ln.startswith('probe-devlost: device marked lost:')] + check(len(marked) == 1 and 'not run, done stayed, submit not recorded' in marked[0], + 'a path went on after the device was marked lost: %s' % (marked or ['no line'])) + check('probe-devlost: FAILED' not in text, 'the probe reports a failure') + check(code == LOST_EXIT, 'exit code %d, expected %d' % (code, LOST_EXIT)) + fatal = [ln for ln in lines if ln.startswith('FATAL: ')] + check(len(fatal) == 1 and 'the GPU device was lost' in fatal[0], '%d FATAL lines, expected one: %s' % (len(fatal), fatal)) + reports = [ln for ln in lines if 'video memory when the device was lost' in ln] + check(len(reports) == 1, '%d device reports, expected one' % len(reports)) + took = ended - reported_at if reported_at else -1 + check(0 <= took < 5, 'the process ended %.1f s after the loss was reported (the watchdog, not the reporting thread?)' % took) + for f in fails: + print('device_lost_check: FAIL:', f) + if fails: + print(text[-3000:]) + return 1 + print('device_lost_check: passed (exit code %d, one report, one FATAL line, %.2f s from the report to the exit)' + % (code, took)) + return 0 + + +if __name__ == '__main__': + sys.exit(main())