End a run whose GPU device is lost in one place: one report, no callbacks of unfinished submissions, and exit code 72.

A lost device used to end the process from whichever submit or wait noticed it first, while the present only logged it
and went on and the other threads kept calling the driver. The completion thread could take the submissions' timeline
as reached and run the callbacks of submissions that never finished.

vk_device_lost_check now takes the result of every submit, present, acquire and wait. The first thread to see the loss
sets a flag, writes the report and ends the process as a fatal error does (the log written, a save commit under way
finished), with an exit code of its own; later threads stop in that call, and a watchdog ends the process after ten
seconds should the first thread hang. The replay, its submit, the uploads, the present and the lazy fetch read the flag
and return without touching the device. The completion thread asks the device with a call of its own before a new
value counts and runs no callback once the flag is set.

The executable's --probe-devlost and tests/device_lost_check.py check this without a lost device, and
device_lost_at in nx2navi.ini ends a real run the same way.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 04:35:30 -05:00
parent 84b0672ec2
commit 534591be04
16 changed files with 319 additions and 41 deletions

View File

@@ -74,7 +74,7 @@ static const char *const known[] = {
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL",
"RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VA_REPORT",
"VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC",
"VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", "DEVICE_LOST_AT",
"COOP", "COOP_NAME", "COOP_SERVER", "COOP_PORT", "COOP_ROOM", "COOP_AUTO", "COOP_DROP", // co-op (coop.c); the last three have no menu row
"COOP_PROGRESS", "COOP_ITEMS", "COOP_SONGS", "COOP_QUEST", "COOP_KEYS", "COOP_RUPEES", "COOP_AMMO", // the room's rules (coop_game.c)
"COOP_TEST_SET", // test: sets flags by hash at given frames (coop_game.c); no menu row
@@ -142,7 +142,7 @@ static const struct { const char *key, *section; } sections[] = {
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"},
};
// The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a
@@ -311,6 +311,7 @@ static const CfgMeta metas[] = {
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha
{"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only
{"VA_REPORT", "0", "0|1|2", RESTART}, // vk_vaddr.c GPU address history for a lost device's report; 2: call stacks
{"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end
{"INPUT_RECORD", "", NULL, RESTART},
};

View File

@@ -59,14 +59,13 @@ void rt_fatal_notes(void) {
__attribute__((weak)) void fs_save_crash_wait(void) {} // fs_host.c; builds without it (the unit checks) have no saves
void rt_fatal(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
// The fatal line, the report's notes and what has to be flushed, then the exit with `code`.
static void fatal_end(int code, const char *fmt, va_list ap) __attribute__((noreturn));
static void fatal_end(int code, const char *fmt, va_list ap) {
rt_log_sync(); // main.c: the lines queued for the log thread, then these, written here
rt_log_raw("FATAL: ", 7);
rt_vlogf(fmt, ap);
rt_log_raw("\n", 1);
va_end(ap);
// A failed allocation (malloc, realloc, VirtualAlloc) means the commit limit was reached: this line tells a leak of
// ours (process private bytes near the limit) from a machine short of page file.
double proc, used, limit;
@@ -81,7 +80,19 @@ void rt_fatal(const char *fmt, ...) {
rtss_limit_restore();
fflush(stdout);
fflush(stderr);
_Exit(70);
_Exit(code);
}
void rt_fatal(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
fatal_end(RT_FATAL_EXIT, fmt, ap);
}
void rt_fatal_code(int code, const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
fatal_end(code, fmt, ap);
}
static const RtModule *module_at(uint64_t pc) {

View File

@@ -51,6 +51,12 @@ RtDispatchStats rt_dispatch_stats(void);
// Runtime fatal error: message to stderr, then exit. Used by the dispatcher and the stubs.
void rt_fatal(const char *fmt, ...) __attribute__((noreturn, format(printf, 1, 2)));
// The same end with an exit code of its own, for a failure that a script or the launcher tells from the others by it
// (vk_device.c vk_device_lost_check ends with RT_LOST_EXIT). rt_fatal ends with RT_FATAL_EXIT; 71 is the crash report's
// (main.c crash_handler).
#define RT_FATAL_EXIT 70
#define RT_LOST_EXIT 72
void rt_fatal_code(int code, const char *fmt, ...) __attribute__((noreturn, format(printf, 2, 3)));
// Lines of context for a fatal error's report, which rt_fatal and the crash report (main.c crash_handler) write after
// their own: fn logs one with rt_log. It runs on the failing thread, which may hold any lock, so it reads only what
// needs none. At most four; adding one twice keeps one. rt_fatal_notes runs them, once per process.

View File

@@ -319,7 +319,7 @@ static void flip_locked(void) {
VkResult r = vk_dlssg_active() ? vk_dlssg_present(vk_ctx()->gfx_queue, &pi)
: vk_fg_active() ? vk_fg_present(&pi) : vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi);
vk_queue_unlock();
vk_device_fault_report(r);
vk_device_lost_check(r, "the present");
if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1);
}
@@ -996,7 +996,7 @@ static void ng_present(VkSwapchainKHR sc, uint32_t idx) {
vk_queue_lock();
VkResult r = vkQueuePresentKHR(vk_ctx()->gfx_queue, &pi);
vk_queue_unlock();
vk_device_fault_report(r);
vk_device_lost_check(r, "the present of a generated frame");
if (r == VK_ERROR_OUT_OF_DATE_KHR || r == VK_SUBOPTIMAL_KHR) plat_xchg32(&g_dirty, 1);
__atomic_add_fetch(&ng.presents, 1, __ATOMIC_RELAXED);
}
@@ -1061,7 +1061,7 @@ static int ng_show(VkImage img, int set, int slot, uint64_t n) {
vk_queue_lock();
r = vkQueueSubmit2(c->gfx_queue, 1, &si, VK_NULL_HANDLE);
vk_queue_unlock();
if (r != VK_SUCCESS) { vk_device_fault_report(r); return 0; }
if (r != VK_SUCCESS) { vk_device_lost_check(r, "the submit of a generated frame"); return 0; }
ng_present(ng.job.sc, j);
return 1;
}
@@ -1331,6 +1331,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w
static VkFormat last = VK_FORMAT_UNDEFINED;
if (src && fmt != last) last = fmt, rt_log("gfx: the game presents %dx%d images of format %d", w, h, (int)fmt);
}
if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): nothing touches the device
gfx_window_start();
VkCtx *c = vk_ctx();
plat_rmutex_lock(&g_pl);
@@ -1359,8 +1360,8 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w
: vk_fg_active() ? vk_fg_acquire(g_sc, 1000000000ull, g_acquire[slot], &idx)
: vkAcquireNextImageKHR(c->device, g_sc, 1000000000ull, g_acquire[slot], VK_NULL_HANDLE, &idx);
}
vk_device_fault_report(r);
// lost, not ready: skip the frame. Acquires that keep timing out (1 s each) mean a swapchain nothing will free an
vk_device_lost_check(r, "the acquire of a swapchain image");
// not ready: skip the frame. Acquires that keep timing out (1 s each) mean a swapchain nothing will free an
// image of again: three in a row rebuild it, where the picture used to stop for good.
static uint32_t n_timeouts;
if (r == VK_TIMEOUT || r == VK_NOT_READY) {
@@ -1554,7 +1555,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w
g_flip_idx = idx;
if (!defer) flip_locked();
} else {
vk_device_fault_report(r);
vk_device_lost_check(r, "the present's submit");
rt_log("gfx: present submit failed (%d)", (int)r);
// The fence was reset after the acquire and the acquire's semaphore is signalled: left so, this slot's fence
// wait would time out (5 s) at every later present and the picture stop for good. An empty batch takes the
@@ -1594,6 +1595,7 @@ void vk_present_blit(VkImage src, VkImageLayout layout, VkFormat fmt, int w, int
void gfx_present_boot(void) { present_image(VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED, VK_FORMAT_UNDEFINED, 1920, 1080, NULL, 0); }
void vk_present_flip(void) {
if (vk_device_is_lost()) return;
plat_rmutex_lock(&g_pl);
flip_locked();
plat_rmutex_unlock(&g_pl);

View File

@@ -428,6 +428,7 @@ int probe_discord(void); // discord_rpc.c
int probe_loader(void), probe_kernel(void), probe_ipc(void), probe_services(void), probe_fs(void);
int probe_gfx_abi(void); // gfx_abi_probe.c
int probe_vkdevice(void); // vk_device.c
int vk_sync_probe_devlost(void); // vk_sync.c: --probe-devlost
int probe_nvn(void), probe_nvn_tex(void); // nvn_probe.c, nvn_tex_probe.c
int probe_window(void); // gfx_window_probe.c
int probe_shader(void); // gfx_shader.c

View File

@@ -534,7 +534,7 @@ static int usage(const char *argv0) {
"--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | "
"--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | "
"--probe-vkexec | --probe-astc | --probe-zstd | --probe-input | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | --probe-coop | "
"--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --boot |--seconds S [--shot PNG]]\n", argv0);
"--prepare-shaders | --warm-shaders | --prepare-pipelines | --probe-prewarm | --probe-devlost | --boot |--seconds S [--shot PNG]]\n", argv0);
return 2;
}
@@ -685,6 +685,7 @@ int main(int argc, char **argv) {
if (!strcmp(mode, "--probe-zstd")) return probe_zstd();
if (!strcmp(mode, "--probe-input")) return probe_gamepad();
if (!strcmp(mode, "--probe-vkdevice")) return probe_vkdevice();
if (!strcmp(mode, "--probe-devlost")) return vk_sync_probe_devlost();
if (!strcmp(mode, "--probe-movie")) { // the AV1 decoder tells the graphics back end of each frame it writes: none here
cfg.gpu = "null";
return probe_movie();

View File

@@ -80,13 +80,25 @@ VkCtx *vk_ctx(void); // NULL until vk_init() succeeded
int vk_init(void); // idempotent; 1 on success, 0 on failure (reason logged with rt_log)
void vk_shutdown(void);
int vk_selftest(void); // 0 on success
// Call with the result of a failed queue submit, present or wait before stopping on it: VK_ERROR_DEVICE_LOST logs
// what VK_EXT_device_fault reports (description, faulting addresses with their precision, vendor codes) once per
// process, after the GPU breadcrumbs (vk_crumbs_report). Any other result returns at once.
void vk_device_fault_report(VkResult r);
// Call with the result of every queue submit, present, acquire or wait. Any result but VK_ERROR_DEVICE_LOST returns at
// once, and the caller goes on with its own handling. A lost device ends the run here, in one place: the first thread
// to arrive sets vk_lost, logs the GPU breadcrumbs (vk_crumbs_report) and what VK_EXT_device_fault reports
// (description, faulting addresses with their precision, vendor codes), then ends the process as rt_fatal does (the
// log written, a save commit under way finished) with exit code RT_LOST_EXIT (dispatch.h). `where` names the call for
// the fatal line. Every later thread stops in this call and never returns to touch the device; the process ends
// within LOST_EXIT_MS (vk_device.c) even when the first thread hangs on a lock one of them holds.
void vk_device_lost_check(VkResult r, const char *where);
// 1 from the moment a thread saw the device lost. The paths that would touch the device (the replay and its submit,
// the uploads, the present, the completion thread's callbacks) read it where they start and return without doing so.
extern volatile int32_t vk_lost;
static inline int vk_device_is_lost(void) { return __atomic_load_n(&vk_lost, __ATOMIC_RELAXED); }
// Asks the device whether it is lost with a call that has a result and needs no lock (the status of a fence nothing
// signals), and ends the run through vk_device_lost_check when it is. The completion thread calls it before it runs
// callbacks: a lost device may report the submissions' timeline as reached (vk_sync.c completion_main).
void vk_device_lost_poll(void);
// ---- GPU address history (vk_vaddr.c, NX2NAVI_VA_REPORT=1, 2 with call stacks): VK_EXT_device_address_binding_report's
// reports, kept so vk_device_fault_report can name the objects that held a faulting address.
// reports, kept so the lost device report (vk_device.c device_fault_report) can name the objects that held a faulting address.
int vk_vaddr_wanted(void);
void vk_vaddr_start(VkInstance instance); // create_instance, after vkCreateInstance
void vk_vaddr_device(int supported); // create_device: the driver reports address bindings
@@ -157,7 +169,7 @@ const VkPassMark *vk_pass_mark_record(uint32_t id); // the ring record of mark
// The report's text for a started and a finished mark, three lines without a trailing newline: which marks (source:
// the extension they came from), then each with its ring record. vk_pass_marks_report logs it; the probe checks it.
void vk_pass_marks_format(const char *source, uint32_t started, uint32_t finished, char *o, size_t size);
void vk_pass_marks_report(void); // vk_device_fault_report: the AMD words and the NVIDIA checkpoints, formatted
void vk_pass_marks_report(void); // vk_device.c device_fault_report: the AMD words and the NVIDIA checkpoints, formatted
// ---- window and present (gfx_present.c). Host threads only.
// vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled

View File

@@ -25,6 +25,10 @@ static uint32_t g_gfx_queues = 1; // queues the graphics family offers (pick_d
static VkCtx g_vk;
static int g_ready;
static PlatMutex g_lock = PLAT_MUTEX_INIT;
// The fence vk_device_lost_poll asks about. Nothing submits with it, so its status is "not ready" for as long as the
// device lives.
static VkFence lost_fence;
static PlatMutex lost_fence_lock = PLAT_MUTEX_INIT;
static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type,
const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) {
@@ -550,6 +554,8 @@ static void destroy_all(void) {
if (g_vk.device) vk_astc_release();
if (g_vk.device) vk_crumbs_release();
if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL);
if (lost_fence) vkDestroyFence(g_vk.device, lost_fence, NULL); // vk_device_lost_poll
lost_fence = VK_NULL_HANDLE;
vk_headroom_stop();
if (g_vk.vma) vmaDestroyAllocator(g_vk.vma);
if (g_vk.device) vkDestroyDevice(g_vk.device, NULL);
@@ -599,15 +605,9 @@ static const char *fault_address_type(VkDeviceFaultAddressTypeEXT t) {
}
}
// The query runs once per process. Threads that hit the loss at the same time wait on the lock until it is logged, so
// none of them reaches its rt_fatal and ends the process before the report is written.
void vk_device_fault_report(VkResult r) {
static PlatMutex lock = PLAT_MUTEX_INIT;
static int done;
if (r != VK_ERROR_DEVICE_LOST) return;
plat_mutex_lock(&lock);
if (done) { plat_mutex_unlock(&lock); return; }
done = 1;
// The report of a lost device: the breadcrumbs, the pass marks, the video memory and the driver's fault information.
// Once per process, on the thread that saw the loss first (vk_device_lost_check).
static void device_fault_report(void) {
vk_crumbs_report();
vk_pass_marks_report();
vk_mem_lost_report();
@@ -619,7 +619,6 @@ void vk_device_fault_report(VkResult r) {
rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point"
: "not enabled");
vk_vaddr_dump(); // the history still names what held a page another report (the kernel's log) gives
plat_mutex_unlock(&lock);
return;
}
VkDeviceFaultCountsEXT n = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT};
@@ -648,7 +647,53 @@ void vk_device_fault_report(VkResult r) {
vk_vaddr_dump();
free(ai);
free(vi);
plat_mutex_unlock(&lock);
}
// ---- the end of a run whose device is lost (vk.h vk_device_lost_check)
// A lost device stays lost: every later call on it fails or, worse, answers as if it had worked, and a driver that is
// called on from several threads after the loss can fault inside itself. So the first thread that sees the loss ends
// the run, and nothing else touches the device meanwhile: the paths that would read vk_lost and return, and a thread
// that gets the loss as a result of its own stops here for good.
// LOST_EXIT_MS bounds the end. The first thread writes the report and waits for a save commit (fs_host.c
// fs_save_crash_wait, 2 s at most); should it hang on a lock that a stopped thread holds, the watchdog ends the
// process with the same code, and the lines already written are in the log (rt_fatal_code wrote them itself).
#define LOST_EXIT_MS 10000
volatile int32_t vk_lost;
static void lost_wait_exit(void *arg) {
(void)arg;
for (int t = 0; t < LOST_EXIT_MS; t += 50) plat_sleep_ms(50);
_Exit(RT_LOST_EXIT);
}
void vk_device_lost_check(VkResult r, const char *where) {
if (r != VK_ERROR_DEVICE_LOST) return;
if (plat_xchg32(&vk_lost, 1)) lost_wait_exit(NULL); // another thread is ending the run: this one stops here
plat_thread_spawn(lost_wait_exit, NULL, "vk lost exit", 0);
device_fault_report();
rt_fatal_code(RT_LOST_EXIT, "vk: the GPU device was lost (seen at: %s). The run ends here; the lines above name what "
"the GPU was doing", where);
}
void vk_device_lost_poll(void) {
VkCtx *c = vk_ctx();
if (!c || vk_device_is_lost()) return;
VkFence f = __atomic_load_n(&lost_fence, __ATOMIC_ACQUIRE);
if (!f) {
plat_mutex_lock(&lost_fence_lock);
if (!lost_fence) {
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
VkFence n = VK_NULL_HANDLE;
VkResult r = vkCreateFence(c->device, &fi, NULL, &n);
if (r == VK_SUCCESS) __atomic_store_n(&lost_fence, n, __ATOMIC_RELEASE);
else vk_device_lost_check(r, "the fence that asks whether the device lives");
}
f = lost_fence;
plat_mutex_unlock(&lost_fence_lock);
if (!f) return;
}
vk_device_lost_check(vkGetFenceStatus(c->device, f), "a poll of the device");
}
int vk_selftest(void) {

View File

@@ -1664,6 +1664,7 @@ static void rec_claim(void) {
}
int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) {
if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): nothing is recorded
uint64_t in_ns = kern_now_ns();
perf_lock_timing_here(); // NX2NAVI_LOCK_TIME
if (prof_replay_tid != plat_thread_id()) perf_replay_thread(); // the profiler follows the thread replaying now

View File

@@ -390,7 +390,7 @@ static uint8_t *draw_prefix(uint8_t *q, prog_t *prog, const pool_t *pool, tex_t
// ---- pass marks part (vk_crumbs.c): one submission with a pass of 2 draws into the draw part's target, a dispatch and
// a pass of 3 draws into a second target. Once its fence is done, the buffer marks must name the last pass as both
// started and finished, the ring records must hold what the three were, and vk_pass_marks_format (the text
// vk_device_fault_report logs) must name it. Without VK_AMD_buffer_marker or with NX2NAVI_PASS_CRUMBS=0 it is skipped:
// vk_device.c device_fault_report logs) must name it. Without VK_AMD_buffer_marker or with NX2NAVI_PASS_CRUMBS=0 it is skipped:
// NVIDIA's checkpoints can only be read once the device is lost.
static char marks_text[2048];
static int marks_skipped;

View File

@@ -311,7 +311,7 @@ static void oq_read(VkDevice dev, uint32_t first, uint32_t n) {
VkResult r = vkGetQueryPoolResults(dev, oq_pool, first, n, sizeof OQB.v[0] * 2 * n, OQB.v, 2 * sizeof OQB.v[0],
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
if (r != VK_SUCCESS && r != VK_NOT_READY) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the read of occlusion query results");
rt_fatal("vk: occlusion query results failed (%d)", (int)r);
}
OQB.first = first, OQB.n = n;
@@ -353,6 +353,7 @@ static void oq_cb(void *arg) {
if (avail) sum += v;
else {
sum += 1;
vk_device_lost_poll(); // a query its submission never wrote: on a lost device the run ends here
LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[i]);
}
}

View File

@@ -2932,6 +2932,7 @@ static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags
// memory.c's fetch (frz_lock held there, so one at a time): [off, off + size) of the mirror h0 into the imported
// buffer h1, after all the GPU work handed over so far.
static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) {
if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): the bytes stay as they are
VkCtx *c = vk_ctx();
VkBufferCopy reg = {off, off, size};
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
@@ -2952,7 +2953,7 @@ static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) {
vk_queue_unlock();
if (r == VK_SUCCESS) r = vkWaitForFences(c->device, 1, &lz_fence, VK_TRUE, 10000000000ull);
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the submit or the wait that brings GPU writes down to guest memory");
rt_fatal("vk: bringing %llu bytes the GPU wrote down to guest memory failed (%d)", (unsigned long long)size, (int)r);
}
vkResetFences(c->device, 1, &lz_fence);
@@ -4109,6 +4110,7 @@ void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) {
}
uint64_t vk_upload_flush(int wait) {
if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): no submit, no wait
uint64_t f0 = kern_now_ns();
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
@@ -4125,7 +4127,7 @@ uint64_t vk_upload_flush(int wait) {
VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence);
vk_queue_unlock();
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the submit of an upload batch");
rt_fatal("vk: upload submit failed (%d)", (int)r);
}
if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight);
@@ -4138,7 +4140,7 @@ uint64_t vk_upload_flush(int wait) {
// loop would spin for good, holding the upload lock, instead of the run ending with the device's report
VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX);
if (w != VK_SUCCESS) {
vk_device_fault_report(w);
vk_device_lost_check(w, "the wait for an upload batch");
rt_fatal("vk: the wait for an upload failed (%d)", (int)w);
}
retire(c, 0);

View File

@@ -114,11 +114,30 @@ int vk_sync_wb_split(void) {
return wb_split && exec_tl != VK_NULL_HANDLE;
}
// NX2NAVI_DEVICE_LOST_AT=n (a test of the lost device's end, vk_device.c vk_device_lost_check): the n-th hand-over to
// the queue is not made and counts as one the driver answered with a lost device. 0 or unset: never.
static int lost_at_due(void) {
static int at = -1;
static int32_t calls;
if (at < 0) {
const char *e = cfg_get("NX2NAVI_DEVICE_LOST_AT");
at = e && *e ? atoi(e) : 0;
if (at < 0) at = 0;
if (at) rt_log("vk: queue hand-over %d will be taken as a lost device (NX2NAVI_DEVICE_LOST_AT)", at);
}
return at && ++calls == at; // queue lock held
}
static void hold_flush_locked(VkCtx *c) {
if (!H.n) return;
VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
if (vk_device_is_lost()) { // the run is ending: the held submissions are dropped, nothing reaches the queue
H.n = 0, H.last = 0;
__atomic_store_n(&held_n, 0, __ATOMIC_RELEASE);
return;
}
VkResult r = lost_at_due() ? VK_ERROR_DEVICE_LOST : vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the submit of the replay's command buffers");
rt_fatal("vk: vkQueueSubmit2 of %d held submissions up to value %llu failed (%d)", H.n,
(unsigned long long)H.last, (int)r);
}
@@ -421,7 +440,7 @@ static void completion_main(void *arg) {
.pSemaphores = &c->timeline, .pValues = &want};
VkResult r = vkWaitSemaphores(c->device, &wi, tmo); // 100 ms: stop is checked between waits
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the completion thread's wait for a submission");
rt_fatal("vk: vkWaitSemaphores failed (%d; device lost?)", (int)r);
}
// A lost device is reported here, where it is first seen. Unchecked, the value was taken as it came: the
@@ -430,19 +449,29 @@ static void completion_main(void *arg) {
// checked queue call failed, an upload's, which then read as the cause.
r = vkGetSemaphoreCounterValue(c->device, c->timeline, &reached);
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the completion thread's read of the submissions' timeline");
rt_fatal("vk: the submissions' timeline could not be read (%d; device lost?)", (int)r);
}
// Both calls may succeed on a lost device and give a value its submissions never reached. So before a
// new value counts, the device is asked once more with a call of its own (vk_device.c
// vk_device_lost_poll, which ends the run when it is lost).
if (reached > __atomic_load_n(&done, __ATOMIC_ACQUIRE)) vk_device_lost_poll();
plat_mutex_lock(&st_lock);
if (reached > issued) reached = issued;
}
// Another thread saw the device lost (vk_lost): no value counts from here on and no callback runs. Theirs is
// work that never finished: frees of what the GPU may still hold, records handed out again, counter reports.
// done stays where it is, the waiters stay asleep, and the thread that saw the loss ends the process.
if (vk_device_is_lost()) break;
while (qn && queue[qhead].fence <= reached) {
Pending p = queue[qhead++];
qn--;
plat_mutex_unlock(&st_lock);
p.fn(p.arg);
plat_mutex_lock(&st_lock);
if (vk_device_is_lost()) break;
}
if (vk_device_is_lost()) break;
if (reached > done) {
// The waiters go first (a fence wait, the frame's first submission, the acquire): freeing what the
// finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each
@@ -595,6 +624,9 @@ uint64_t vk_sync_reserve(void) {
static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCommandBuffer, void *), void *arg) {
vk_sync_start();
// The run is ending (vk_device.c vk_device_lost_check): nothing is recorded or submitted and no value is issued.
// The value returned is the last one issued; a wait for it sleeps until the process ends.
if (vk_device_is_lost()) return __atomic_load_n(&issued, __ATOMIC_ACQUIRE);
VkCtx *c = vk_ctx();
plat_mutex_lock(&sub_lock);
vk_mem_dirty_submit(); // NX2NAVI_TRACE=perf: what a mirror would upload before this submit
@@ -730,6 +762,9 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
// fence callback, after the executed timeline: a wait that returned there let the game rewrite a pool that an
// older write-back then overwrote (Kokiri Forest: a sky coefficient the game had reset came back, and the world
// flickered washed out). So the wait covers the write-backs there, as without the split.
// The run is ending (vk_device.c vk_device_lost_check): no wait on the lost device, and none for a value that
// will not be reached.
if (vk_device_is_lost()) return __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value;
if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns);
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1;
uint64_t p0 = perf_begin();
@@ -739,7 +774,7 @@ int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
.pSemaphores = &exec_tl, .pValues = &value};
VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns);
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the wait for a submission's commands");
rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r);
}
perf_end(PERF_FENCE, p0);
@@ -791,6 +826,84 @@ void vk_sync_shutdown(void) {
plat_mutex_unlock(&start_lock);
}
// ---- --probe-devlost (host.h): the end of a run whose device is lost, without a lost device and without a window.
// The device itself stays healthy, so what the driver answers after a real loss is not covered here.
// 1. A callback queued for the next value runs when the timeline reaches it (the check that step 2 can fail).
// 2. With vk_lost set, as after another thread saw the loss: a callback queued for the next value does not run although
// the timeline reaches it, done stays, and a submit, an upload flush and a wait for a submission's commands return
// at once without issuing a value.
// 3. With vk_lost clear again, four threads report a lost device at the same moment: the process must end with
// RT_LOST_EXIT, one report and one FATAL line in the output (tests/device_lost_check.py counts them). Reaching the
// end of this function is the failure.
static volatile int32_t probe_ran[2];
static void probe_lost_cb(void *arg) { plat_xchg32((volatile int32_t *)arg, 1); }
static void probe_lost_record(VkCommandBuffer cb, void *arg) { (void)cb; plat_xchg32((volatile int32_t *)arg, 1); }
static volatile int32_t probe_go;
static void probe_lost_thread(void *arg) {
static const char *const where[4] = {"probe thread 0", "probe thread 1", "probe thread 2", "probe thread 3"};
while (!__atomic_load_n(&probe_go, __ATOMIC_ACQUIRE)) {}
vk_device_lost_check(VK_ERROR_DEVICE_LOST, where[(intptr_t)arg & 3]);
}
// Makes the timeline reach the next value as a finished submission would: issued moves on, the semaphore is signalled
// from the host, and the completion thread is woken. Returns the value.
static uint64_t probe_lost_advance(VkCtx *c, volatile int32_t *ran) {
plat_mutex_lock(&st_lock);
uint64_t v = issued + 1;
plat_mutex_unlock(&st_lock);
vk_completion_add(v, probe_lost_cb, (void *)ran);
VkSemaphoreSignalInfo si = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, .semaphore = c->timeline, .value = v};
if (vkSignalSemaphore(c->device, &si) != VK_SUCCESS) return 0;
plat_mutex_lock(&st_lock);
__atomic_store_n(&issued, v, __ATOMIC_RELEASE);
plat_cond_broadcast(&cv_work);
plat_mutex_unlock(&st_lock);
return v;
}
int vk_sync_probe_devlost(void) {
if (!vk_init()) {
printf("probe-devlost: SKIP (no Vulkan device on this machine)\n");
return 0;
}
vk_sync_start();
VkCtx *c = vk_ctx();
uint64_t v1 = probe_lost_advance(c, &probe_ran[0]);
for (int i = 0; i < 200 && !plat_cas32(&probe_ran[0], 1, 1); i++) plat_sleep_ms(10);
if (!v1 || !plat_cas32(&probe_ran[0], 1, 1) || !vk_sync_fence_wait(v1, 2000000000ull)) {
printf("probe-devlost: FAILED: a callback did not run on a healthy device (value %llu)\n", (unsigned long long)v1);
return 1;
}
printf("probe-devlost: healthy device: the callback of value %llu ran\n", (unsigned long long)v1);
plat_xchg32(&vk_lost, 1);
uint64_t v2 = probe_lost_advance(c, &probe_ran[1]);
plat_sleep_ms(500); // five of the completion thread's 100 ms waits
int ran = plat_cas32(&probe_ran[1], 1, 1), reached = vk_sync_fence_done() >= v2;
volatile int32_t recorded = 0;
uint64_t before = vk_sync_issued(), t0 = plat_now_ms();
uint64_t got = vk_sync_probe_submit(probe_lost_record, (void *)&recorded);
vk_upload_flush(1);
int waited = vk_sync_exec_wait(v2, 5000000000ull);
uint64_t ms = plat_now_ms() - t0;
int ok = v2 && !ran && !reached && !recorded && got == before && vk_sync_issued() == before && !waited && ms < 1000;
printf("probe-devlost: device marked lost: callback of value %llu %s, done %s, submit %s (issued %llu -> %llu), "
"the paths returned in %llu ms\n", (unsigned long long)v2, ran ? "RAN" : "not run", reached ? "ADVANCED" : "stayed",
recorded ? "RECORDED" : "not recorded", (unsigned long long)before, (unsigned long long)vk_sync_issued(),
(unsigned long long)ms);
if (!ok) { printf("probe-devlost: FAILED\n"); return 1; }
printf("probe-devlost: four threads report the loss now; the process must end with exit code %d\n", RT_LOST_EXIT);
fflush(stdout);
plat_xchg32(&vk_lost, 0);
for (intptr_t i = 0; i < 4; i++) plat_thread_spawn(probe_lost_thread, (void *)i, "probe lost", 0);
plat_sleep_ms(50);
__atomic_store_n(&probe_go, 1, __ATOMIC_RELEASE);
plat_sleep_ms(20000);
printf("probe-devlost: FAILED: still running 20 s after the loss was reported\n");
return 1;
}
// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): command buffer slots (read without sub_lock, which a whole replay holds),
// the completion queue and the submissions held back.
void vk_sync_sizes(PerfSizes *s) {