End a run whose GPU device is lost in one place: one report, no callbacks of unfinished submissions, and exit code 72.

A lost device used to end the process from whichever submit or wait noticed it first, while the present only logged it
and went on and the other threads kept calling the driver. The completion thread could take the submissions' timeline
as reached and run the callbacks of submissions that never finished.

vk_device_lost_check now takes the result of every submit, present, acquire and wait. The first thread to see the loss
sets a flag, writes the report and ends the process as a fatal error does (the log written, a save commit under way
finished), with an exit code of its own; later threads stop in that call, and a watchdog ends the process after ten
seconds should the first thread hang. The replay, its submit, the uploads, the present and the lazy fetch read the flag
and return without touching the device. The completion thread asks the device with a call of its own before a new
value counts and runs no callback once the flag is set.

The executable's --probe-devlost and tests/device_lost_check.py check this without a lost device, and
device_lost_at in nx2navi.ini ends a real run the same way.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 04:35:30 -05:00
parent 84b0672ec2
commit 534591be04
16 changed files with 319 additions and 41 deletions

View File

@@ -2932,6 +2932,7 @@ static void mbarrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags
// memory.c's fetch (frz_lock held there, so one at a time): [off, off + size) of the mirror h0 into the imported
// buffer h1, after all the GPU work handed over so far.
static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) {
if (vk_device_is_lost()) return; // the run is ending (vk_device.c vk_device_lost_check): the bytes stay as they are
VkCtx *c = vk_ctx();
VkBufferCopy reg = {off, off, size};
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
@@ -2952,7 +2953,7 @@ static void lazy_fetch(uint64_t h0, uint64_t h1, uint64_t off, uint64_t size) {
vk_queue_unlock();
if (r == VK_SUCCESS) r = vkWaitForFences(c->device, 1, &lz_fence, VK_TRUE, 10000000000ull);
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the submit or the wait that brings GPU writes down to guest memory");
rt_fatal("vk: bringing %llu bytes the GPU wrote down to guest memory failed (%d)", (unsigned long long)size, (int)r);
}
vkResetFences(c->device, 1, &lz_fence);
@@ -4109,6 +4110,7 @@ void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) {
}
uint64_t vk_upload_flush(int wait) {
if (vk_device_is_lost()) return 0; // the run is ending (vk_device.c vk_device_lost_check): no submit, no wait
uint64_t f0 = kern_now_ns();
plat_once(&cmd_once, cmd_init, NULL);
plat_rmutex_lock(&cmd_lock);
@@ -4125,7 +4127,7 @@ uint64_t vk_upload_flush(int wait) {
VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence);
vk_queue_unlock();
if (r != VK_SUCCESS) {
vk_device_fault_report(r);
vk_device_lost_check(r, "the submit of an upload batch");
rt_fatal("vk: upload submit failed (%d)", (int)r);
}
if (ninflight == cap_inflight) grow((void **)&inflight, &cap_inflight, sizeof *inflight);
@@ -4138,7 +4140,7 @@ uint64_t vk_upload_flush(int wait) {
// loop would spin for good, holding the upload lock, instead of the run ending with the device's report
VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX);
if (w != VK_SUCCESS) {
vk_device_fault_report(w);
vk_device_lost_check(w, "the wait for an upload batch");
rt_fatal("vk: the wait for an upload failed (%d)", (int)w);
}
retire(c, 0);