mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-18 00:29:58 -04:00
End a run whose GPU device is lost in one place: one report, no callbacks of unfinished submissions, and exit code 72.
A lost device used to end the process from whichever submit or wait noticed it first, while the present only logged it and went on and the other threads kept calling the driver. The completion thread could take the submissions' timeline as reached and run the callbacks of submissions that never finished. vk_device_lost_check now takes the result of every submit, present, acquire and wait. The first thread to see the loss sets a flag, writes the report and ends the process as a fatal error does (the log written, a save commit under way finished), with an exit code of its own; later threads stop in that call, and a watchdog ends the process after ten seconds should the first thread hang. The replay, its submit, the uploads, the present and the lazy fetch read the flag and return without touching the device. The completion thread asks the device with a call of its own before a new value counts and runs no callback once the flag is set. The executable's --probe-devlost and tests/device_lost_check.py check this without a lost device, and device_lost_at in nx2navi.ini ends a real run the same way. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,10 @@ static uint32_t g_gfx_queues = 1; // queues the graphics family offers (pick_d
|
||||
static VkCtx g_vk;
|
||||
static int g_ready;
|
||||
static PlatMutex g_lock = PLAT_MUTEX_INIT;
|
||||
// The fence vk_device_lost_poll asks about. Nothing submits with it, so its status is "not ready" for as long as the
|
||||
// device lives.
|
||||
static VkFence lost_fence;
|
||||
static PlatMutex lost_fence_lock = PLAT_MUTEX_INIT;
|
||||
|
||||
static VKAPI_ATTR VkBool32 VKAPI_CALL debug_cb(VkDebugUtilsMessageSeverityFlagBitsEXT sev, VkDebugUtilsMessageTypeFlagsEXT type,
|
||||
const VkDebugUtilsMessengerCallbackDataEXT *d, void *user) {
|
||||
@@ -550,6 +554,8 @@ static void destroy_all(void) {
|
||||
if (g_vk.device) vk_astc_release();
|
||||
if (g_vk.device) vk_crumbs_release();
|
||||
if (g_vk.timeline) vkDestroySemaphore(g_vk.device, g_vk.timeline, NULL);
|
||||
if (lost_fence) vkDestroyFence(g_vk.device, lost_fence, NULL); // vk_device_lost_poll
|
||||
lost_fence = VK_NULL_HANDLE;
|
||||
vk_headroom_stop();
|
||||
if (g_vk.vma) vmaDestroyAllocator(g_vk.vma);
|
||||
if (g_vk.device) vkDestroyDevice(g_vk.device, NULL);
|
||||
@@ -599,15 +605,9 @@ static const char *fault_address_type(VkDeviceFaultAddressTypeEXT t) {
|
||||
}
|
||||
}
|
||||
|
||||
// The query runs once per process. Threads that hit the loss at the same time wait on the lock until it is logged, so
|
||||
// none of them reaches its rt_fatal and ends the process before the report is written.
|
||||
void vk_device_fault_report(VkResult r) {
|
||||
static PlatMutex lock = PLAT_MUTEX_INIT;
|
||||
static int done;
|
||||
if (r != VK_ERROR_DEVICE_LOST) return;
|
||||
plat_mutex_lock(&lock);
|
||||
if (done) { plat_mutex_unlock(&lock); return; }
|
||||
done = 1;
|
||||
// The report of a lost device: the breadcrumbs, the pass marks, the video memory and the driver's fault information.
|
||||
// Once per process, on the thread that saw the loss first (vk_device_lost_check).
|
||||
static void device_fault_report(void) {
|
||||
vk_crumbs_report();
|
||||
vk_pass_marks_report();
|
||||
vk_mem_lost_report();
|
||||
@@ -619,7 +619,6 @@ void vk_device_fault_report(VkResult r) {
|
||||
rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point"
|
||||
: "not enabled");
|
||||
vk_vaddr_dump(); // the history still names what held a page another report (the kernel's log) gives
|
||||
plat_mutex_unlock(&lock);
|
||||
return;
|
||||
}
|
||||
VkDeviceFaultCountsEXT n = {.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT};
|
||||
@@ -648,7 +647,53 @@ void vk_device_fault_report(VkResult r) {
|
||||
vk_vaddr_dump();
|
||||
free(ai);
|
||||
free(vi);
|
||||
plat_mutex_unlock(&lock);
|
||||
}
|
||||
|
||||
// ---- the end of a run whose device is lost (vk.h vk_device_lost_check)
|
||||
// A lost device stays lost: every later call on it fails or, worse, answers as if it had worked, and a driver that is
|
||||
// called on from several threads after the loss can fault inside itself. So the first thread that sees the loss ends
|
||||
// the run, and nothing else touches the device meanwhile: the paths that would read vk_lost and return, and a thread
|
||||
// that gets the loss as a result of its own stops here for good.
|
||||
// LOST_EXIT_MS bounds the end. The first thread writes the report and waits for a save commit (fs_host.c
|
||||
// fs_save_crash_wait, 2 s at most); should it hang on a lock that a stopped thread holds, the watchdog ends the
|
||||
// process with the same code, and the lines already written are in the log (rt_fatal_code wrote them itself).
|
||||
#define LOST_EXIT_MS 10000
|
||||
volatile int32_t vk_lost;
|
||||
|
||||
static void lost_wait_exit(void *arg) {
|
||||
(void)arg;
|
||||
for (int t = 0; t < LOST_EXIT_MS; t += 50) plat_sleep_ms(50);
|
||||
_Exit(RT_LOST_EXIT);
|
||||
}
|
||||
|
||||
void vk_device_lost_check(VkResult r, const char *where) {
|
||||
if (r != VK_ERROR_DEVICE_LOST) return;
|
||||
if (plat_xchg32(&vk_lost, 1)) lost_wait_exit(NULL); // another thread is ending the run: this one stops here
|
||||
plat_thread_spawn(lost_wait_exit, NULL, "vk lost exit", 0);
|
||||
device_fault_report();
|
||||
rt_fatal_code(RT_LOST_EXIT, "vk: the GPU device was lost (seen at: %s). The run ends here; the lines above name what "
|
||||
"the GPU was doing", where);
|
||||
}
|
||||
|
||||
|
||||
void vk_device_lost_poll(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!c || vk_device_is_lost()) return;
|
||||
VkFence f = __atomic_load_n(&lost_fence, __ATOMIC_ACQUIRE);
|
||||
if (!f) {
|
||||
plat_mutex_lock(&lost_fence_lock);
|
||||
if (!lost_fence) {
|
||||
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
||||
VkFence n = VK_NULL_HANDLE;
|
||||
VkResult r = vkCreateFence(c->device, &fi, NULL, &n);
|
||||
if (r == VK_SUCCESS) __atomic_store_n(&lost_fence, n, __ATOMIC_RELEASE);
|
||||
else vk_device_lost_check(r, "the fence that asks whether the device lives");
|
||||
}
|
||||
f = lost_fence;
|
||||
plat_mutex_unlock(&lost_fence_lock);
|
||||
if (!f) return;
|
||||
}
|
||||
vk_device_lost_check(vkGetFenceStatus(c->device, f), "a poll of the device");
|
||||
}
|
||||
|
||||
int vk_selftest(void) {
|
||||
|
||||
Reference in New Issue
Block a user