diff --git a/docs/building.md b/docs/building.md index da13975a..0d7f0c89 100644 --- a/docs/building.md +++ b/docs/building.md @@ -278,6 +278,13 @@ Before a commit: `git config core.hooksPath .githooks` runs it before each local commit. - `build\\nx2navi.exe --selftest`: layout, dispatch table and host-function registry, without game files. +GPU page faults: `NX2NAVI_VA_REPORT=1` keeps the driver's GPU address bind/unbind reports +(VK_EXT_device_address_binding_report, `runtime/vk_vaddr.c`); `=2` also keeps the host call stack of ranges of 1 MiB +or more. After a lost device, the log names the reports whose range covered each faulting page, or the nearest ranges +when none did, and `vaddr-history.txt` beside the log holds the whole history. Stack frames are module offsets: add +the executable's load address (the link map's base) before looking them up in `ocarina.map`. Each bind and unbind +costs a callback, and `=2`'s stack walks enough to hide a timing-dependent fault. + A change to the in-game menu's look (`runtime/ui/menu_skin.cpp`, `option_rows.cpp`, `perf_panel.cpp`) can be looked at without game files: `python tools/menu_preview.py` builds the menu's own drawing code with the host's clang++ and writes the Screen tab, the full performance overlay and the boot screen to `build/menu-preview/` (`menu.png`, diff --git a/nx2navi.ini.example b/nx2navi.ini.example index eb75b735..87c50f13 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -621,6 +621,12 @@ # 1: samples each new ASTC texture's blocks on the CPU and logs how many are opaque ("ASTC alpha probe" lines), which # could be stored at half the video memory. Slows area loads. Default: 0. # astc_bc_probe = 0 +# GPU address history for the report of a lost graphics device, on drivers with +# VK_EXT_device_address_binding_report. 1: the game keeps every address range the driver binds to a buffer or image and +# every range it unbinds; after a lost device the log names what held each faulting address, and vaddr-history.txt +# beside the log holds the whole history. 2: also the call stack that made each range of 1 MiB or more, which slows +# area loads enough to hide a fault that depends on timing. Default: 0. +# va_report = 0 # Flight recorder: keeps the last frames and saves them to a flight folder next to the folder of nx2navi.exe when a frame # takes longer than flight_spike_ms (0 to 60000). # flight = 0 diff --git a/runtime/cfg.c b/runtime/cfg.c index 2605166f..e4fcaf27 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -68,7 +68,7 @@ static const char *const known[] = { "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", - "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", + "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VA_REPORT", "VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC", }; @@ -125,7 +125,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_NAN_PRESERVE", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, - {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, + {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; @@ -268,6 +268,7 @@ static const CfgMeta metas[] = { {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only + {"VA_REPORT", "0", "0|1|2", RESTART}, // vk_vaddr.c GPU address history for a lost device's report; 2: call stacks {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index 66b7cb2d..ccb6eac0 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -1567,6 +1567,22 @@ int plat_module_of(uintptr_t addr, char *name, size_t cap, uintptr_t *base) { return snprintf(name, cap, "%s", n ? n + 1 : d.dli_fname) < (int)cap; } +typedef struct { uintptr_t *pc; int n, max; } StackWalk; + +static _Unwind_Reason_Code stack_step(struct _Unwind_Context *ctx, void *p) { + StackWalk *w = p; + uintptr_t ip = _Unwind_GetIP(ctx); + if (!ip || w->n == w->max) return _URC_END_OF_STACK; + w->pc[w->n++] = ip; + return _URC_NO_REASON; +} + +int plat_stack(uintptr_t *pc, int max) { + StackWalk w = {pc, 0, max}; + _Unwind_Backtrace(stack_step, &w); + return w.n; +} + int plat_exe_path(char *out, size_t cap) { ssize_t n = readlink("/proc/self/exe", out, cap - 1); if (n <= 0 || (size_t)n >= cap - 1) return 0; diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index df8bad8b..81c7f741 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -416,6 +416,8 @@ uintptr_t plat_exe_base(void); // where the executabl // The module (executable or library) holding code address addr: its file name without directories into name, its load // address into *base. 0 when none. int plat_module_of(uintptr_t addr, char *name, size_t cap, uintptr_t *base); +// Return addresses of the calling thread's stack into pc, innermost first, at most max: their count. +int plat_stack(uintptr_t *pc, int max); #define PLAT_PATH_MAX 1024 // path buffers of the runtime #ifdef _WIN32 #define PLAT_EXE_SUFFIX ".exe" // of the programs the runtime starts (nx2navi_xlate) diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index 7cbfa47c..9f8138d1 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -1562,6 +1562,13 @@ int plat_module_of(uintptr_t addr, char *name, size_t cap, uintptr_t *base) { return WideCharToMultiByte(CP_UTF8, 0, n, -1, name, (int)cap, NULL, NULL) != 0; } +int plat_stack(uintptr_t *pc, int max) { + void *f[62]; + USHORT n = RtlCaptureStackBackTrace(1, max < 62 ? (DWORD)max : 62, f, NULL); + for (USHORT i = 0; i < n; i++) pc[i] = (uintptr_t)f[i]; + return n; +} + int plat_exe_path(char *out, size_t cap) { wchar_t w[WPATH]; DWORD n = GetModuleFileNameW(NULL, w, WPATH); diff --git a/runtime/vk.h b/runtime/vk.h index 8b3ca0db..55147b61 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -82,6 +82,15 @@ int vk_selftest(void); // 0 on success // process, after the GPU breadcrumbs (vk_crumbs_report). Any other result returns at once. void vk_device_fault_report(VkResult r); +// ---- GPU address history (vk_vaddr.c, NX2NAVI_VA_REPORT=1, 2 with call stacks): VK_EXT_device_address_binding_report's +// reports, kept so vk_device_fault_report can name the objects that held a faulting address. +int vk_vaddr_wanted(void); +void vk_vaddr_start(VkInstance instance); // create_instance, after vkCreateInstance +void vk_vaddr_device(int supported); // create_device: the driver reports address bindings +void vk_vaddr_fault(uint64_t addr, uint64_t precision); // logs the reports whose range covered addr's page +void vk_vaddr_dump(void); // the whole history to vaddr-history.txt beside the log +void vk_vaddr_stop(VkInstance instance); // before vkDestroyInstance + // ---- GPU breadcrumbs (vk_crumbs.c): with VK_AMD_buffer_marker, each crumb has the GPU write its code into host memory // twice, when the command processor reaches it and when all work before it has finished. After a lost device the two // codes bracket the work that hung. Without the extension every call returns at once. diff --git a/runtime/vk_device.c b/runtime/vk_device.c index c72af9a8..485fcaef 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -141,8 +141,9 @@ static int create_instance(void) { rt_log("vk: NX2NAVI_VK_VALIDATION set but %s is not installed; continuing without", layers[0]); validate = 0; } - // DLSS Frame Generation: Streamline's plugins call its naming functions even when they say it is off - if (validate || (vk_dlssg_wanted() && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME))) + // DLSS Frame Generation: Streamline's plugins call its naming functions even when they say it is off; the GPU + // address history (vk_vaddr.c) gets the driver's binding reports through a messenger + if (validate || ((vk_dlssg_wanted() || vk_vaddr_wanted()) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME))) ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME; // armed, not wanted: the menu switches HDR on during the run (vk_hdr.c), and the extensions are fixed here if (vk_hdr_armed() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats @@ -168,6 +169,7 @@ static int create_instance(void) { .pfnUserCallback = debug_cb}; if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger); } + vk_vaddr_start(g_vk.instance); return 1; } @@ -299,6 +301,7 @@ typedef struct { VkPhysicalDeviceShaderMaximalReconvergenceFeaturesKHR mrc; VkPhysicalDeviceMaintenance5FeaturesKHR m5; VkPhysicalDevicePipelineBinaryFeaturesKHR pbin; + VkPhysicalDeviceAddressBindingReportFeaturesEXT abr; } FeatureChain; static int create_device(void) { @@ -362,6 +365,12 @@ static int create_device(void) { CHAIN(fbl, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_FEATURES_EXT); CHAIN(lr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_LOCAL_READ_FEATURES_KHR); } + // The GPU address history for a lost device's report (vk_vaddr.c, NX2NAVI_VA_REPORT=1) + int want_va = vk_vaddr_wanted() && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME); + if (want_va) { + ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME; + CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT); + } vk_dlaa_device_exts(ep, ne, ext, &n_ext, 40); // NGX's; the eight slots after it stay for the eight below // FSR 3.1 (vk_ffx.cpp): its Vulkan back end looks vkGetBufferMemoryRequirements2KHR up by that name, which resolves // only with the extension enabled (the core function has no suffix); without it the back end called address 0. @@ -431,6 +440,7 @@ static int create_device(void) { else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } if (!c->fault.deviceFault) g_vk.has_device_fault = 0; + if (vk_vaddr_wanted()) vk_vaddr_device(want_va && c->abr.reportAddressBinding); if (g_vk.has_vrs) { // the rate is set per draw; no rate attachment and no per-primitive rates c->vrs.attachmentFragmentShadingRate = c->vrs.primitiveFragmentShadingRate = VK_FALSE; if (!c->vrs.pipelineFragmentShadingRate) g_vk.has_vrs = 0; @@ -537,6 +547,7 @@ static void destroy_all(void) { (PFN_vkDestroyDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkDestroyDebugUtilsMessengerEXT"); if (f) f(g_vk.instance, g_vk.messenger, NULL); } + vk_vaddr_stop(g_vk.instance); if (g_vk.instance) vkDestroyInstance(g_vk.instance, NULL); g_errors_total += g_vk.validation_errors; memset(&g_vk, 0, sizeof g_vk); @@ -596,6 +607,7 @@ void vk_device_fault_report(VkResult r) { if (!get) { rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point" : "not enabled"); + vk_vaddr_dump(); // the history still names what held a page another report (the kernel's log) gives plat_mutex_unlock(&lock); return; } @@ -613,13 +625,16 @@ void vk_device_fault_report(VkResult r) { rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description, n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary); // The faulting access lies in the precision-aligned range around the reported address. - for (uint32_t i = 0; i < n.addressInfoCount; i++) + for (uint32_t i = 0; i < n.addressInfoCount; i++) { rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType), (unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision); + vk_vaddr_fault(ai[i].reportedAddress, ai[i].addressPrecision); + } for (uint32_t i = 0; i < n.vendorInfoCount; i++) rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description, (unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData); } + vk_vaddr_dump(); free(ai); free(vi); plat_mutex_unlock(&lock); diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c index 481e8eed..811d7cfd 100644 --- a/runtime/vk_headroom.c +++ b/runtime/vk_headroom.c @@ -157,6 +157,7 @@ static void top_up(const VkCtx *c, uint32_t type) { static void headroom_main(void *arg) { (void)arg; __atomic_store_n(&H.tid, plat_thread_id(), __ATOMIC_RELEASE); + if (vk_vaddr_wanted()) rt_log("vk: headroom: thread %u", H.tid); // names this thread in the address history (vk_vaddr.c) plat_thread_priority(PLAT_PRIO_BELOW_NORMAL); uint64_t watch = 0; // memory types seen so far for (;;) {