diff --git a/nx2navi.ini.example b/nx2navi.ini.example index cae47bd6..579f560b 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -845,6 +845,11 @@ # last one it finished. Unset: on AMD GPUs (VK_AMD_buffer_marker). 1: also on NVIDIA GPUs # (VK_NV_device_diagnostic_checkpoints), which NVIDIA says costs a little. 0: off. # pass_crumbs = 1 +# GPU draw marks, for a GPU crash (device lost) that comes back in the same place on an AMD GPU: inside each render pass +# a mark is written wherever the shaders change, and after the crash the log names the shaders of the draws the GPU had +# not finished, not only the pass. It costs a little frame rate, so it is off unless a crash is being tracked down. +# Needs the pass marks above (AMD GPUs). Default: 0. +# draw_crumbs = 1 # A test of how the game ends when the video card stops answering (device lost): the hand-over to the GPU queue with # this number is not made and is taken as one the driver refused with a lost device. The log then holds one report and # one FATAL line, and the game ends with exit code 72. 0: never. Default: 0. diff --git a/runtime/cfg.c b/runtime/cfg.c index ac6d7b94..ac223d57 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -44,7 +44,7 @@ static const char *const known[] = { "BIND_SKIP", "BIND_STATS", "BIND_UP", "BIND_X", "BIND_Y", "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CACHE_DIE", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COMPUTE_KEEP", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", - "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", + "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DRAW_CRUMBS", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "GPU_TIMELINE", "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", @@ -150,7 +150,7 @@ static const struct { const char *key, *section; } sections[] = { {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"}, - {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, + {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"DRAW_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"MESH_FALLBACK_WORKERS", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"MIRROR_CAPTURE_KB", "diagnostics"}, {"WB_SKIP_MB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"WAIT_WATCH_S", "diagnostics"}, @@ -354,6 +354,7 @@ static const CfgMeta metas[] = { {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC1", "1", "0|1", RESTART}, // vk_astc.c bc1_on {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only + {"DRAW_CRUMBS", "0", "0|1", RESTART}, // vk_crumbs.c draw marks: a mark at each change of shaders inside a pass {"STAGING_FAIL_AT", "0", "0..1000000000", RESTART}, // vk_memory.c staging_fail_now: a test of give_back {"VRAM_BALLAST_MB", "0", "0..131072", RESTART}, // vk_memory.c vk_mem_ballast: a test of a smaller card {"DEVICE_LOST_AT", "0", "0..1000000000", RESTART}, // vk_sync.c lost_at_due: a test of the lost device's end diff --git a/runtime/vk.h b/runtime/vk.h index f0b3cbbe..7e20fafc 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -148,12 +148,14 @@ typedef struct { const GfxShader *first[2], *last[2];// vertex and fragment shader of the first and last draw; a dispatch: first[0] uint32_t gp; // NX2NAVI_GPU_PASSES: the timestamp pair of this mark plus one; 0 none uint32_t layer; // NX2NAVI_GPU_PASSES: the first layer of the depth target (a shadow cascade) + uint32_t gap; // bits of VK_GAP_*: what the replay recorded between the mark before and this one } VkPassMark; void vk_gpu_passes_frame(uint64_t frame); // vk_crumbs.c, each present: NX2NAVI_GPU_PASSES, GPU time per pass and dispatch // NX2NAVI_GPU_PASSES also times what a part of a submission spends outside its passes and dispatches (vk_crumbs.c, "the // gap ledger"). vk_gpu_passes_part: vk_sync.c, at the start (end 0) and the end (end 1) of each part's command buffer. // vk_gpu_gap_note: the replay is about to record commands of this kind outside a pass or dispatch; units is their size -// (bytes for buffers, texels for images, 0 unknown). Both do nothing outside the timed frames. Replay thread only. +// (bytes for buffers, texels for images, 0 unknown). Outside the timed frames the first does nothing and the second +// only notes the kind for the next pass mark (VkPassMark.gap, a lost device's report). Replay thread only. enum { VK_GAP_COPY_B2B, VK_GAP_COPY_B2T, VK_GAP_COPY_T2B, VK_GAP_COPY_T2T, VK_GAP_CLEAR_BUF, VK_GAP_CLEAR_TEX, VK_GAP_CLEAR_TARGET, VK_GAP_UPSCALER, VK_GAP_WB_MERGE, VK_GAP_WB_COPY, VK_GAP_TWIN, VK_GAP_POOL_MARKS, @@ -184,6 +186,11 @@ const VkPassMark *vk_pass_mark_record(uint32_t id); // the ring record of mark // the extension they came from), then each with its ring record. vk_pass_marks_report logs it; the probe checks it. void vk_pass_marks_format(const char *source, uint32_t started, uint32_t finished, char *o, size_t size); void vk_pass_marks_report(void); // vk_device.c device_fault_report: the AMD words and the NVIDIA checkpoints, formatted +// Draw marks (vk_crumbs.c, NX2NAVI_DRAW_CRUMBS=1 with VK_AMD_buffer_marker; off by default): a mark where the shaders +// or the pipeline state change inside pass m, before the draw with index draw in that pass, so a lost device's report +// names the shaders of the draws the GPU had not finished. Replay thread only; nothing while vk_draw_marks_on is 0. +extern int vk_draw_marks_on; +void vk_draw_mark(VkCommandBuffer cb, const VkPassMark *m, const GfxShader *vs, const GfxShader *fs, uint32_t draw); // ---- window and present (gfx_present.c). Host threads only. // vk_present_image: blits the crop rectangle {x, y, w, h} of src (NULL, or w/h <= 0: the whole w x h image), scaled diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c index 110355aa..6cd57fbd 100644 --- a/runtime/vk_crumbs.c +++ b/runtime/vk_crumbs.c @@ -26,7 +26,7 @@ static PFN_vkCmdWriteBufferMarkerAMD write_marker; static VkBuffer buf; static VmaAllocation alloc; -static volatile uint32_t *words; // 0, 1: crumbs; 2, 3: pass marks +static volatile uint32_t *words; // 0, 1: crumbs; 2, 3: pass marks; 4, 5: draw marks // Pass marks (below) int vk_pass_marks_on; @@ -41,7 +41,7 @@ static volatile uint8_t pt_timed[VK_PASS_MARKS]; // the slot's mark writes tim static void buffer_init(VkCtx *c) { write_marker = (PFN_vkCmdWriteBufferMarkerAMD)vk_dpa(c->device, "vkCmdWriteBufferMarkerAMD"); - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 16, + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 24, .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, @@ -53,7 +53,7 @@ static void buffer_init(VkCtx *c) { return; } words = info.pMappedData; - words[0] = words[1] = words[2] = words[3] = 0; + words[0] = words[1] = words[2] = words[3] = words[4] = words[5] = 0; rt_log("vk: GPU breadcrumbs on (VK_AMD_buffer_marker)"); } @@ -69,7 +69,7 @@ void vk_crumbs_release(void) { if (pt_pool && c) vkDestroyQueryPool(c->device, pt_pool, NULL); pt_pool = VK_NULL_HANDLE; buf = VK_NULL_HANDLE, alloc = NULL, words = NULL, write_marker = NULL; - vk_pass_marks_on = pm_amd = 0, set_checkpoint = NULL; + vk_pass_marks_on = pm_amd = vk_draw_marks_on = 0, set_checkpoint = NULL; } void vk_crumb(VkCommandBuffer cb, int kind, uint64_t n) { @@ -132,10 +132,44 @@ static void pass_marks_init(VkCtx *c) { else rt_log("vk: pass marks off (%s)", off ? "NX2NAVI_PASS_CRUMBS=0" : "no VK_AMD_buffer_marker; NX2NAVI_PASS_CRUMBS=1 asks for NVIDIA checkpoints"); + vk_draw_marks_on = pm_amd && cfg_bool("NX2NAVI_DRAW_CRUMBS"); + if (vk_draw_marks_on) + rt_log("vk: draw marks on (NX2NAVI_DRAW_CRUMBS): a mark at every change of shaders inside a pass, so a lost " + "device names the shaders of the draws the GPU had not finished"); + else if (cfg_bool("NX2NAVI_DRAW_CRUMBS")) + rt_log("vk: draw marks off: NX2NAVI_DRAW_CRUMBS needs the pass marks of VK_AMD_buffer_marker"); } +// ---- draw marks (NX2NAVI_DRAW_CRUMBS=1, off by default; with VK_AMD_buffer_marker only) +// A pass mark names the pass the GPU was in and the shaders of its first and last draw. The losses reported on RDNA 2 +// cards end inside one scene pass of 30 to 100 draws, and which draw the GPU never finished is not known. With this +// on, the replay writes a mark each time the shaders or the pipeline state change inside a pass (vk_exec_draw.c +// prepare_draw_ex), into words 4 and 5 of the breadcrumb buffer as a pass mark goes into words 2 and 3, and a ring +// keeps the pass, the draw's index in it and the two shaders of each. After a loss word 5 names the last mark before +// which every draw had finished: the draw that hung was recorded after it, with its shaders or those of one of the few +// marks up to the one in word 4. It costs two marker writes per change, a few thousand a frame, so it is for the +// player who can reproduce a loss, not for everyone. +#define VK_DRAW_MARKS 8192 // power of two +typedef struct { uint32_t id, pass, draw; const GfxShader *vs, *fs; } DrawMark; +static DrawMark dring[VK_DRAW_MARKS]; +static uint32_t dseq; // draw marks made; only the replay thread writes it +int vk_draw_marks_on; + +void vk_draw_mark(VkCommandBuffer cb, const VkPassMark *m, const GfxShader *vs, const GfxShader *fs, uint32_t draw) { + if (!vk_draw_marks_on || !cb) return; + uint32_t id = ++dseq; + if (!id) id = ++dseq; // 0 means no mark + dring[id & (VK_DRAW_MARKS - 1)] = (DrawMark){id, m ? m->id : 0, draw, vs, fs}; + rec_pass_mark(cb, id, 2); +} void vk_pass_mark_cmd(VkCommandBuffer cb, uint32_t id, int end) { + if (end == 2) { // a draw mark (vk_draw_mark): the same two points, into its own words + if (!pm_amd) return; + write_marker(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, buf, 16, id); + write_marker(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, buf, 20, id); + return; + } if (pt_pool && pt_timed[id & (VK_PASS_MARKS - 1)] == 1) vkCmdWriteTimestamp(cb, end ? VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, pt_pool, (id & (VK_PASS_MARKS - 1)) * 2 + (end != 0)); @@ -238,7 +272,12 @@ static void gp_gap_close(uint32_t to) { gp_gap_from = UINT32_MAX, gp_gap_kinds = 0; } +// The kinds of commands noted since the last mark ended (bits of VK_GAP_*), for a lost device's report: the next mark +// takes them as the commands recorded before it (VkPassMark.gap). Replay thread only, as the notes are. +static uint32_t gap_now; + void vk_gpu_gap_note(int kind, uint64_t units) { + if (kind >= 0 && kind < VK_GAP_N) gap_now |= 1u << kind; if (!gp_capture || gp_part == UINT32_MAX || kind < 0 || kind >= VK_GAP_N) return; gp_gap_kinds |= 1u << kind; gp_kind_n[kind]++, gp_kind_units[kind] += units; @@ -571,6 +610,7 @@ VkPassMark *vk_pass_mark_begin(VkCommandBuffer cb, int kind, uint64_t fence) { if (!id) id = ++seq; // 0 means no mark VkPassMark *m = &ring[id & (VK_PASS_MARKS - 1)]; m->id = id, m->kind = (uint8_t)kind, m->fence = fence, m->draws = UINT32_MAX; + m->gap = gap_now, gap_now = 0; // what was recorded between the mark before and this one // timed unless the slot's previous mark is still waiting to be read (its queries are not reset yet) if (pt_pool) pt_timed[id & (VK_PASS_MARKS - 1)] = !pt_timed[id & (VK_PASS_MARKS - 1)] ? 1 : 2; rec_pass_mark(cb, id, 0); @@ -584,6 +624,7 @@ void vk_pass_mark_end(VkCommandBuffer cb, const VkPassMark *m) { gp_in_mark = 0; gp_end(cb, m); rec_pass_mark(cb, m->id, 1); + gap_now = 0; // the gap after this mark starts empty } int vk_pass_marks_read(uint32_t *started, uint32_t *finished) { @@ -703,6 +744,67 @@ static void nv_report(void) { log_lines(t); } +// The kinds of bits (VK_GAP_*) as text; "barriers and state only" when none was noted. +static void gap_text(uint32_t kinds, char *o, size_t size) { + size_t n = 0; + o[0] = 0; + for (int b = 0; b < VK_GAP_N && n < size; b++) + if (kinds >> b & 1) n += (size_t)snprintf(o + n, size - n, "%s%s", n ? ", " : "", gp_kind_names[b]); + if (!n) snprintf(o, size, "barriers and state only"); +} + +// A GPU that finished the last mark it started stopped between two marks, where the two lines of +// vk_pass_marks_format name nothing: in a player's log both were one compute dispatch, with 51 later marks made and +// none of them started. This adds the mark that never started and what the replay recorded between the two (the gap +// notes: copies, clears, the upscaler, write-backs), or what it recorded after the last mark of all. +static void report_between(uint32_t started, uint32_t finished) { + if (!finished || started != finished) return; + char what[512], next[640]; + const VkPassMark *m = vk_pass_mark_record(finished + 1); + if (m) { + describe_mark(finished + 1, next, sizeof next); + gap_text(m->gap, what, sizeof what); + rt_log("vk: the GPU stopped after mark %u and before the next one, which it never started: %s", (unsigned)finished, + next); + rt_log("vk: recorded between the two: %s", what); + } else if (finished == seq) { + gap_text(gap_now, what, sizeof what); + rt_log("vk: the GPU stopped after mark %u, the last one made; recorded since: %s", (unsigned)finished, what); + } else { + rt_log("vk: the GPU stopped after mark %u; the record of the mark after it is gone (a later mark took its slot)", + (unsigned)finished); + } +} + +static void describe_draw_mark(uint32_t id, char *o, size_t size) { + const DrawMark *d = &dring[id & (VK_DRAW_MARKS - 1)]; + if (!id) snprintf(o, size, "nothing yet"); + else if (d->id != id) snprintf(o, size, "no record (a later mark took its slot)"); + else + snprintf(o, size, "before draw %u of pass mark %u: vs %016llx fs %016llx", d->draw, d->pass, shader_id(d->vs), + shader_id(d->fs)); +} + +// Words 4 and 5 (vk_draw_mark): the draws the GPU had not finished were recorded after the mark in word 5, so its +// shaders and those of the marks up to the one in word 4 are the candidates; up to DRAW_MARK_LINES of them are listed. +#define DRAW_MARK_LINES 12 +static void draw_marks_report(void) { + if (!vk_draw_marks_on || !words) return; + uint32_t reached = words[4], finished = words[5]; + char t[256]; + rt_log("vk: draw marks (NX2NAVI_DRAW_CRUMBS, %u made): the GPU last reached draw mark %u, and every draw before draw " + "mark %u had finished; the draws it had not finished follow these marks:", (unsigned)dseq, (unsigned)reached, + (unsigned)finished); + uint32_t n = reached >= finished ? reached - finished + 1 : 1; + if (n > DRAW_MARK_LINES) n = DRAW_MARK_LINES; + for (uint32_t i = 0; i < n; i++) { + describe_draw_mark(finished + i, t, sizeof t); + rt_log("vk: draw mark %u: %s", (unsigned)(finished + i), t); + } + if (reached >= finished && reached - finished + 1 > n) + rt_log("vk: and %u more up to draw mark %u", (unsigned)(reached - finished + 1 - n), (unsigned)reached); +} + void vk_pass_marks_report(void) { if (!vk_pass_marks_on) return; uint32_t started, finished; @@ -710,6 +812,8 @@ void vk_pass_marks_report(void) { if (vk_pass_marks_read(&started, &finished)) { vk_pass_marks_format("VK_AMD_buffer_marker", started, finished, t, sizeof t); log_lines(t); + report_between(started, finished); + draw_marks_report(); } nv_report(); } diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 1c743a08..3caadc02 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -4588,6 +4588,8 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int ex->recorded++; cx->B.st = s, cx->B.st_valid = 1; if (ex->mark && vk_exec_frame.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; + if (vk_draw_marks_on) // NX2NAVI_DRAW_CRUMBS: a lost device then names these shaders (vk_crumbs.c vk_draw_mark) + vk_draw_mark(ex->cb, ex->mark, vs, fs, (uint32_t)(vk_exec_frame.draws - ex->mark_draws0)); } vk_pipeline_line_width(ex->cb, &s, ex->line_width); // a draw of lines: the game's width; else nothing if (!mesh) {