diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 685c2015..f2538825 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -755,6 +755,10 @@ # Extra lines in the log, comma separated: svc, host, ipc, fs, gfx, perf, sched, audio, present, coop or all. The menu's # Performance log sets perf (frame timings). Default: none. # trace = perf +# A game thread that waits this many seconds for a lock, or for another thread to end, is logged once with the thread +# it waits for and what that one is doing (at most 24 lines a run): a game that stops then leaves the chain in its log. +# 0: off. Default: 10. +# wait_watch_s = 10 # 1: with trace = perf, walks of the process's address space count its private and write-combined memory every 600 # frames (the "mem: process" line) and where its committed memory lies every 3600 frames (the "commit census" lines). # Each walk holds the address-space lock for 100-240 ms and delays the next frame by as much, so they are off unless diff --git a/runtime/cfg.c b/runtime/cfg.c index 4839610b..b53b8808 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -72,7 +72,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "DBG_TEST", "DEBUG_MENU", "MENU_TEST_DEBUG", "FSR4", "FSR4_FG", "FG4_VERSION", "FG4_SWAPCHAIN", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_INIT_INLINE", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_DISUSE_EVICT", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_SPLIT_FILL", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "WAIT_WATCH_S", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", @@ -142,6 +142,7 @@ static const struct { const char *key, *section; } sections[] = { {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, + {"WAIT_WATCH_S", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, }; @@ -310,6 +311,7 @@ static const CfgMeta metas[] = { {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2|3", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame + {"WAIT_WATCH_S", "10", "0..3600", RESTART}, // kernel.c wait_watch: a lock or thread-end wait this long is logged {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only diff --git a/runtime/kernel.c b/runtime/kernel.c index 312ed790..fb52b694 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -90,6 +90,8 @@ struct RtThread { const uint32_t *whs; // a WaitSynchronization that blocks: its handles, in the waiter's own memory (holding G) int wn; uint32_t waker; // the thread whose signal ended the last wait (0: a host thread or a timeout) + uint64_t blk_since; // when the current blocking wait began, 0 while not blocked (holding G; wait_watch) + int blk_told; // wait_watch logged the current wait uint64_t wake_at, grant_at; // when a wait was signalled and a core granted, until the thread runs again uint64_t lw_ns, lw_n, lw_max, lg_ns, lg_n, lg_max, sl_n, sl_req, sl_act; // those latencies; SleepThread asked/took uint64_t sp_n, sp_ho, sp_site; // spin wait passes and core handoffs, the last loop head (the thread writes them) @@ -939,11 +941,63 @@ static uint32_t block(RtThread *t, int kind, int64_t ns) { core_put(t); acct(t, st_of(kind)); t->nblk++; + t->blk_since = kern_now_ns(), t->blk_told = 0; uint32_t r = block_inner(t, kind, ns); + t->blk_since = 0; core_take(t); return r; } +// ---- long waits (a freeze's first clue) +// A guest thread that waits NX2NAVI_WAIT_WATCH_S seconds (10; 0 off) for a lock, or for another thread to end, is +// logged once per wait with the thread it waits for and what that one is doing: a game that stops with nothing in its +// log then names the chain. Waits on condition variables, events and addresses are left out: idle worker threads wait +// on them for good. +#define WAIT_WATCH_LINES 24 + +static void wait_watch_holder(char *out, size_t cap, const RtThread *h) { + if (!h) { snprintf(out, cap, "no such thread"); return; } + if (h->exited) { snprintf(out, cap, "thread %#x, which has ended", h->handle); return; } + if (!h->blk_since) { snprintf(out, cap, "thread %#x (priority %d), not waiting", h->handle, h->prio); return; } + static const char *const kinds[] = {"nothing", "a lock", "a condition variable", "an address", "a handle"}; + int k = h->wkind >= 0 && h->wkind <= W_SYNC ? h->wkind : 0; + snprintf(out, cap, "thread %#x (priority %d), itself waiting %.1f s for %s (%#llx)", h->handle, h->prio, + (kern_now_ns() - h->blk_since) / 1e9, kinds[k], (unsigned long long)h->wobj); +} + +static void wait_watch(void *arg) { + (void)arg; + uint64_t limit = (uint64_t)atoi(cfg_value("NX2NAVI_WAIT_WATCH_S")) * 1000000000ull; + if (!limit) return; + int lines = 0; + while (lines < WAIT_WATCH_LINES) { + plat_wait_ns(2000000000ull); + lock(); + uint64_t now = kern_now_ns(); + for (RtThread *t = g_all; t && lines < WAIT_WATCH_LINES; t = t->all) { + if (!t->blk_since || t->blk_told || now - t->blk_since < limit) continue; + char who[160]; + if (t->wkind == W_LOCK) { // the lock word's owner field is the holder's handle + wait_watch_holder(who, sizeof who, h_peek((uint32_t)t->wobj, O_THREAD)); + rt_log("kernel: thread %#x (priority %d) has waited %.0f s for the lock at %#llx held by %s", + t->handle, t->prio, (now - t->blk_since) / 1e9, (unsigned long long)t->waddr, who); + } else if (t->wkind == W_SYNC && t->whs) { + const RtThread *end = NULL; + for (int i = 0; i < t->wn && !end; i++) end = h_peek(t->whs[i], O_THREAD); + if (!end) continue; + wait_watch_holder(who, sizeof who, end); + rt_log("kernel: thread %#x (priority %d) has waited %.0f s for the end of %s", t->handle, t->prio, + (now - t->blk_since) / 1e9, who); + } else { + continue; + } + t->blk_told = 1, lines++; + } + unlock(); + } + rt_log("kernel: %d long waits logged: no more", lines); +} + int kern_core_away(void) { RtThread *t = tcur; if (!t || t->cur_core < 0) return 0; @@ -1482,6 +1536,11 @@ static void guest_main(void *p) { } void kern_run_on_guest_main(void (*fn)(Cpu *c)) { + static int watching; + if (!watching++) { + PlatThread *w = plat_thread_new(wait_watch, NULL, "wait watch"); + if (w) plat_thread_detach(w); + } PlatThread *h = plat_thread_new_ex(guest_main, (void *)fn, "guest main", HOST_STACK); if (!h) rt_fatal("kernel: cannot start the guest main thread"); plat_thread_join(h);