mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-18 17:29:57 -04:00
Log a game thread that waits 10 s for a lock or for another thread to end, with what the other thread is doing.
A game that stops with nothing in its log leaves no clue to which threads wait for which. A thread of its own now
looks every 2 s for guest threads blocked that long on a lock (ArbitrateLock) or on another thread's end
(WaitSynchronization on a thread handle) and logs each such wait once: the waiter, the lock's holder or the thread
waited for, and whether that one is itself waiting and on what. Waits on condition variables, events and addresses
are left out, since idle worker threads wait on them for good. At most 24 lines a run. NX2NAVI_WAIT_WATCH_S sets the
threshold (10; 0 off). The wait's start is now kept for every blocking wait, not only under the sched trace.
Checked: the runtime compiles and tests/sync_wake_check.py passes. A 90 s run with WAIT_WATCH_S=1 logged one chain
during an area load ("thread 0x102d has waited 3 s for the lock ... held by thread 0x105a, itself waiting 2.6 s"),
and nothing in steady play; with the default of 10 s, earlier runs log nothing.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -755,6 +755,10 @@
|
||||
# Extra lines in the log, comma separated: svc, host, ipc, fs, gfx, perf, sched, audio, present, coop or all. The menu's
|
||||
# Performance log sets perf (frame timings). Default: none.
|
||||
# trace = perf
|
||||
# A game thread that waits this many seconds for a lock, or for another thread to end, is logged once with the thread
|
||||
# it waits for and what that one is doing (at most 24 lines a run): a game that stops then leaves the chain in its log.
|
||||
# 0: off. Default: 10.
|
||||
# wait_watch_s = 10
|
||||
# 1: with trace = perf, walks of the process's address space count its private and write-combined memory every 600
|
||||
# frames (the "mem: process" line) and where its committed memory lies every 3600 frames (the "commit census" lines).
|
||||
# Each walk holds the address-space lock for 100-240 ms and delays the next frame by as much, so they are off unless
|
||||
|
||||
@@ -72,7 +72,7 @@ static const char *const known[] = {
|
||||
"SWKBD_ANSWER",
|
||||
"SET_AT", "SWKBD_TEST", "TAA", "DBG_TEST", "DEBUG_MENU", "MENU_TEST_DEBUG", "FSR4", "FSR4_FG", "FG4_VERSION", "FG4_SWAPCHAIN", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_INIT_INLINE", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_DISUSE_EVICT", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_SPLIT_FILL", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
|
||||
"VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
|
||||
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
|
||||
"WAIT_WATCH_S", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
|
||||
"ZSTD_CHECK", "ZSTD_NATIVE",
|
||||
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
|
||||
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL",
|
||||
@@ -142,6 +142,7 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
|
||||
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
|
||||
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"},
|
||||
{"WAIT_WATCH_S", "diagnostics"},
|
||||
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"},
|
||||
};
|
||||
|
||||
@@ -310,6 +311,7 @@ static const CfgMeta metas[] = {
|
||||
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
|
||||
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2|3", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
|
||||
{"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame
|
||||
{"WAIT_WATCH_S", "10", "0..3600", RESTART}, // kernel.c wait_watch: a lock or thread-end wait this long is logged
|
||||
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
|
||||
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha
|
||||
{"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only
|
||||
|
||||
@@ -90,6 +90,8 @@ struct RtThread {
|
||||
const uint32_t *whs; // a WaitSynchronization that blocks: its handles, in the waiter's own memory (holding G)
|
||||
int wn;
|
||||
uint32_t waker; // the thread whose signal ended the last wait (0: a host thread or a timeout)
|
||||
uint64_t blk_since; // when the current blocking wait began, 0 while not blocked (holding G; wait_watch)
|
||||
int blk_told; // wait_watch logged the current wait
|
||||
uint64_t wake_at, grant_at; // when a wait was signalled and a core granted, until the thread runs again
|
||||
uint64_t lw_ns, lw_n, lw_max, lg_ns, lg_n, lg_max, sl_n, sl_req, sl_act; // those latencies; SleepThread asked/took
|
||||
uint64_t sp_n, sp_ho, sp_site; // spin wait passes and core handoffs, the last loop head (the thread writes them)
|
||||
@@ -939,11 +941,63 @@ static uint32_t block(RtThread *t, int kind, int64_t ns) {
|
||||
core_put(t);
|
||||
acct(t, st_of(kind));
|
||||
t->nblk++;
|
||||
t->blk_since = kern_now_ns(), t->blk_told = 0;
|
||||
uint32_t r = block_inner(t, kind, ns);
|
||||
t->blk_since = 0;
|
||||
core_take(t);
|
||||
return r;
|
||||
}
|
||||
|
||||
// ---- long waits (a freeze's first clue)
|
||||
// A guest thread that waits NX2NAVI_WAIT_WATCH_S seconds (10; 0 off) for a lock, or for another thread to end, is
|
||||
// logged once per wait with the thread it waits for and what that one is doing: a game that stops with nothing in its
|
||||
// log then names the chain. Waits on condition variables, events and addresses are left out: idle worker threads wait
|
||||
// on them for good.
|
||||
#define WAIT_WATCH_LINES 24
|
||||
|
||||
static void wait_watch_holder(char *out, size_t cap, const RtThread *h) {
|
||||
if (!h) { snprintf(out, cap, "no such thread"); return; }
|
||||
if (h->exited) { snprintf(out, cap, "thread %#x, which has ended", h->handle); return; }
|
||||
if (!h->blk_since) { snprintf(out, cap, "thread %#x (priority %d), not waiting", h->handle, h->prio); return; }
|
||||
static const char *const kinds[] = {"nothing", "a lock", "a condition variable", "an address", "a handle"};
|
||||
int k = h->wkind >= 0 && h->wkind <= W_SYNC ? h->wkind : 0;
|
||||
snprintf(out, cap, "thread %#x (priority %d), itself waiting %.1f s for %s (%#llx)", h->handle, h->prio,
|
||||
(kern_now_ns() - h->blk_since) / 1e9, kinds[k], (unsigned long long)h->wobj);
|
||||
}
|
||||
|
||||
static void wait_watch(void *arg) {
|
||||
(void)arg;
|
||||
uint64_t limit = (uint64_t)atoi(cfg_value("NX2NAVI_WAIT_WATCH_S")) * 1000000000ull;
|
||||
if (!limit) return;
|
||||
int lines = 0;
|
||||
while (lines < WAIT_WATCH_LINES) {
|
||||
plat_wait_ns(2000000000ull);
|
||||
lock();
|
||||
uint64_t now = kern_now_ns();
|
||||
for (RtThread *t = g_all; t && lines < WAIT_WATCH_LINES; t = t->all) {
|
||||
if (!t->blk_since || t->blk_told || now - t->blk_since < limit) continue;
|
||||
char who[160];
|
||||
if (t->wkind == W_LOCK) { // the lock word's owner field is the holder's handle
|
||||
wait_watch_holder(who, sizeof who, h_peek((uint32_t)t->wobj, O_THREAD));
|
||||
rt_log("kernel: thread %#x (priority %d) has waited %.0f s for the lock at %#llx held by %s",
|
||||
t->handle, t->prio, (now - t->blk_since) / 1e9, (unsigned long long)t->waddr, who);
|
||||
} else if (t->wkind == W_SYNC && t->whs) {
|
||||
const RtThread *end = NULL;
|
||||
for (int i = 0; i < t->wn && !end; i++) end = h_peek(t->whs[i], O_THREAD);
|
||||
if (!end) continue;
|
||||
wait_watch_holder(who, sizeof who, end);
|
||||
rt_log("kernel: thread %#x (priority %d) has waited %.0f s for the end of %s", t->handle, t->prio,
|
||||
(now - t->blk_since) / 1e9, who);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
t->blk_told = 1, lines++;
|
||||
}
|
||||
unlock();
|
||||
}
|
||||
rt_log("kernel: %d long waits logged: no more", lines);
|
||||
}
|
||||
|
||||
int kern_core_away(void) {
|
||||
RtThread *t = tcur;
|
||||
if (!t || t->cur_core < 0) return 0;
|
||||
@@ -1482,6 +1536,11 @@ static void guest_main(void *p) {
|
||||
}
|
||||
|
||||
void kern_run_on_guest_main(void (*fn)(Cpu *c)) {
|
||||
static int watching;
|
||||
if (!watching++) {
|
||||
PlatThread *w = plat_thread_new(wait_watch, NULL, "wait watch");
|
||||
if (w) plat_thread_detach(w);
|
||||
}
|
||||
PlatThread *h = plat_thread_new_ex(guest_main, (void *)fn, "guest main", HOST_STACK);
|
||||
if (!h) rt_fatal("kernel: cannot start the guest main thread");
|
||||
plat_thread_join(h);
|
||||
|
||||
Reference in New Issue
Block a user