diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 87ccfb53..53189bc8 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -473,3 +473,7 @@ # gets a "texup:" line in the log that splits it: creation, staging, storage copy, ASTC recording, scratch buffers and # new device memory (up to 400 lines; 0: none; 16 catches the area-change hitches). # texup_log_ms = 0 +# 1: the frame-time summary also lists the locks the replay waited for, by the code that asked (addresses; the line's +# "ref" is perf_lock_timing_here, which the link map names), the recorder's waits, and the texture lock's holds over +# 1 ms. Off by default. +# lock_time = 0 diff --git a/runtime/cfg.c b/runtime/cfg.c index 2c99b1cf..3befaa9a 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -51,7 +51,7 @@ static const char *const known[] = { "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC", - "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", + "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", @@ -66,7 +66,7 @@ static const char *const known[] = { "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", - "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", + "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", }; // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). @@ -190,7 +190,7 @@ static const CfgMeta metas[] = { {"DYN_RES", "0", "0|1", LIVE}, {"DYN_RES_TARGET", "16.7", "1..100", LIVE}, {"DYN_RES_STEP", "45", "8..600", LIVE}, // the recording thread: a core of its own (auto: with 12 or more logical processors) and the highest priority // (vk_exec.c vk_exec_record) - {"RECORDER_CORE", "auto", "auto|0|1", RESTART}, {"RECORDER_PRIO", "1", "0|1", RESTART}, {"REPLAY_TIME", "0", "0|1", RESTART}, + {"RECORDER_CORE", "auto", "auto|0|1", RESTART}, {"RECORDER_PRIO", "1", "0|1", RESTART}, {"REPLAY_TIME", "0", "0|1", RESTART}, {"LOCK_TIME", "0", "0|1", RESTART}, // held submissions handed to the queue together (vk_sync.c, submit batching) {"SUBMIT_BATCH", "4", "1..16", RESTART}, // shaders and pipelines diff --git a/runtime/host.h b/runtime/host.h index 0e8fe3a6..fb6934a5 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -52,6 +52,12 @@ void perf_count(int kind); // and resource binds between them; xfer is copies and clears. enum { PERF_RP_DRAW, PERF_RP_BIND, PERF_RP_XFER, PERF_RP_OTHER, PERF_RP_N }; void perf_replay_op(int cat, uint64_t ns); +void perf_replay_op_at(int cat, uint32_t op, uint64_t ns); +void perf_replay_cpu(uint64_t ns); // the replaying thread's CPU time over one submission's replay (vk_sync.c) // as perf_replay_op, also kept per frame for slow frames +void perf_lock_timing_here(void); +int perf_lock_holds_on(void); // NX2NAVI_LOCK_TIME +void perf_lock_hold(void *site, uint64_t ns); // a lock held ns from site (vk_texture.c, holds over 1 ms) // NX2NAVI_LOCK_TIME: this thread's lock waits go into the frame-time summary +extern void (*perf_rec_waits)(uint64_t *full_ns, uint64_t *drain_ns); // set by the back end (vk_rec.c totals) void perf_present(uint64_t frame); void perf_summary_flush(void); // prints the frame-time summary of the running period (exit); the 60 s lines come from perf_present void gfx_exit_log(void); // back end's lines for the exit summary (weak no-op in prof_win.c) diff --git a/runtime/perf.c b/runtime/perf.c index 6774e60e..b6c1d3c1 100644 --- a/runtime/perf.c +++ b/runtime/perf.c @@ -9,6 +9,7 @@ #include "flight.h" #include "gfx.h" #include "host.h" +#include "nvn_api.h" #include "perfmon.h" #include "prof.h" #include "platform/platform.h" @@ -95,6 +96,12 @@ void perf_gpu_wait(uint64_t t0) { static volatile int64_t replay_ns; // perf_replay: summed over the window, any thread static volatile int64_t frame_replay_ns; // the same since the previous present (frame-time summary) +// The replaying thread's CPU time over the same spans (perf_replay_cpu): the rest of the replay's wall time it was +// waiting (a lock, the recorder) or not running (preempted). Since the previous present, and over the period. +static volatile int64_t frame_replay_cpu_ns; +static uint64_t per_replay_ns, per_replay_cpu_ns; + +void perf_replay_cpu(uint64_t ns) { plat_add64(&frame_replay_cpu_ns, (int64_t)ns); } void perf_replay(uint64_t t0) { if (!t0) return; @@ -115,7 +122,12 @@ void perf_count(int kind) { #define FT_BINS 1024 // 0.25 ms bins up to 256 ms; slower frames land in the last bin #define FT_WORST 5 #define FT_PERIOD_NS 60000000000ull -typedef struct { uint64_t frame, ns, wait[PERF_N], replay; } FtWorst; +typedef struct { + uint64_t frame, ns, wait[PERF_N], replay; + uint64_t op_ns[PERF_RP_N], op_n[PERF_RP_N], max_ns, hold_ns; // NX2NAVI_REPLAY_TIME: the frame's replay ops + uint64_t replay_cpu; // the replaying thread's CPU time in the frame's replay + uint32_t max_op; +} FtWorst; // What a frame over 25 ms spent its time on, so a count of slow frames says where to look instead of only that they // happened. A frame is charged to the largest part when that part is at least a quarter of it, and to "unattributed" // otherwise - the part of a slow frame none of the recorded waits explains (the GPU, or a stall outside them). @@ -133,11 +145,119 @@ static uint64_t summary_t0; // start of the current period (0: not started) // NX2NAVI_REPLAY_TIME=1: how the replay splits between the kinds of op it walks, reported with the frame-time summary. // The replay is recorded on one thread, so these need no atomics. The kinds are PERF_RP_* in host.h. +// NX2NAVI_LOCK_TIME=1 (default 0): the replay threads time every lock they have to wait for (platform.h +// plat_lock_timing), kept per asking site, and the frame-time summary lists the sites that waited longest since the last +// summary, with the recorder's full-ring and drain waits. Sites are code addresses; "ref" is perf_lock_timing_here's, +// which the link map names, so the others resolve as offsets from it. +#define LT_N 512 +static struct { void *site; uint64_t ns, n, max, ns0, n0; } lt[LT_N]; +static struct { void *site; uint64_t ns, n, max, ns0, n0; } lh[LT_N]; // texture lock holds over 1 ms (perf_lock_hold) +static uint64_t lt_lost_ns, lt_rec0[2]; +static volatile int64_t fr_hold_ns; // the texture lock's holds over 1 ms since the last present (ft_frame) +void (*perf_rec_waits)(uint64_t *full_ns, uint64_t *drain_ns); + +static void lt_add(typeof(lt[0]) *tab, void *site, uint64_t ns) { + uint32_t h = (uint32_t)(((uintptr_t)site * 0x9E3779B97F4A7C15ull) >> 55); + for (uint32_t k = 0; k < 32; k++) { + typeof(lt[0]) *e = &tab[(h + k) % LT_N]; + void *cur = __atomic_load_n(&e->site, __ATOMIC_ACQUIRE); + if (!cur && __atomic_compare_exchange_n(&e->site, &cur, site, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) cur = site; + if (cur != site) continue; + __atomic_fetch_add(&e->ns, ns, __ATOMIC_RELAXED); + __atomic_fetch_add(&e->n, 1, __ATOMIC_RELAXED); + uint64_t m = __atomic_load_n(&e->max, __ATOMIC_RELAXED); + while (ns > m && !__atomic_compare_exchange_n(&e->max, &m, ns, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} + return; + } + __atomic_fetch_add(<_lost_ns, ns, __ATOMIC_RELAXED); +} + +static void lt_note(void *site, uint64_t ns) { lt_add(lt, site, ns); } +void perf_lock_hold(void *site, uint64_t ns) { + lt_add((typeof(lt[0]) *)lh, site, ns); + plat_add64(&fr_hold_ns, (int64_t)ns); +} + +static int lt_on = -1; +int perf_lock_holds_on(void) { + if (lt_on < 0) lt_on = cfg_bool("NX2NAVI_LOCK_TIME"); + return lt_on; +} +void perf_lock_timing_here(void) { + if (lt_on < 0) lt_on = cfg_bool("NX2NAVI_LOCK_TIME"); + if (lt_on) plat_lock_timing(lt_note); +} + +// The eight sites of tab with the most time since the last summary, appended to line; returns the period's total and +// sets *count, then starts a new period. +static uint64_t lt_top(typeof(lt[0]) *tab, char *line, int *n, size_t cap, uint64_t *count) { + uint64_t all = 0, cnt = 0; + int top[8], ntop = 0; + for (int i = 0; i < LT_N; i++) { + if (!__atomic_load_n(&tab[i].site, __ATOMIC_ACQUIRE)) continue; + uint64_t d = __atomic_load_n(&tab[i].ns, __ATOMIC_RELAXED) - tab[i].ns0; + all += d, cnt += __atomic_load_n(&tab[i].n, __ATOMIC_RELAXED) - tab[i].n0; + int at; + if (ntop < 8) at = ntop++; + else if (d > tab[top[7]].ns - tab[top[7]].ns0) at = 7; + else continue; + top[at] = i; + for (int j = at; j > 0; j--) { // keep top sorted by this period's time + uint64_t x = tab[top[j]].ns - tab[top[j]].ns0, y = tab[top[j - 1]].ns - tab[top[j - 1]].ns0; + if (x <= y) break; + int t = top[j]; top[j] = top[j - 1], top[j - 1] = t; + } + } + for (int j = 0; j < ntop && *n >= 0 && *n < (int)cap - 80; j++) { + typeof(lt[0]) *e = &tab[top[j]]; + *n += snprintf(line + *n, cap - *n, " %p %.1f ms (%llu, max %.2f ms);", e->site, (e->ns - e->ns0) / 1e6, + (unsigned long long)(e->n - e->n0), e->max / 1e6); + } + for (int i = 0; i < LT_N; i++) tab[i].ns0 = tab[i].ns, tab[i].n0 = tab[i].n, tab[i].max = 0; + *count = cnt; + return all; +} + +// The lock lines of a summary: the replay threads' waits with the recorder's, then the texture lock's long holds. +static void lt_report(void) { + uint64_t rec[2] = {0, 0}, waits, holds; + if (perf_rec_waits) perf_rec_waits(&rec[0], &rec[1]); + char line[1400], top[1200]; + int n = 0; + top[0] = 0; + uint64_t all = lt_top(lt, top, &n, sizeof top, &waits); + rt_log("frametime: replay lock waits: %.0f ms (%llu waits, %.0f ms unplaced); recorder ring full %.0f ms, drains " + "%.0f ms; ref %p:%s", all / 1e6, (unsigned long long)waits, lt_lost_ns / 1e6, (rec[0] - lt_rec0[0]) / 1e6, + (rec[1] - lt_rec0[1]) / 1e6, (void *)perf_lock_timing_here, top); + n = 0, top[0] = 0; + all = lt_top((typeof(lt[0]) *)lh, top, &n, sizeof top, &holds); + snprintf(line, sizeof line, "frametime: texture lock held over 1 ms: %.0f ms (%llu holds):%s", all / 1e6, + (unsigned long long)holds, top); + rt_log("%s", line); + lt_rec0[0] = rec[0], lt_rec0[1] = rec[1], lt_lost_ns = 0; +} + static uint64_t rp_ns[PERF_RP_N], rp_n[PERF_RP_N]; void perf_replay_op(int cat, uint64_t ns) { if ((unsigned)cat < PERF_RP_N) rp_ns[cat] += ns, rp_n[cat]++; } +// The replay ops since the last present (NX2NAVI_REPLAY_TIME), so a frame over 25 ms charged to the replay can say +// what its replay was: how many ops of each kind, their time, the slowest op and how long the texture lock was held +// over 1 ms (NX2NAVI_LOCK_TIME). The present thread takes and clears them in ft_frame; the replay adds. +static volatile int64_t fr_ns[PERF_RP_N], fr_n[PERF_RP_N], fr_max_ns, fr_max_op; +// Over the period: the frames over 25 ms charged to the replay, their sums, and how often each op was a frame's slowest. +static uint64_t sr_n, sr_ns[PERF_RP_N], sr_cnt[PERF_RP_N], sr_max_ns, sr_hold_ns, sr_max_op[256], sr_wall, sr_cpu; + +void perf_replay_op_at(int cat, uint32_t op, uint64_t ns) { + if ((unsigned)cat >= PERF_RP_N) return; + perf_replay_op(cat, ns); + plat_add64(&fr_ns[cat], (int64_t)ns); + plat_add64(&fr_n[cat], 1); + // one replay thread at a time records ops; a race with the present only misnames the frame's slowest op + if ((int64_t)ns > fr_max_ns) fr_max_ns = (int64_t)ns, fr_max_op = op; +} + static void ft_report(uint64_t now, const char *tag) { if (ft.frames && ft.sum) { uint64_t cut = (ft.frames + 99) / 100, acc = 0, low = 0; // the slowest 1% of frames @@ -166,17 +286,54 @@ static void ft_report(uint64_t now, const char *tag) { (unsigned long long)rp_n[PERF_RP_BIND], rp_ns[PERF_RP_XFER] / 1e6, (unsigned long long)rp_n[PERF_RP_XFER], rp_ns[PERF_RP_OTHER] / 1e6, (unsigned long long)rp_n[PERF_RP_OTHER]); } + if (lt_on > 0) lt_report(); + if (per_replay_ns && per_replay_cpu_ns) + rt_log("frametime: replay: %.0f ms, the replaying thread on the CPU for %.0f ms of it (%.0f%%); the frames over " + "25 ms charged to the replay: %.0f ms, on the CPU %.0f ms (%.0f%%)", per_replay_ns / 1e6, + per_replay_cpu_ns / 1e6, 100.0 * per_replay_cpu_ns / per_replay_ns, sr_wall / 1e6, sr_cpu / 1e6, + sr_wall ? 100.0 * sr_cpu / sr_wall : 0.0); + if (sr_n && (sr_cnt[PERF_RP_DRAW] || sr_cnt[PERF_RP_BIND])) { // the replay of the frames over 25 ms charged to it, per frame + int top[3] = {-1, -1, -1}; + for (int o = 0; o < 256; o++) + for (int j = 0; j < 3; j++) + if (sr_max_op[o] && (top[j] < 0 || sr_max_op[o] > sr_max_op[top[j]])) { + for (int k = 2; k > j; k--) top[k] = top[k - 1]; + top[j] = o; + break; + } + char ops[160]; + int n = 0; + ops[0] = 0; + for (int j = 0; j < 3 && top[j] >= 0; j++) + n += snprintf(ops + n, sizeof ops - n, "%s%s %llu", j ? ", " : "", nvn_op_name((uint32_t)top[j]), + (unsigned long long)sr_max_op[top[j]]); + rt_log("frametime: slow replay frames %llu, per frame: draws %.1f ms (%.0f), binds %.1f ms (%.0f), transfers " + "%.1f ms (%.0f); its slowest op %.1f ms (%s); texture lock held over 1 ms %.1f ms", + (unsigned long long)sr_n, sr_ns[PERF_RP_DRAW] / 1e6 / sr_n, (double)sr_cnt[PERF_RP_DRAW] / sr_n, + sr_ns[PERF_RP_BIND] / 1e6 / sr_n, (double)sr_cnt[PERF_RP_BIND] / sr_n, sr_ns[PERF_RP_XFER] / 1e6 / sr_n, + (double)sr_cnt[PERF_RP_XFER] / sr_n, sr_max_ns / 1e6 / sr_n, ops, sr_hold_ns / 1e6 / sr_n); + } for (int i = 0; i < ft.nworst; i++) { const FtWorst *w = &ft.worst[i]; - rt_log("frametime: worst frame %llu: %.1f ms; shader translation %.1f, pipeline build %.1f, texture upload %.1f (astc %.1f), replay %.1f, fence/present wait %.1f, pacing %.1f", + char ops[200] = ""; + if (w->op_n[PERF_RP_DRAW] || w->op_n[PERF_RP_BIND] || w->op_n[PERF_RP_XFER]) + snprintf(ops, sizeof ops, "; ops: draws %.1f ms (%llu), binds %.1f ms (%llu), transfers %.1f ms (%llu), slowest " + "%s %.1f ms, texture lock held %.1f ms", w->op_ns[PERF_RP_DRAW] / 1e6, + (unsigned long long)w->op_n[PERF_RP_DRAW], w->op_ns[PERF_RP_BIND] / 1e6, + (unsigned long long)w->op_n[PERF_RP_BIND], w->op_ns[PERF_RP_XFER] / 1e6, + (unsigned long long)w->op_n[PERF_RP_XFER], nvn_op_name(w->max_op), w->max_ns / 1e6, w->hold_ns / 1e6); + rt_log("frametime: worst frame %llu: %.1f ms; shader translation %.1f, pipeline build %.1f, texture upload %.1f (astc %.1f), replay %.1f (on the CPU %.1f), fence/present wait %.1f, pacing %.1f%s", (unsigned long long)w->frame, w->ns / 1e6, w->wait[PERF_XLATE] / 1e6, w->wait[PERF_PIPE] / 1e6, - w->wait[PERF_TEX] / 1e6, w->wait[PERF_ASTC] / 1e6, w->replay / 1e6, - (w->wait[PERF_FENCE] + w->wait[PERF_DRAIN]) / 1e6, w->wait[PERF_PACE] / 1e6); + w->wait[PERF_TEX] / 1e6, w->wait[PERF_ASTC] / 1e6, w->replay / 1e6, w->replay_cpu / 1e6, + (w->wait[PERF_FENCE] + w->wait[PERF_DRAIN]) / 1e6, w->wait[PERF_PACE] / 1e6, ops); } } memset(&ft, 0, sizeof ft); memset(rp_ns, 0, sizeof rp_ns); memset(rp_n, 0, sizeof rp_n); + sr_n = sr_max_ns = sr_hold_ns = sr_wall = sr_cpu = 0; + per_replay_ns = per_replay_cpu_ns = 0; + memset(sr_ns, 0, sizeof sr_ns), memset(sr_cnt, 0, sizeof sr_cnt), memset(sr_max_op, 0, sizeof sr_max_op); ft.t0 = now; } @@ -196,6 +353,12 @@ void perf_summary_flush(void) { // at exit (prof_write) static uint64_t ft_frame(uint64_t frame, uint64_t now, uint64_t d) { int64_t rep = plat_xchg64(&frame_replay_ns, 0); vk_texstat_frame(frame, d); + uint64_t f_cpu = (uint64_t)plat_xchg64(&frame_replay_cpu_ns, 0); + per_replay_ns += rep > 0 ? (uint64_t)rep : 0, per_replay_cpu_ns += f_cpu; + uint64_t f_ns[PERF_RP_N], f_n[PERF_RP_N], f_max = (uint64_t)plat_xchg64(&fr_max_ns, 0); + uint64_t f_hold = (uint64_t)plat_xchg64(&fr_hold_ns, 0); + uint32_t f_op = (uint32_t)fr_max_op; + for (int c = 0; c < PERF_RP_N; c++) f_ns[c] = (uint64_t)plat_xchg64(&fr_ns[c], 0), f_n[c] = (uint64_t)plat_xchg64(&fr_n[c], 0); if (!summary_t0) { // the first present starts the period; there is no interval yet summary_t0 = now; ft.t0 = now; @@ -225,6 +388,11 @@ static uint64_t ft_frame(uint64_t frame, uint64_t now, uint64_t d) { if (part[best] * 4 < d) best = FT_C_OTHER; // no recorded part explains a quarter of the frame ft.slow_n[best]++; ft.slow_ns[best] += d; + if (best == FT_C_REPLAY) { + sr_n++, sr_max_ns += f_max, sr_hold_ns += f_hold, sr_max_op[f_op & 255]++; + sr_wall += rep > 0 ? (uint64_t)rep : 0, sr_cpu += f_cpu; + for (int c = 0; c < PERF_RP_N; c++) sr_ns[c] += f_ns[c], sr_cnt[c] += f_n[c]; + } } if (d > 25000000ull && (ft.nworst < FT_WORST || d > ft.worst[FT_WORST - 1].ns)) { int i = ft.nworst < FT_WORST ? ft.nworst++ : FT_WORST - 1; @@ -233,6 +401,8 @@ static uint64_t ft_frame(uint64_t frame, uint64_t now, uint64_t d) { ft.worst[i].ns = d; memcpy(ft.worst[i].wait, frame_ns, sizeof frame_ns); ft.worst[i].replay = (uint64_t)rep; + memcpy(ft.worst[i].op_ns, f_ns, sizeof f_ns), memcpy(ft.worst[i].op_n, f_n, sizeof f_n); + ft.worst[i].max_ns = f_max, ft.worst[i].max_op = f_op, ft.worst[i].hold_ns = f_hold, ft.worst[i].replay_cpu = f_cpu; } if (now - summary_t0 >= FT_PERIOD_NS) { ft_report(now, "60 s"); diff --git a/runtime/platform/common.c b/runtime/platform/common.c index 38c7e3ca..8c799e7a 100644 --- a/runtime/platform/common.c +++ b/runtime/platform/common.c @@ -4,17 +4,19 @@ // ---- recursive mutex: owner is the holder's thread id (0 when free); only the holder writes depth. -void plat_rmutex_lock(PlatRMutex *m) { +void plat_rmutex_lock_at(PlatRMutex *m, void *site) { uint32_t me = plat_thread_id(); if (__atomic_load_n(&m->owner, __ATOMIC_RELAXED) == me) { m->depth++; return; } - plat_mutex_lock(&m->m); + plat_mutex_lock_at(&m->m, site); __atomic_store_n(&m->owner, me, __ATOMIC_RELAXED); m->depth = 1; } +void plat_rmutex_lock(PlatRMutex *m) { plat_rmutex_lock_at(m, __builtin_return_address(0)); } + int plat_rmutex_trylock(PlatRMutex *m) { uint32_t me = plat_thread_id(); if (__atomic_load_n(&m->owner, __ATOMIC_RELAXED) == me) { diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index 8cfa0612..4b96d39b 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -562,6 +562,10 @@ int plat_mutex_trylock(PlatMutex *m) { return 0; } +// Lock-wait timing is Windows only (platform.h): here the note is kept and never called. +void plat_lock_timing(PlatLockNote fn) { (void)fn; } +void plat_mutex_lock_at(PlatMutex *m, void *site) { (void)site; plat_mutex_lock(m); } + void plat_mutex_lock(PlatMutex *m) { volatile uint32_t *w = lk(m); for (int spin = 0;; spin++) { diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 108a29f4..39071559 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -161,6 +161,14 @@ void plat_rmutex_lock(PlatRMutex *m); int plat_rmutex_trylock(PlatRMutex *m); // 1 when acquired (or already held by this thread) void plat_rmutex_unlock(PlatRMutex *m); +// Lock-wait timing (NX2NAVI_LOCK_TIME, perf.c): after plat_lock_timing(fn) on a thread, each plat_mutex_lock(_shared) +// or plat_rmutex_lock there that has to wait calls fn with the address of the code that asked for the lock (for a +// recursive mutex, the caller of plat_rmutex_lock) and the nanoseconds it waited. NULL turns it off. Windows only. +typedef void (*PlatLockNote)(void *site, uint64_t ns); +void plat_lock_timing(PlatLockNote fn); +void plat_mutex_lock_at(PlatMutex *m, void *site); // plat_mutex_lock reporting site as the asker +void plat_rmutex_lock_at(PlatRMutex *m, void *site); // plat_rmutex_lock reporting site as the asker + // One-time initialisation: the first caller runs fn(arg), every other caller waits until it returned. Zero-initialised // (PLAT_ONCE_INIT) has not run. typedef struct { volatile int32_t state; } PlatOnce; diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index a91968b6..3027f07f 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -465,11 +465,35 @@ void plat_sleep_ms(uint32_t ms) { Sleep(ms); } // ---- locks and condition variables void plat_mutex_init(PlatMutex *m) { InitializeSRWLock((PSRWLOCK)m); } -void plat_mutex_lock(PlatMutex *m) { AcquireSRWLockExclusive((PSRWLOCK)m); } +static _Thread_local PlatLockNote lock_note; // plat_lock_timing +void plat_lock_timing(PlatLockNote fn) { lock_note = fn; } + +void plat_mutex_lock_at(PlatMutex *m, void *site) { + if (!lock_note || TryAcquireSRWLockExclusive((PSRWLOCK)m)) { + if (!lock_note) AcquireSRWLockExclusive((PSRWLOCK)m); + return; + } + uint64_t t0 = plat_now_ns(); + AcquireSRWLockExclusive((PSRWLOCK)m); + lock_note(site, plat_now_ns() - t0); +} + +void plat_mutex_lock(PlatMutex *m) { + if (!lock_note) { AcquireSRWLockExclusive((PSRWLOCK)m); return; } + plat_mutex_lock_at(m, __builtin_return_address(0)); +} int plat_mutex_trylock(PlatMutex *m) { return TryAcquireSRWLockExclusive((PSRWLOCK)m) != 0; } void plat_mutex_unlock(PlatMutex *m) { ReleaseSRWLockExclusive((PSRWLOCK)m); } -void plat_mutex_lock_shared(PlatMutex *m) { AcquireSRWLockShared((PSRWLOCK)m); } +void plat_mutex_lock_shared(PlatMutex *m) { + if (!lock_note || TryAcquireSRWLockShared((PSRWLOCK)m)) { + if (!lock_note) AcquireSRWLockShared((PSRWLOCK)m); + return; + } + uint64_t t0 = plat_now_ns(); + AcquireSRWLockShared((PSRWLOCK)m); + lock_note(__builtin_return_address(0), plat_now_ns() - t0); +} int plat_mutex_trylock_shared(PlatMutex *m) { return TryAcquireSRWLockShared((PSRWLOCK)m) != 0; } void plat_mutex_unlock_shared(PlatMutex *m) { ReleaseSRWLockShared((PSRWLOCK)m); } diff --git a/runtime/prof_win.c b/runtime/prof_win.c index cb160358..c5047ca1 100644 --- a/runtime/prof_win.c +++ b/runtime/prof_win.c @@ -61,6 +61,7 @@ static volatile LONG stop, written; static uintptr_t exe_lo, exe_hi; static int only_render; // NX2NAVI_PROFILE_RENDER=1: sample the render thread only static int only_main; // NX2NAVI_PROFILE_MAIN=1: sample the guest main thread only +static int only_replay; // NX2NAVI_PROFILE_REPLAY=1: sample the replay thread only, without call chains static int chains; // call chains are kept: NX2NAVI_PROFILE_RENDER or NX2NAVI_PROFILE_CHAINS static volatile uint64_t cur_frame; // last presented frame static uint64_t frame_lo, frame_hi; // NX2NAVI_PROFILE_FRAMES=-: sample only while presents lo..hi-1 are pending @@ -262,7 +263,8 @@ static DWORD WINAPI sampler_main(void *arg) { uint64_t now = slow_ns ? kern_now_ns() : 0; for (int i = 0; i < n_threads; i++) { ThreadSlot *t = &threads[i]; - if ((only_render && t->tid != render_tid) || (only_main && t->tid != main_tid)) continue; + if ((only_render && t->tid != render_tid) || (only_main && t->tid != main_tid) || (only_replay && t->tid != replay_tid)) + continue; uint64_t cyc = 0; if (slow_ns && t->depth && QueryThreadCycleTime(t->h, &cyc) && cyc == t->cycles) { // still waiting Sample *s = &ring[ring_n++ & (SLOW_RING - 1)]; @@ -365,6 +367,7 @@ int prof_sampler_start(void) { } only_render = cfg_get("NX2NAVI_PROFILE_RENDER") != NULL; only_main = !only_render && cfg_get("NX2NAVI_PROFILE_MAIN") != NULL; + only_replay = !only_render && !only_main && cfg_get("NX2NAVI_PROFILE_REPLAY") != NULL; const char *ch = cfg_get("NX2NAVI_PROFILE_CHAINS"); const char *sl = cfg_get("NX2NAVI_PROFILE_SLOW"); slow_ns = sl && atoi(sl) > 0 ? (uint64_t)atoi(sl) * 1000000ull : 0; diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index c2444c56..3bbe4abb 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -21,6 +21,7 @@ #include "nvn_api.h" #include "nvn_tex.h" #include "platform/platform.h" +#include "prof.h" #include "tex.h" #include "vk.h" #include "vk_exec.h" @@ -1205,6 +1206,9 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t if (!said++) rt_log("vk: a submission is recorded on another thread than the first one; it runs without the " "recorder's priority and core"); } + perf_lock_timing_here(); // NX2NAVI_LOCK_TIME + if (prof_replay_tid != plat_thread_id()) perf_replay_thread(); // the profiler follows the thread replaying now + perf_rec_waits = rec_wait_totals; VkExec *ex = &X; if (!x_ready) state_reset(ex), vk_dump_init(), x_ready = 1; const tex_t *keep[9]; // the render targets carry over to this submission (set_rt compares them by pointer) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index dfada1d2..027fe517 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -4881,5 +4881,5 @@ void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) { if (!on) { op_draw_impl(ex, op, payload); return; } uint64_t t0 = plat_now_ns(); op_draw_impl(ex, op, payload); - perf_replay_op(op_is_draw(op) ? PERF_RP_DRAW : PERF_RP_BIND, plat_now_ns() - t0); + perf_replay_op_at(op_is_draw(op) ? PERF_RP_DRAW : PERF_RP_BIND, op, plat_now_ns() - t0); } diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 57109bb3..440123e4 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -817,5 +817,5 @@ void vk_exec_op_xfer(VkExec *ex, uint32_t op, const void *payload) { if (!on) { op_xfer_impl(ex, op, payload); return; } uint64_t t0 = plat_now_ns(); op_xfer_impl(ex, op, payload); - perf_replay_op(PERF_RP_XFER, plat_now_ns() - t0); + perf_replay_op_at(PERF_RP_XFER, op, plat_now_ns() - t0); } diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index b0d892b4..b7fe82b6 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -31,6 +31,7 @@ static volatile uint64_t part_fence; // the value of the part being recorded ( // Producer statistics, logged by rec_stats_log. static struct { uint64_t packets, words, drains, fallbacks, drain_ns, full_waits, parts, parts_on; } RS_; +static volatile uint64_t full_ns_all, drain_ns_all; // since the start, for rec_wait_totals (never reset) enum { R_PAD, R_BeginQuery, R_BeginRendering, R_BindDescriptorSets, R_BindIndexBuffer, R_BindPipeline, R_BindVertexBuffers2, @@ -92,9 +93,11 @@ static void wake(void) { static void wait_space(uint64_t need) { if (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) <= RING_WORDS) return; RS_.full_waits++; + uint64_t t0 = now_ns(); __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); wake(); while (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) > RING_WORDS) _mm_pause(); + full_ns_all += now_ns() - t0; } // Room for a packet of bytes of arguments; the caller fills it and calls pub(). @@ -130,9 +133,12 @@ void rec_drain(void) { for (int i = 0; __atomic_load_n(&tail, __ATOMIC_ACQUIRE) != put_h; i++) if (i < 20000) _mm_pause(); else plat_yield(); - RS_.drains++, RS_.drain_ns += now_ns() - t0; + uint64_t d = now_ns() - t0; + RS_.drains++, RS_.drain_ns += d, drain_ns_all += d; } +void rec_wait_totals(uint64_t *full_ns, uint64_t *drain_ns) { *full_ns = full_ns_all, *drain_ns = drain_ns_all; } + void rec_begin(VkCommandBuffer cb, uint64_t fence) { static int mode = -1; if (mode < 0) { diff --git a/runtime/vk_rec.h b/runtime/vk_rec.h index 62c5be1a..9261ee16 100644 --- a/runtime/vk_rec.h +++ b/runtime/vk_rec.h @@ -20,6 +20,7 @@ void rec_end(void); // waits for the recorder; no-op when rec_ // when it records cb, and the wait for it (1); otherwise rec_end and the breadcrumb, and the caller ends cb (0). int rec_end_cb(VkCommandBuffer cb, int crumb, uint64_t n); void rec_drain(void); // waits until the recorder made every call queued so far +void rec_wait_totals(uint64_t *full_ns, uint64_t *drain_ns); // producer time spent on a full ring and in drains void rec_stats_log(void); // NX2NAVI_TRACE=perf: packets, bytes, drains and waits since the last call // NX2NAVI_REC_CHECK=1 (debug, recorder on): vk_memory.c reports each object it defers (destroy 0, stamped tl) or destroys // (destroy 1) with the value the GPU reached (gpu); a packet of the part being recorded that still names it is logged diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 4cb06ada..088e0144 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -524,7 +524,11 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { CmdSlot *s = take_slot(c, fence); VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; - uint64_t t0 = perf_begin(); + // NX2NAVI_REPLAY_TIME: the replaying thread's CPU time over the replay too (perf_replay_cpu), against its wall time + static int cpu_on = -1; + if (cpu_on < 0) cpu_on = cfg_bool("NX2NAVI_REPLAY_TIME"); + uint32_t me = plat_thread_id(); + uint64_t t0 = perf_begin(), cpu0 = cpu_on ? plat_thread_cpu_ns(me) : 0; vkBeginCommandBuffer(s->cb, &bi); vk_crumb(s->cb, VK_CRUMB_SUBMIT, fence); ts_begin(s->cb); @@ -536,6 +540,7 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { // The recorder is done with the part: the end-of-submission crumb, then the end, by the recorder when it records. if (!rec_end_cb(s->cb, VK_CRUMB_SUBMIT_END, fence)) vkEndCommandBuffer(s->cb); perf_replay(t0); + if (cpu0) perf_replay_cpu(plat_thread_cpu_ns(me) - cpu0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. vk_upload_flush(0); VkCommandBuffer pro = prologue(c, fence); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 217ec08e..eefdf380 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -204,11 +204,23 @@ static void tex_init(void *p) { __atomic_store_n(&tex_ready, 1, __ATOMIC_RELEASE); return; } -static void lock(void) { +// NX2NAVI_LOCK_TIME: the outermost hold of the lock is timed from here, and one over 1 ms is reported with the code +// that took it (perf_lock_hold); lock is not inlined so that its return address names that code. +static uint64_t hold_t0; +static void *hold_site; +__attribute__((noinline)) static void lock(void) { if (!__atomic_load_n(&tex_ready, __ATOMIC_ACQUIRE)) plat_once(&tex_once, tex_init, NULL); - plat_rmutex_lock(&tex_lock); + plat_rmutex_lock_at(&tex_lock, __builtin_return_address(0)); + if (tex_lock.depth == 1 && perf_lock_holds_on()) hold_t0 = plat_now_ns(), hold_site = __builtin_return_address(0); +} +static void unlock(void) { + if (tex_lock.depth == 1 && hold_t0) { + uint64_t d = plat_now_ns() - hold_t0; + hold_t0 = 0; + if (d > 1000000) perf_lock_hold(hold_site, d); + } + plat_rmutex_unlock(&tex_lock); } -static void unlock(void) { plat_rmutex_unlock(&tex_lock); } static uint32_t umax(uint32_t a, uint32_t b) { return a > b ? a : b; } static uint32_t mip(uint32_t v, unsigned lv) { return umax(v >> lv, 1); }