From 2f640fb8ed33f70270837cf2afdda25c29bab820 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Fri, 13 Nov 2026 12:54:05 -0500 Subject: [PATCH] Make the stutter map count runs of long frames as one hitch and the texture pool's area loads as loads. In the 2026-11-13 session three frames of 87, 91 and 96 ms after an area load each counted as a change of pace, as each was followed by one as long, and loads that only the texture pool saw (no area file read, so no scene line) were charged to play. Consecutive long frames are now one episode, counted once with their summed time, its line giving that time and the first frame's parts; a run of more than 8 long frames none of which stands out is still a change of pace. The texture pool's start and end of an area load mark an area load for the map, as the game's own loads do. With trace = perf, the slow write-watch queries and freezes past the 200 lines they may log are counted and given every 60 frames, so a late area load still shows them; the session's lines had run out by frame 36,387. Co-Authored-By: Claude Opus 5.5 --- runtime/stutter_map.c | 71 +++++++++++++++++++++++++++++++------ runtime/stutter_map.h | 3 ++ runtime/vk_exec_draw.c | 2 ++ runtime/vk_memory.c | 25 +++++++++++-- tests/stutter_hitch_check.c | 27 +++++++++++--- tests/stutter_map_check.py | 8 ++++- 6 files changed, 118 insertions(+), 18 deletions(-) diff --git a/runtime/stutter_map.c b/runtime/stutter_map.c index 6c81c1ca..f0e63b1f 100644 --- a/runtime/stutter_map.c +++ b/runtime/stutter_map.c @@ -10,7 +10,11 @@ // 60 or 30 fps after 140 is a slower rate, not a run of hitches, and the median takes the new rate within 16 frames. // The decision waits two presents; the line and the counts carry what the frame itself waited for. Frames within // SM_LOAD_NS of an area's load count apart, as the load's: an area load shows as a few long frames that are not a -// stutter in play. +// stutter in play. An area load is the game's (stutter_area) or the texture pool's (stutter_load: the pool makes an +// area's textures, vk_exec_draw.c pool_loading), which comes without the game's when it streams the next area in. +// Consecutive long frames are one episode, a hitch counted once with their summed time: a change of pace is a run of +// more than SM_EPISODE_MAX long frames none of which stands out from the ones after it. Three frames of 87, 91 and +// 96 ms after an area load (2026-11-13) each counted as a change of pace before, as each was followed by one as long. // // What a hitch went to comes from the frame-time summary's split (perf.c ft_frame: pipeline, translation, texture, // fence, pacing, replay, unattributed), with the draws that met a pipeline or a shader missing between the previous @@ -46,6 +50,7 @@ int stutter_map_on; #define SM_MEDIAN_N 32 // frames the median is taken over #define SM_AFTER 2 // frames after a candidate that must be shorter... #define SM_AFTER_RATIO 1.5 // ...by this factor +#define SM_EPISODE_MAX 8 // a run of long frames up to this long is an episode, longer a new pace #define SM_PIPES 4096 // pipelines kept for the ranking (open addressing) #define SM_RING 256 // draws that met something missing, between two presents #define SM_LINES_MAX 4000 @@ -93,11 +98,21 @@ static uint64_t t_first, t_report, line_sec, lines_in_sec, lines, frames_seen, p // draws that met something missing then), its area and what it is charged to. typedef struct { char line[1400]; - uint64_t frame, d, after_max; + uint64_t frame, d, after_max, med; + double t_s; // seconds since the first frame int area, load, cause, after; + int tail; // where the line's text after its head ("stutter: frame ... in [, after its load]:") starts } Cand; static Cand cand[SM_AFTER + 1]; static int n_cand; +// The episode being gathered: decided candidates of consecutive frames, the first one's line and area (cand_decide). +static struct { + Cand first; + uint64_t last, total; // the last frame in it; the frames' summed intervals + int n, hitch, open; // frames in it; one of them stood out from the frames after it (or was over the sure line) +} ep; +static int pace_run; // more than SM_EPISODE_MAX long frames, none standing out: a new pace, until it ends +static uint64_t pace_last; void stutter_map_init(void) { stutter_map_on = cfg_bool("NX2NAVI_STUTTER_MAP"); @@ -116,6 +131,10 @@ static uint32_t id4(const uint8_t *id) { static uint64_t key_of(const uint8_t *vi, const uint8_t *fi) { return (uint64_t)id4(vi) << 32 | id4(fi); } +void stutter_load(void) { + if (stutter_map_on) __atomic_store_n(&load_at, kern_now_ns(), __ATOMIC_RELAXED); +} + void stutter_area(const char *name) { if (!stutter_map_on || !name || !*name) return; plat_mutex_lock(&sm_lock); @@ -230,10 +249,12 @@ static uint64_t median_recent(void) { return v[n / 2]; } -// A candidate the SM_AFTER frames after it have been seen for: a hitch, counted and its line logged, or a change of -// pace (the frames after it as long), left out. -static void cand_decide(const Cand *c, uint64_t now) { - if (c->d <= SM_HITCH_SURE_NS && (double)c->after_max * SM_AFTER_RATIO >= (double)c->d) { +// The episode ends: a hitch with its frames' summed time, counted in its first frame's area and charged to that frame's +// cause, and its line logged; a single frame that did not stand out from the ones after it is a change of pace. +static void episode_close(uint64_t now) { + ep.open = 0; + const Cand *c = &ep.first; + if (!ep.hitch && ep.n < 2) { paced_out++; return; } @@ -241,17 +262,43 @@ static void cand_decide(const Cand *c, uint64_t now) { if (c->load) a->load_hitch++; else { a->hitch++; - if (c->cause >= 0 && c->cause < SM_CAUSES) a->by_cause[c->cause]++, a->cause_ns[c->cause] += c->d; - if (c->d > a->worst_ns) a->worst_ns = c->d, a->worst_frame = c->frame; + if (c->cause >= 0 && c->cause < SM_CAUSES) a->by_cause[c->cause]++, a->cause_ns[c->cause] += ep.total; + if (ep.total > a->worst_ns) a->worst_ns = ep.total, a->worst_frame = c->frame; } uint64_t sec = (now - t_first) / 1000000000ull; if (sec != line_sec) line_sec = sec, lines_in_sec = 0; if (lines >= SM_LINES_MAX || lines_in_sec >= SM_LINES_PER_S) return; - rt_log("%s", c->line); + if (ep.n == 1) rt_log("%s", c->line); + else // the head with the episode's time; the parts and the draws are its first frame's + rt_log("stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:%s; an episode of %d frames to frame " + "%llu, %.1f ms the first, the parts above the first frame's", (unsigned long long)c->frame, c->t_s, + ep.total / 1e6, c->med / 1e6, areas[c->area].name, c->load ? ", after its load" : "", c->line + c->tail, + ep.n, (unsigned long long)ep.last, c->d / 1e6); lines++, lines_in_sec++; if (lines == SM_LINES_MAX) rt_log("stutter: the last such line of this run; the map's counts go on"); } +// A candidate the SM_AFTER frames after it have been seen for: it stands out (a hitch) or the frames after it are as +// long (paced). It joins the episode of the frame before it, or starts one. An episode of more than SM_EPISODE_MAX +// frames none of which stood out is a change of pace: they and the paced frames right after them are left out. +static void cand_decide(const Cand *c, uint64_t now) { + int paced = c->d <= SM_HITCH_SURE_NS && (double)c->after_max * SM_AFTER_RATIO >= (double)c->d; + if (pace_run) { + if (paced && c->frame == pace_last + 1) { + paced_out++, pace_last = c->frame; + return; + } + pace_run = 0; + } + if (ep.open && c->frame == ep.last + 1) { + ep.last = c->frame, ep.total += c->d, ep.n++, ep.hitch |= !paced; + if (!ep.hitch && ep.n > SM_EPISODE_MAX) paced_out += (uint64_t)ep.n, ep.open = 0, pace_run = 1, pace_last = c->frame; + return; + } + if (ep.open) episode_close(now); + ep.first = *c, ep.last = c->frame, ep.total = c->d, ep.n = 1, ep.hitch = !paced, ep.open = 1; +} + void stutter_frame(uint64_t frame, uint64_t now, uint64_t d, const uint64_t *part, const char *const *names, int n, int cause) { if (!stutter_map_on || !d) return; @@ -272,6 +319,7 @@ void stutter_frame(uint64_t frame, uint64_t now, uint64_t d, const uint64_t *par memmove(&cand[i], &cand[i + 1], (size_t)(n_cand - i - 1) * sizeof cand[0]); // at most two others n_cand--; } + if (ep.open && frame > ep.last + SM_AFTER) episode_close(now); // the frame after its last was no candidate uint64_t med = median_recent(); recent[frames_seen % SM_MEDIAN_N] = d; frames_seen++; @@ -284,9 +332,10 @@ void stutter_frame(uint64_t frame, uint64_t now, uint64_t d, const uint64_t *par Cand *c = &cand[n_cand++]; c->frame = frame, c->d = d, c->after_max = 0, c->after = 0, c->area = ai, c->cause = cause; c->load = now - __atomic_load_n(&load_at, __ATOMIC_RELAXED) < SM_LOAD_NS; + c->med = med, c->t_s = (now - t_first) / 1e9; int o = snprintf(c->line, sizeof c->line, "stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:", - (unsigned long long)frame, (now - t_first) / 1e9, d / 1e6, med / 1e6, a->name, - c->load ? ", after its load" : ""); + (unsigned long long)frame, c->t_s, d / 1e6, med / 1e6, a->name, c->load ? ", after its load" : ""); + c->tail = o < (int)sizeof c->line ? o : (int)sizeof c->line - 1; for (int k = 0; k < n && o < (int)sizeof c->line - 40; k++) if (part[k] >= 500000) o += snprintf(c->line + o, sizeof c->line - (size_t)o, " %s %.1f", names[k], part[k] / 1e6); if (cause >= 0 && cause < n && o < (int)sizeof c->line - 40) diff --git a/runtime/stutter_map.h b/runtime/stutter_map.h index 42584060..5a4afcd6 100644 --- a/runtime/stutter_map.h +++ b/runtime/stutter_map.h @@ -10,6 +10,9 @@ void stutter_map_init(void); // An area's load (perf.c perf_scene, from romfs.c): hitches from here on are counted in that area, and those within // SM_LOAD_NS of the load apart from play. void stutter_area(const char *name); +// The texture pool began or ended making an area's textures (vk_exec_draw.c pool_loading): frames within SM_LOAD_NS of +// it count as an area load's too, in the area of the last stutter_area. +void stutter_load(void); // A draw met its pipeline (shader 0) or one of its shaders (shader 1) missing and the replay thread spent ns on it // before it was bound or skipped (vk_exec_draw.c, beside wait_note). vi, fi: the vertex (or mesh) and fragment shaders' // input ids, NULL when not known (still translating, or no fragment stage). diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 5dac3f10..d95305c5 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -2967,8 +2967,10 @@ static void pool_sync_entries(VkExec *ex) { if (!pool_loading && backlog > 0 && retry >= backlog) { pool_load_made = (uint64_t)retry, pool_load_frame0 = PL.frame; __atomic_store_n(&pool_loading, 1, __ATOMIC_RELEASE); + stutter_load(); // the stutter map counts the frames around it as an area load's } else if (pool_loading && retry < POOL_LOAD_END) { __atomic_store_n(&pool_loading, 0, __ATOMIC_RELEASE); + stutter_load(); rt_log("vk: texture pool: an area load of about %llu textures was made in %llu frames (NX2NAVI_POOL_LOAD_BACKLOG)", (unsigned long long)pool_load_made, (unsigned long long)(PL.frame - pool_load_frame0 + 1)); } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index fa6457a3..8a98ae4b 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -975,6 +975,7 @@ static void owner_log(uint64_t frame); // the memory report, after the staging static void compact_look(void); static void outside_watch(uint64_t frame); static void up_ts_log(uint64_t frame); +static void slow_mem_window(uint64_t frame); static volatile uint64_t mirror_frame; // the frame vk_mem_census last saw, for mirror_sync's log lines @@ -1004,7 +1005,8 @@ void vk_mem_census(uint64_t frame) { qpc_ms(h2 - h1)); } if (vk_texture_compact_due()) compact_look(); // NX2NAVI_TEX_COMPACT: 0 at once with the setting off - if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60); + if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) + outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60), slow_mem_window(frame); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame), shadow_log(frame), span_log(frame); // mirror_log also logs the write-watch line @@ -1184,14 +1186,33 @@ static uint64_t ww_calls, ww_bytes, ww_ticks, ww_dirty_calls, ww_dirty_ticks, ww // 200). A frame some ten frames after an area load took 51.6 ms of replay with 1.2 ms of it in the replay's ops; the // replaying thread's samples sat in GetWriteWatch under vk_mem_mirror_begin and in the freeze's VirtualProtect, and no // thread of the process could be sampled for 28 ms in between. +// Past the 200 lines the calls are counted, and slow_mem_window gives the count, their time and the longest every 60 +// frames: a session of 2026-11-13 used the 200 lines up by frame 36,387, and its last area load had none. +static volatile int64_t slow_more_n, slow_more_ns, slow_more_max; + static void slow_mem_call(const char *what, uint64_t addr, uint64_t size, uint64_t pages, uint64_t ns) { static uint32_t logged; - if (ns < 4000000ull || !(rt_trace & RT_TRACE_PERF) || __atomic_add_fetch(&logged, 1, __ATOMIC_RELAXED) > 200) return; + if (ns < 4000000ull || !(rt_trace & RT_TRACE_PERF)) return; + if (__atomic_add_fetch(&logged, 1, __ATOMIC_RELAXED) > 200) { + plat_add64(&slow_more_n, 1), plat_add64(&slow_more_ns, (int64_t)ns); + int64_t m = __atomic_load_n(&slow_more_max, __ATOMIC_RELAXED); + while ((int64_t)ns > m && !__atomic_compare_exchange_n(&slow_more_max, &m, (int64_t)ns, 1, __ATOMIC_RELAXED, + __ATOMIC_RELAXED)) {} + return; + } rt_log("mem: slow %s: %.1f ms over %#llx + %.1f MiB, %lld pages written (frame %llu)", what, ns / 1e6, (unsigned long long)addr, size / 1048576.0, pages == UINT64_MAX ? -1ll : (long long)pages, (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED)); } +static void slow_mem_window(uint64_t frame) { // NX2NAVI_TRACE=perf, every 60 frames + int64_t n = plat_xchg64(&slow_more_n, 0); + if (!n) return; + int64_t ns = plat_xchg64(&slow_more_ns, 0), max = plat_xchg64(&slow_more_max, 0); + rt_log("mem: frame %llu: %lld more slow write-watch queries and freezes (over 4 ms) since the last such line, %.1f ms, " + "the longest %.1f ms", (unsigned long long)frame, (long long)n, ns / 1e6, max / 1e6); +} + static uint64_t ww_query(uint64_t addr, uint64_t size) { uint64_t t0 = qpc(); uint64_t n = mem_written_pages(addr, size, gen_pages, sizeof gen_pages / sizeof *gen_pages); diff --git a/tests/stutter_hitch_check.c b/tests/stutter_hitch_check.c index 1d537ec8..45932f6a 100644 --- a/tests/stutter_hitch_check.c +++ b/tests/stutter_hitch_check.c @@ -48,14 +48,33 @@ int main(void) { frames(200, 7.0); for (int k = 0; k < 5; k++) frames(1, 30.0), frames(60, 7.0); // five isolated spikes expect("five isolated spikes", areas[0].hitch, 5); - frames(1, 150.0), frames(1, 150.0), frames(60, 7.0); // two long frames in a row: the second stands out - expect("two frames over the sure line", areas[0].hitch, 7); - frames(1, 40.0), frames(1, 40.0), frames(60, 7.0); // a double spike: the second one counts - expect("a double spike", areas[0].hitch, 8); + frames(1, 150.0), frames(1, 150.0), frames(60, 7.0); // two long frames in a row: one episode of 300 ms + expect("two frames over the sure line", areas[0].hitch, 6); + expect("their episode's time", areas[0].worst_ns, 300000000ull); + frames(1, 40.0), frames(1, 40.0), frames(60, 7.0); // a double spike: one episode + expect("a double spike", areas[0].hitch, 7); + // three long frames, each followed by one as long, as after an area load in a session of 2026-11-13 (87, 91 and + // 96 ms): one episode, not three changes of pace + uint64_t paced0 = paced_out, charged0 = areas[0].cause_ns[7]; + frames(1, 87.0), frames(1, 91.0), frames(1, 96.0), frames(60, 7.0); + expect("three slow frames in a row", areas[0].hitch, 8); + expect("not a change of pace", paced_out, paced0); + expect("the episode's time", areas[0].cause_ns[7] - charged0, 274000000ull); + // a change to 30 fps from 140: a run of long frames longer than SM_EPISODE_MAX, a change of pace + frames(200, 7.0); + frames(300, 33.3); + expect("a change to 30 fps", areas[0].hitch, 8); + frames(200, 7.0); stutter_area("Kakariko"); // a load: what follows within SM_LOAD_NS is the load's frames(1, 60.0), frames(30, 7.0); expect("a spike right after a load (play)", areas[1].hitch, 0); expect("a spike right after a load (load)", areas[1].load_hitch, 1); + clock_ns += 10000000000ull; + frames(60, 7.0); + stutter_load(); // the texture pool's load of an area, without the game's + frames(1, 60.0), frames(30, 7.0); + expect("a spike right after the pool's load (play)", areas[1].hitch, 0); + expect("a spike right after the pool's load (load)", areas[1].load_hitch, 2); if (!logged) printf("FAIL nothing was logged\n"), fails++; if (fails) return 1; printf("stutter_hitch_check: all checks passed\n"); diff --git a/tests/stutter_map_check.py b/tests/stutter_map_check.py index acb688f9..93379abd 100644 --- a/tests/stutter_map_check.py +++ b/tests/stutter_map_check.py @@ -28,6 +28,7 @@ def main(): '"; %s %llu, %.1f ms avg"', '"), worst %.1f ms (frame %llu); over 25 ms %llu; after its "', '"loads %llu; draws that met a pipeline missing %llu, a shader %llu, %.1f ms on the replay thread"', 'stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:', '" (charged to %s)"', + '"stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:%s; an episode of %d frames to frame "', 'stutter map: %s %s/%s first met in %s: %llu draws met it missing, %llu its shader, %.1f ms on the ', '"replay thread; usable after %.0f ms at most (%llu times); %s%s"'): assert piece in src, f'runtime/stutter_map.c no longer has: {piece}' @@ -37,6 +38,10 @@ def main(): 'pipeline missing: vertex 1a2b3c4d/5e6f7a8b 23.9 ms (skipped)', '[host] ' + c_fmt('stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:', 1300, 80.0, 60.0, 7.0, 'Kakariko Village', ', after its load') + ' unattributed 55.0 (charged to unattributed)', + # an episode of three frames: its head carries their summed time + '[host] ' + c_fmt('stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:%s; an episode of %d ' + "frames to frame %llu, %.1f ms the first, the parts above the first frame's", 1400, 90.0, 274.0, + 7.0, 'Kakariko Village', '', ' fence 77.1 replay 4.0 (charged to fence)', 3, 1402, 87.0), '[host] ' + c_fmt('stutter map (%s): %.1f min of play in %d areas; %llu hitches in play and %llu ' 'after area loads; %llu long frames were a change of pace (the frames after them as long), not ' 'hitches; in play by cause:', 'exit', 12.5, 2, 3, 1, 7) + ' pipeline 2 texture 1', @@ -72,7 +77,8 @@ def main(): p = block['pipes'][0] assert p['v'] == '1a2b3c4d' and p['f'] == '5e6f7a8b' and p['ready'] == 160 and p['misses'] == 3, p assert p['state'].startswith('new to this install'), p - assert len(hitches) == 2 and hitches[0]['cause'] == 'pipeline' and hitches[1]['load'], hitches + assert len(hitches) == 3 and hitches[0]['cause'] == 'pipeline' and hitches[1]['load'], hitches + assert hitches[2]['ms'] == 274.0 and hitches[2]['cause'] == 'fence' and not hitches[2]['load'], hitches[2] out = Path(d) / 'map.md' stutter_map.report([log], out) text = out.read_text(encoding='utf-8')