diff --git a/nx2navi.ini.example b/nx2navi.ini.example index c1e86f92..ba58f1a0 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -1002,6 +1002,11 @@ # flight_flash = 1 # flight_flash_diff = 24 # flight_flash_cells = 3 +# Stutter map: the log gets a line for each hitch (a frame over 2.5 times the frames before it and at least 12 ms, or +# over 25 ms) with the area, what the frame waited for and the pipelines and shaders its draws met missing, and every 10 +# minutes and at exit a map of the areas by their hitches and of the pipelines that cost the most. tools/stutter_map.py +# reads it. 0: off. Read at start. +# stutter_map = 0 # GPU pass marks: after a GPU crash (device lost) the log names the render pass or dispatch the GPU had started and the # last one it finished. Unset: on AMD GPUs (VK_AMD_buffer_marker). 1: also on NVIDIA GPUs # (VK_NV_device_diagnostic_checkpoints), which NVIDIA says costs a little. 0: off. diff --git a/runtime/cfg.c b/runtime/cfg.c index 962c08e3..ff6f455e 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -45,7 +45,7 @@ static const char *const known[] = { "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DMABUF", "DMABUF_MB", "DRAW_CRUMBS", "GPA_LATE", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_CAM_KEEP", "DLAA_CUT", "DLSS_IDLE_FREE", "DLAA_EDGE_FILL", "DLAA_AT_UPSCALER", "FFX_DEBUG", "FFX_KEYS", "FSR3_SHARPEN", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TARGET_KEEP", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_PREBLOOM", "DLSS_RANGE", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "DUMP_RAW", "DUMP_SHADERS", "DUMP_FILE", "DS3", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "DUMP_MOVIE", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CRASH_FILTER_GUARD", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "GPU_TIMELINE", - "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLIGHT_FLASH", "FLIGHT_FLASH_DIFF", "FLIGHT_FLASH_CELLS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", + "FFX_PIPE_CACHE", "FLIGHT_SPIKE_MS", "STUTTER_MAP", "FLIGHT_THUMBS", "FLIGHT_EVERY", "FLIGHT_FLASH", "FLIGHT_FLASH_DIFF", "FLIGHT_FLASH_CELLS", "FLUSH_SPAN", "FLUSH_UPLOAD", "FOG_CULL", "FOLIAGE_SHADOW_CULL", "FOLIAGE_SHADOW_VRS", "FOV", "FOV_PATCH", "FG_HUDLESS_SHOW", "FG_MULT", "FPS60", "FPS_LIMIT", "FRAMEGEN", "FRAMEGEN_MULTIPLIER", "FREEZE_IDLE", "FREEZE_ASYNC", "FREEZE_BUDGET_US", "FREE_CORES", "FULLSCREEN", "FXAA", "GAME_ASPECT", "GAME_RES", "GPA", "GPA_EMPTY", "GPA_TWIN", "GPA_TWIN_MAX_KB", "MIRROR_CAPTURE_KB", "WB_SKIP_MB", "GPL", "GPL_DRAW_BUILD", "GPL_FEEDBACK", "GPL_MESH", "PIPE_AHEAD", "PIPE_FAST_WORKERS", "PIPE_FRAME_WAIT_MS", "PIPE_LINK_DRAW_MS", "PIPE_SKIP_STEADY", "DRAW_WRITERS_WAIT", "PROG_LEAD", "GPU", "GPU_INDEX", "GPU_PASSES", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_LIVE", "HEAP_LAZY", "HDR_PEAK_NITS", "HDR_UI_NITS", // retired: the HDR look's settings of earlier versions, now fixed (vk_hdr.c, shaders/hdr_tonemap.frag); read by @@ -156,7 +156,7 @@ static const struct { const char *key, *section; } sections[] = { {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"}, - {"DRAW_CRUMBS", "diagnostics"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, + {"DRAW_CRUMBS", "diagnostics"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"STUTTER_MAP", "diagnostics"}, {"FLIGHT_FLASH", "diagnostics"}, {"FLIGHT_FLASH_DIFF", "diagnostics"}, {"FLIGHT_FLASH_CELLS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"MESH_FALLBACK_WORKERS", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"MIRROR_CAPTURE_KB", "diagnostics"}, {"WB_SKIP_MB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, @@ -393,6 +393,8 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "", "0|1|mesh", RESTART}, // vk_pipeline.c noopt_flag_for; unset: mesh on AMD's driver for RDNA 1 and 2 // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, + // stutter_map.c: hitches by area and cause, and the pipelines and shaders draws met missing (0 off, 1 on) + {"STUTTER_MAP", "0", "0|1", RESTART}, // flight.c "The flash trigger": on or off, and flight_flash.h flash_test's two thresholds {"FLIGHT_FLASH", "1", "0|1", RESTART}, {"FLIGHT_FLASH_DIFF", "24", "4..255", RESTART}, {"FLIGHT_FLASH_CELLS", "3", "1..144", RESTART}, {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1|2|3", RESTART}, {"WB_LAZY", WB_LAZY_DEFAULT, "0|1", RESTART}, {"HEAP_LAZY", "0", "0|1", RESTART}, {"WB_LAZY_TOUCH", "0", "0..16", RESTART}, {"WB_LAZY_CHUNK", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, diff --git a/runtime/perf.c b/runtime/perf.c index 13951ec5..070cca87 100644 --- a/runtime/perf.c +++ b/runtime/perf.c @@ -12,6 +12,7 @@ #include "nvn_api.h" #include "perfmon.h" #include "prof.h" +#include "stutter_map.h" #include "platform/platform.h" volatile uint32_t prof_render_tid, prof_main_tid, prof_replay_tid; @@ -53,6 +54,7 @@ uint64_t perf_gpu_busy_total(void) { return (uint64_t)__atomic_load_n(&gpu_busy_ void prof_start(void) { flight_init(); + stutter_map_init(); atexit(perf_summary_flush); // boot.c's _exit path goes through prof_write instead perf_base = (rt_trace & RT_TRACE_PERF) || flight_on; perf_on = perf_base || overlay == 2; @@ -259,6 +261,7 @@ void perf_segment(const char *name) { } void perf_scene(const char *name) { + stutter_area(name); if (!(rt_trace & RT_TRACE_PERF)) return; char wb[24]; rt_log("scene: %s loads (frame %llu, world %s s)", name, @@ -581,6 +584,7 @@ void perf_summary_flush(void) { // at exit (prof_write) if (rt_trace & RT_TRACE_PERF) seg_close(__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), kern_now_ns(), 1); if (summary_t0) { ft_report(kern_now_ns(), "exit"); + stutter_report("exit"); gfx_exit_log(); } summary_t0 = 0; @@ -608,8 +612,11 @@ static uint64_t ft_frame(uint64_t frame, uint64_t now, uint64_t d) { if (d > 25000000ull) ft.over25++; if (d > 50000000ull) ft.over50++; if (d > 100000000ull) ft.over100++; - if (d > 25000000ull) { // which part it went to - uint64_t part[FT_C_N]; + // which part it went to: for a frame over 25 ms, and for the stutter map's hitches, which are 12 ms at least + // (stutter_map.c) + uint64_t part[FT_C_N] = {0}; + int best = FT_C_OTHER; + if (d > 25000000ull || (stutter_map_on && d >= 12000000ull)) { part[FT_C_PIPE] = frame_ns[PERF_PIPE]; part[FT_C_XLATE] = frame_ns[PERF_XLATE]; part[FT_C_TEX] = frame_ns[PERF_TEX]; @@ -621,10 +628,12 @@ static uint64_t ft_frame(uint64_t frame, uint64_t now, uint64_t d) { uint64_t known = 0; for (int c = 0; c < FT_C_OTHER; c++) known += part[c]; part[FT_C_OTHER] = d > known ? d - known : 0; - int best = FT_C_OTHER; for (int c = 0; c < FT_C_N; c++) if (part[c] > part[best]) best = c; if (part[best] * 4 < d) best = FT_C_OTHER; // no recorded part explains a quarter of the frame + } + stutter_frame(frame, now, d, part, ft_cause_name, FT_C_N, best); + if (d > 25000000ull) { ft.slow_n[best]++; ft.slow_ns[best] += d; if (best == FT_C_REPLAY) { diff --git a/runtime/stutter_map.c b/runtime/stutter_map.c new file mode 100644 index 00000000..0a000367 --- /dev/null +++ b/runtime/stutter_map.c @@ -0,0 +1,333 @@ +// The stutter map (NX2NAVI_STUTTER_MAP=1): what a play session's hitches were, area by area, and which pipelines and +// shaders the draws met missing, so a long session says where the remaining stutters are and what the next pipeline +// list (tools/pipeline_list.py merge) and shader preparation must take in. Off by default; while it is off every hook +// returns at its first test. +// +// A hitch is a frame whose interval is over 2.5 times the mean of the frames before it (a running mean of the frames +// that were not hitches), and at least 12 ms; any frame over 25 ms is one too. At 140 fps that is a frame over 17.5 ms, +// at 60 fps one over 25 ms. Frames within SM_LOAD_NS of an area's load count apart, as the load's: an area load +// shows as a few long frames that are not a stutter in play. +// +// What a hitch went to comes from the frame-time summary's split (perf.c ft_frame: pipeline, translation, texture, +// fence, pacing, replay, unattributed), with the draws that met a pipeline or a shader missing between the previous +// present and this one (stutter_miss, from the replay thread). Each hitch gets a "stutter:" line, at most +// SM_LINES_PER_S a second and SM_LINES_MAX in all; the counts go on past that. Every SM_REPORT_NS of play and at exit, +// "stutter map" lines rank the areas by their hitches and the pipelines by the time draws lost to them, each with +// whether this install had recorded it before (the pipeline list or an earlier run) and whether one of its shaders was +// translated in play rather than prepared. +// +// Threads: stutter_area runs on the game's loading threads, stutter_miss on the replay thread, stutter_ready on the +// pipeline workers (under vk_pipeline.c's P.lock; sm_lock is taken inside it and never takes it), stutter_frame and +// the periodic report on the present thread. The area table's play counts belong to the present thread; the rest is +// under sm_lock. +#include +#include +#include + +#include "cfg.h" +#include "host.h" +#include "stutter_map.h" +#include "platform/platform.h" + +int stutter_map_on; + +#define SM_AREAS 128 // areas kept by name; later ones share the last slot +#define SM_CAUSES 8 +#define SM_LOAD_NS 3000000000ull // after an area load, frames count as the load's for this long +#define SM_HITCH_MIN_NS 12000000.0 // a hitch is at least this long... +#define SM_HITCH_ANY_NS 25000000ull // ...and any frame over this is one +#define SM_HITCH_FACTOR 2.5 // times the running mean of the frames that were not hitches +#define SM_PIPES 4096 // pipelines kept for the ranking (open addressing) +#define SM_RING 256 // draws that met something missing, between two presents +#define SM_LINES_MAX 4000 +#define SM_LINES_PER_S 5 +#define SM_REPORT_NS 600000000000ull // the map every 10 minutes of play +#define SM_TOP_AREAS 40 +#define SM_TOP_PIPES 40 + +typedef struct { + char name[64]; + uint64_t ns, frames, over25; // present thread + uint64_t hitch, load_hitch, by_cause[SM_CAUSES], cause_ns[SM_CAUSES], worst_ns, worst_frame; + uint64_t misses, shader_misses, miss_ns; // draws that met a pipeline / a shader missing +} Area; + +typedef struct { + uint64_t key; // vertex (or mesh) and fragment input ids' first four bytes each; 0: not known + uint8_t used, mesh, recorded, fresh; + uint16_t area; // where draws first met it + uint64_t misses, shader_misses, wait_ns, ready_n; + double ready_max, ready_sum; +} Pipe; + +typedef struct { + uint64_t key, ns; + uint8_t shader, mesh, skipped, has_v, has_f; +} Miss; + +static PlatMutex sm_lock = PLAT_MUTEX_INIT; +static Area areas[SM_AREAS]; +static int n_areas = 1; // areas[0]: from the start to the first area load +static volatile int cur_area; +static volatile uint64_t load_at; // kern_now_ns of the last area load +static Pipe pipes[SM_PIPES]; +static uint32_t n_pipes, pipes_lost; +static Miss ring[SM_RING]; +static volatile uint64_t ring_head; // sm_lock: draws noted so far +static uint64_t ring_tail; // present thread: draws taken so far +static uint64_t ring_lost; + +// present thread +static double mean_ns; +static uint64_t t_first, t_report, line_sec, lines_in_sec, lines, frames_seen; + +void stutter_map_init(void) { + stutter_map_on = cfg_bool("NX2NAVI_STUTTER_MAP"); + if (!stutter_map_on) return; + snprintf(areas[0].name, sizeof areas[0].name, "%s", "(before the first area)"); + rt_log("stutter map: on (NX2NAVI_STUTTER_MAP): a hitch is a frame over %.1f times the frames before it and at " + "least %.0f ms, or over %llu ms; frames within %llu s of an area load count as the load's; the map every %llu " + "minutes and at exit", SM_HITCH_FACTOR, SM_HITCH_MIN_NS / 1e6, SM_HITCH_ANY_NS / 1000000ull, + SM_LOAD_NS / 1000000000ull, SM_REPORT_NS / 60000000000ull); +} + +static uint32_t id4(const uint8_t *id) { + return id ? (uint32_t)id[0] << 24 | (uint32_t)id[1] << 16 | (uint32_t)id[2] << 8 | id[3] : 0; +} + +static uint64_t key_of(const uint8_t *vi, const uint8_t *fi) { return (uint64_t)id4(vi) << 32 | id4(fi); } + +void stutter_area(const char *name) { + if (!stutter_map_on || !name || !*name) return; + plat_mutex_lock(&sm_lock); + int i = 0; + while (i < n_areas && strcmp(areas[i].name, name)) i++; + if (i == n_areas) { + if (n_areas < SM_AREAS) snprintf(areas[n_areas++].name, sizeof areas[0].name, "%s", name); + else { + i = SM_AREAS - 1; + snprintf(areas[i].name, sizeof areas[i].name, "%s", "(more areas)"); + } + } + __atomic_store_n(&cur_area, i, __ATOMIC_RELAXED); + __atomic_store_n(&load_at, kern_now_ns(), __ATOMIC_RELAXED); + plat_mutex_unlock(&sm_lock); +} + +// sm_lock held. The entry for key (mesh apart), made on first use; NULL once the table is full. +static Pipe *pipe_at(uint64_t key, int mesh) { + uint64_t h = (key ^ (uint64_t)mesh) * 0x9E3779B97F4A7C15ull; + for (uint32_t k = 0; k < 64; k++) { + Pipe *p = &pipes[((h >> 52) + k) & (SM_PIPES - 1)]; + if (p->used && p->key == key && p->mesh == mesh) return p; + if (!p->used) { + if (n_pipes >= SM_PIPES * 3 / 4) break; + memset(p, 0, sizeof *p); + p->used = 1, p->key = key, p->mesh = (uint8_t)mesh; + p->area = (uint16_t)__atomic_load_n(&cur_area, __ATOMIC_RELAXED); + n_pipes++; + return p; + } + } + pipes_lost++; + return NULL; +} + +void stutter_miss(int shader, int mesh, const uint8_t *vi, const uint8_t *fi, uint64_t ns, int skipped) { + if (!stutter_map_on) return; + uint64_t key = key_of(vi, fi); + plat_mutex_lock(&sm_lock); + uint64_t h = ring_head; + if (h - ring_tail >= SM_RING) ring_lost++; // the present has not taken the oldest yet: it is overwritten + Miss *m = &ring[h % SM_RING]; + m->key = key, m->ns = ns, m->shader = (uint8_t)(shader != 0), m->mesh = (uint8_t)(mesh != 0); + m->skipped = (uint8_t)(skipped != 0), m->has_v = vi != NULL, m->has_f = fi != NULL; + __atomic_store_n(&ring_head, h + 1, __ATOMIC_RELEASE); + Pipe *p = pipe_at(key, mesh != 0); + if (p) { + if (shader) p->shader_misses++; + else p->misses++; + p->wait_ns += ns; + } + plat_mutex_unlock(&sm_lock); +} + +void stutter_ready(int mesh, const uint8_t *vi, const uint8_t *fi, double ms, int recorded, int fresh, int fast) { + (void)fast; + if (!stutter_map_on) return; + plat_mutex_lock(&sm_lock); + Pipe *p = pipe_at(key_of(vi, fi), mesh != 0); + if (p) { + p->ready_n++, p->ready_sum += ms; + if (ms > p->ready_max) p->ready_max = ms; + p->recorded |= (uint8_t)(recorded != 0), p->fresh |= (uint8_t)(fresh != 0); + } + plat_mutex_unlock(&sm_lock); +} + +static void hex8(char *b, uint32_t v, int known) { + if (known) snprintf(b, 9, "%08x", v); + else snprintf(b, 9, "%s", "?"); +} + +// The draws noted since the previous present: counted for the area, and the first few named in line at *o. +static void take_misses(Area *a, char *line, int *o, size_t cap) { + uint64_t head = __atomic_load_n(&ring_head, __ATOMIC_ACQUIRE); + if (head == ring_tail) return; + plat_mutex_lock(&sm_lock); + head = ring_head; + if (head - ring_tail > SM_RING) ring_tail = head - SM_RING; + int named = 0; + uint64_t more = 0; + for (; ring_tail < head; ring_tail++) { + const Miss *m = &ring[ring_tail % SM_RING]; + if (m->shader) a->shader_misses++; + else a->misses++; + a->miss_ns += m->ns; + if (!line) continue; + if (named >= 4 || *o >= (int)cap - 120) { more++; continue; } + char v[9], f[9]; + hex8(v, (uint32_t)(m->key >> 32), m->has_v); + hex8(f, (uint32_t)m->key, m->has_f); + *o += snprintf(line + *o, cap - (size_t)*o, "%s %s %s %s/%s %.1f ms%s", named ? "," : "; draws that met", + m->shader ? "a shader missing:" : "a pipeline missing:", m->mesh ? "mesh" : "vertex", v, + m->has_f ? f : "none", m->ns / 1e6, m->skipped ? " (skipped)" : ""); + named++; + } + if (more && line && *o < (int)cap - 32) *o += snprintf(line + *o, cap - (size_t)*o, " and %llu more", (unsigned long long)more); + plat_mutex_unlock(&sm_lock); +} + +void stutter_frame(uint64_t frame, uint64_t now, uint64_t d, const uint64_t *part, const char *const *names, int n, + int cause) { + if (!stutter_map_on || !d) return; + if (!t_first) t_first = t_report = now; + int ai = __atomic_load_n(&cur_area, __ATOMIC_RELAXED); + Area *a = &areas[ai]; + a->ns += d, a->frames++; + a->over25 += d > SM_HITCH_ANY_NS; + frames_seen++; + double lim = mean_ns * SM_HITCH_FACTOR; + if (lim < SM_HITCH_MIN_NS) lim = SM_HITCH_MIN_NS; + int hitch = (frames_seen > 30 && (double)d > lim) || d > SM_HITCH_ANY_NS; + if (!hitch) { + mean_ns = mean_ns > 0 ? mean_ns + ((double)d - mean_ns) / 32.0 : (double)d; + take_misses(a, NULL, NULL, 0); + } else { + int load = now - __atomic_load_n(&load_at, __ATOMIC_RELAXED) < SM_LOAD_NS; + if (load) a->load_hitch++; + else { + a->hitch++; + if (cause >= 0 && cause < SM_CAUSES) a->by_cause[cause]++, a->cause_ns[cause] += d; + if (d > a->worst_ns) a->worst_ns = d, a->worst_frame = frame; + } + char line[1400]; + int o = 0; + uint64_t sec = (now - t_first) / 1000000000ull; + if (sec != line_sec) line_sec = sec, lines_in_sec = 0; + int say = lines < SM_LINES_MAX && lines_in_sec < SM_LINES_PER_S; + if (say) { + o = snprintf(line, sizeof line, "stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:", + (unsigned long long)frame, (now - t_first) / 1e9, d / 1e6, mean_ns / 1e6, a->name, + load ? ", after its load" : ""); + for (int c = 0; c < n && o < (int)sizeof line - 40; c++) + if (part[c] >= 500000) o += snprintf(line + o, sizeof line - (size_t)o, " %s %.1f", names[c], part[c] / 1e6); + if (cause >= 0 && cause < n && o < (int)sizeof line - 40) + o += snprintf(line + o, sizeof line - (size_t)o, " (charged to %s)", names[cause]); + } + take_misses(a, say ? line : NULL, &o, sizeof line); + if (say) { + rt_log("%s", line); + lines++, lines_in_sec++; + if (lines == SM_LINES_MAX) rt_log("stutter: the last such line of this run; the map's counts go on"); + } + } + if (now - t_report >= SM_REPORT_NS) { + t_report = now; + char tag[32]; + snprintf(tag, sizeof tag, "%.0f min", (now - t_first) / 6e10); + stutter_report(tag); + } +} + +static const char *const cause_short[SM_CAUSES] = {"pipeline", "translation", "texture", "fence", "pacing", "replay", + "unattributed", "?"}; + +static int area_cmp(const void *x, const void *y) { + const Area *a = &areas[*(const int *)x], *b = &areas[*(const int *)y]; + uint64_t ka = a->hitch + a->load_hitch, kb = b->hitch + b->load_hitch; + if (ka != kb) return ka < kb ? 1 : -1; + return a->ns < b->ns ? 1 : a->ns > b->ns ? -1 : 0; +} + +static int pipe_cmp(const void *x, const void *y) { + const Pipe *a = *(Pipe *const *)x, *b = *(Pipe *const *)y; + double ka = a->wait_ns / 1e6 + a->ready_max, kb = b->wait_ns / 1e6 + b->ready_max; + return ka < kb ? 1 : ka > kb ? -1 : 0; +} + +void stutter_report(const char *tag) { + if (!stutter_map_on) return; + plat_mutex_lock(&sm_lock); + uint64_t hitch = 0, load = 0, ns = 0, cause[SM_CAUSES] = {0}; + int order[SM_AREAS], na = n_areas; + for (int i = 0; i < na; i++) { + order[i] = i; + hitch += areas[i].hitch, load += areas[i].load_hitch, ns += areas[i].ns; + for (int c = 0; c < SM_CAUSES; c++) cause[c] += areas[i].by_cause[c]; + } + char line[1400]; + int o = snprintf(line, sizeof line, "stutter map (%s): %.1f min of play in %d areas; %llu hitches in play and %llu " + "after area loads; in play by cause:", tag, ns / 6e10, na, (unsigned long long)hitch, + (unsigned long long)load); + for (int c = 0; c < SM_CAUSES; c++) + if (cause[c]) o += snprintf(line + o, sizeof line - (size_t)o, " %s %llu", cause_short[c], (unsigned long long)cause[c]); + rt_log("%s", line); + qsort(order, (size_t)na, sizeof order[0], area_cmp); + for (int k = 0; k < na && k < SM_TOP_AREAS; k++) { + const Area *a = &areas[order[k]]; + if (!a->frames) continue; + o = snprintf(line, sizeof line, "stutter map: %s: %.1f min at %.0f fps; hitches %llu (%.2f a minute", + a->name, a->ns / 6e10, a->ns ? a->frames * 1e9 / a->ns : 0.0, (unsigned long long)a->hitch, + a->ns ? a->hitch * 6e10 / a->ns : 0.0); + for (int c = 0; c < SM_CAUSES; c++) + if (a->by_cause[c]) + o += snprintf(line + o, sizeof line - (size_t)o, "; %s %llu, %.1f ms avg", cause_short[c], + (unsigned long long)a->by_cause[c], a->cause_ns[c] / 1e6 / a->by_cause[c]); + o += snprintf(line + o, sizeof line - (size_t)o, "), worst %.1f ms (frame %llu); over 25 ms %llu; after its " + "loads %llu; draws that met a pipeline missing %llu, a shader %llu, %.1f ms on the replay thread", + a->worst_ns / 1e6, (unsigned long long)a->worst_frame, (unsigned long long)a->over25, + (unsigned long long)a->load_hitch, (unsigned long long)a->misses, + (unsigned long long)a->shader_misses, a->miss_ns / 1e6); + rt_log("%s", line); + } + static Pipe *list[SM_PIPES]; + uint32_t np = 0, nrec = 0, nnew = 0, nfresh = 0, nunknown = 0; + for (uint32_t i = 0; i < SM_PIPES; i++) + if (pipes[i].used) { + list[np++] = &pipes[i]; + if (!pipes[i].key) nunknown++; + else if (pipes[i].ready_n) nrec += pipes[i].recorded, nnew += !pipes[i].recorded; + nfresh += pipes[i].fresh; + } + qsort(list, np, sizeof list[0], pipe_cmp); + rt_log("stutter map: %u pipelines draws met missing: %u that this install had recorded before (its prebuild had not " + "come to them), %u new to it, %u with a shader translated in play, %u still translating when met; %u not " + "kept (table full), %llu draws not seen by a present (ring full); the %d that cost the most:", + np, nrec, nnew, nfresh, nunknown, pipes_lost, (unsigned long long)ring_lost, SM_TOP_PIPES); + for (uint32_t k = 0; k < np && k < SM_TOP_PIPES; k++) { + const Pipe *p = list[k]; + char v[9], f[9]; + hex8(v, (uint32_t)(p->key >> 32), p->key >> 32 != 0); + hex8(f, (uint32_t)p->key, 1); + rt_log("stutter map: %s %s/%s first met in %s: %llu draws met it missing, %llu its shader, %.1f ms on the " + "replay thread; usable after %.0f ms at most (%llu times); %s%s", + p->mesh ? "mesh" : "vertex", v, (uint32_t)p->key ? f : "none", areas[p->area].name, + (unsigned long long)p->misses, (unsigned long long)p->shader_misses, p->wait_ns / 1e6, p->ready_max, + (unsigned long long)p->ready_n, + !p->key ? "its shaders still translating" : !p->ready_n ? "not seen ready" : p->recorded ? "recorded before" + : "new to this install", + p->fresh ? ", a shader translated in play" : ""); + } + plat_mutex_unlock(&sm_lock); +} diff --git a/runtime/stutter_map.h b/runtime/stutter_map.h new file mode 100644 index 00000000..2e3b6e6e --- /dev/null +++ b/runtime/stutter_map.h @@ -0,0 +1,24 @@ +// The stutter map (NX2NAVI_STUTTER_MAP=1, stutter_map.c): for a play session, which frames hitched, in which area, what +// the frame waited for, and which pipelines and shaders draws met missing, so the next pipeline list and shader +// preparation can take them in. +#pragma once +#include + +extern int stutter_map_on; // set once at start (stutter_map_init); every hook below returns at once while it is 0 + +void stutter_map_init(void); +// An area's load (perf.c perf_scene, from romfs.c): hitches from here on are counted in that area, and those within +// SM_LOAD_NS of the load apart from play. +void stutter_area(const char *name); +// A draw met its pipeline (shader 0) or one of its shaders (shader 1) missing and the replay thread spent ns on it +// before it was bound or skipped (vk_exec_draw.c, beside wait_note). vi, fi: the vertex (or mesh) and fragment shaders' +// input ids, NULL when not known (still translating, or no fragment stage). +void stutter_miss(int shader, int mesh, const uint8_t *vi, const uint8_t *fi, uint64_t ns, int skipped); +// A pipeline a draw missed became usable ms after the miss (vk_pipeline.c ready_note): recorded 1 when pipeline-keys.bin +// (or the seed list) named it, fresh 1 when one of its shaders was translated in this session (not prepared). +void stutter_ready(int mesh, const uint8_t *vi, const uint8_t *fi, double ms, int recorded, int fresh, int fast); +// Every present (perf.c ft_frame): d the frame's interval, part the time it went to by cause (names[i], n of them; +// the last one is the part nothing recorded explains), cause the one the frame is charged to. +void stutter_frame(uint64_t frame, uint64_t now, uint64_t d, const uint64_t *part, const char *const *names, int n, + int cause); +void stutter_report(const char *tag); // the map so far; every 10 minutes of play and at exit diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index baaefb2b..f3406b2e 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -34,6 +34,7 @@ #include "vk_mesh_fb.h" #include "vk_scratch_lru.h" #include "platform/platform.h" +#include "stutter_map.h" #include "vk_dlaa.h" #include "mesh_emu_spirv.h" @@ -4535,6 +4536,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int if (fs_wait && (ms = pool_ms(pool, x0))) fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, ms); pool_charge(pool, x0); wait_note(WK_SHADER, x0, vs_wait || ts_wait || fs_wait); + if (stutter_map_on) // the shaders translated by now name it; one still translating is not known yet + stutter_miss(1, mesh && !fb, vs && !vs_wait ? gfx_shader_input_id(vs) : NULL, + fs && !fs_wait ? gfx_shader_input_id(fs) : NULL, kern_now_ns() - x0, vs_wait || ts_wait || fs_wait); if (vs_wait || ts_wait || fs_wait) return perf_count(PERF_CNT_SHADER_SKIP), draw_skipped(ex, mesh, SK_SHADER_PENDING); } @@ -4651,6 +4655,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int pool_charge(last, b0); } if (m0) wait_note(kind, m0, pending), draw_stats_tick(); + if (m0 && stutter_map_on) + stutter_miss(0, mesh && !fb, vs ? gfx_shader_input_id(vs) : NULL, fs ? gfx_shader_input_id(fs) : NULL, + kern_now_ns() - m0, pending); cx->PM.valid = !pending && p && info; if (cx->PM.valid) cx->PM.s = s, cx->PM.info = info; } diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 97c195a2..3f9dbf03 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -52,6 +52,7 @@ #include "platform/platform.h" #include "prp_file.h" #include "spirv_water.h" +#include "stutter_map.h" #include "tex.h" #include "vk_build_gate.h" #include "vk_cache_file.h" @@ -4721,6 +4722,8 @@ static void ready_note(Pipe *p, int fast) { int fresh = gfx_shader_origin(j->s.vs) == 2 || (j->s.fs && gfx_shader_origin(j->s.fs) == 2) || (j->s.ts && gfx_shader_origin(j->s.ts) == 2); P.g.ready_mesh += (uint64_t)mesh, P.g.ready_recorded += j->prebuild, P.g.ready_fresh += (uint64_t)fresh; + stutter_ready(mesh, gfx_shader_input_id(j->s.vs), j->s.fs ? gfx_shader_input_id(j->s.fs) : NULL, ms, j->prebuild, + fresh, fast); if (mesh) { P.g.m_ready_fast += fast != 0, P.g.m_ready_ms += ms; if (ms > P.g.m_ready_max) P.g.m_ready_max = ms; diff --git a/tests/stutter_map_check.py b/tests/stutter_map_check.py new file mode 100644 index 00000000..22f025b4 --- /dev/null +++ b/tests/stutter_map_check.py @@ -0,0 +1,83 @@ +"""tools/stutter_map.py reads the lines runtime/stutter_map.c writes: lines made with the runtime's format strings +(copied below; keep them in step with stutter_map_init's neighbours stutter_frame and stutter_report) parse into the +areas, causes and pipelines they hold. + + python tests/stutter_map_check.py +""" +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'tools')) +import stutter_map # noqa: E402 + +C_SOURCE = Path(__file__).resolve().parent.parent / 'runtime' / 'stutter_map.c' + + +def c_fmt(fmt, *args): + """A C printf format with %llu, %u and %d as Python's %d.""" + return fmt.replace('%llu', '%d').replace('%u', '%d') % args + + +def main(): + src = C_SOURCE.read_text(encoding='utf-8') + # the format strings the parser depends on must still be in the runtime + for piece in ('stutter map (%s): %.1f min of play in %d areas; %llu hitches in play and %llu ', + '"after area loads; in play by cause:"', 'stutter map: %s: %.1f min at %.0f fps; hitches %llu (%.2f a minute', + '"; %s %llu, %.1f ms avg"', '"), worst %.1f ms (frame %llu); over 25 ms %llu; after its "', + '"loads %llu; draws that met a pipeline missing %llu, a shader %llu, %.1f ms on the replay thread"', + 'stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:', '" (charged to %s)"', + 'stutter map: %s %s/%s first met in %s: %llu draws met it missing, %llu its shader, %.1f ms on the ', + '"replay thread; usable after %.0f ms at most (%llu times); %s%s"'): + assert piece in src, f'runtime/stutter_map.c no longer has: {piece}' + lines = [ + '[host] ' + c_fmt('stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:', 1200, 75.2, 31.5, 7.1, + 'Kakariko Village', '') + ' pipeline 24.0 replay 3.1 (charged to pipeline); draws that met a ' + 'pipeline missing: vertex 1a2b3c4d/5e6f7a8b 23.9 ms (skipped)', + '[host] ' + c_fmt('stutter: frame %llu at %.1f s, %.1f ms (frames about %.1f ms) in %s%s:', 1300, 80.0, 60.0, 7.0, + 'Kakariko Village', ', after its load') + ' unattributed 55.0 (charged to unattributed)', + '[host] ' + c_fmt('stutter map (%s): %.1f min of play in %d areas; %llu hitches in play and %llu ' + 'after area loads; in play by cause:', 'exit', 12.5, 2, 3, 1) + ' pipeline 2 texture 1', + '[host] ' + c_fmt('stutter map: %s: %.1f min at %.0f fps; hitches %llu (%.2f a minute', 'Kakariko Village', 10.0, + 139.6, 3, 0.3) + + c_fmt('; %s %llu, %.1f ms avg', 'pipeline', 2, 30.2) + c_fmt('; %s %llu, %.1f ms avg', 'texture', 1, 19.0) + + c_fmt('), worst %.1f ms (frame %llu); over 25 ms %llu; after its ' + 'loads %llu; draws that met a pipeline missing %llu, a shader %llu, %.1f ms on the replay thread', + 31.5, 1200, 1, 1, 4, 1, 40.2), + '[host] ' + c_fmt('stutter map: %s: %.1f min at %.0f fps; hitches %llu (%.2f a minute', '(before the first area)', + 2.5, 120.0, 0, 0.0) + + c_fmt('), worst %.1f ms (frame %llu); over 25 ms %llu; after its ' + 'loads %llu; draws that met a pipeline missing %llu, a shader %llu, %.1f ms on the replay thread', + 0.0, 0, 0, 0, 0, 0, 0.0), + '[host] stutter map: 2 pipelines draws met missing: 1 that this install had recorded before (its prebuild had ' + 'not come to them), 1 new to it, 1 with a shader translated in play, 0 still translating when met; 0 not kept ' + '(table full), 0 draws not seen by a present (ring full); the 40 that cost the most:', + '[host] ' + c_fmt('stutter map: %s %s/%s first met in %s: %llu draws met it missing, %llu its shader, %.1f ms on ' + 'the replay thread; usable after %.0f ms at most (%llu times); %s%s', 'vertex', '1a2b3c4d', + '5e6f7a8b', 'Kakariko Village', 3, 1, 30.0, 160.0, 1, 'new to this install', + ', a shader translated in play'), + ] + with tempfile.TemporaryDirectory() as d: + log = Path(d) / 'nx2navi.log' + log.write_text('\n'.join(lines) + '\n', encoding='utf-8') + block, hitches = stutter_map.read_log(log) + assert block and block['head'][0] == 'exit' and block['head'][3] == '3', block and block['head'] + a = block['areas'][0] + assert a['name'] == 'Kakariko Village' and a['hitch'] == 3 and a['causes'] == {'pipeline': 2, 'texture': 1}, a + assert a['pmiss'] == 4 and a['smiss'] == 1 and a['load'] == 1 and a['over25'] == 1, a + assert block['areas'][1]['causes'] == {}, block['areas'][1] + assert block['sum'] == (2, 1, 1, 1, 0), block['sum'] + p = block['pipes'][0] + assert p['v'] == '1a2b3c4d' and p['f'] == '5e6f7a8b' and p['ready'] == 160 and p['misses'] == 3, p + assert p['state'].startswith('new to this install'), p + assert len(hitches) == 2 and hitches[0]['cause'] == 'pipeline' and hitches[1]['load'], hitches + out = Path(d) / 'map.md' + stutter_map.report([log], out) + text = out.read_text(encoding='utf-8') + assert '**3 hitches in play**' in text and '| Kakariko Village | 10.0 | 140 |' in text, text + print('stutter_map_check: ok') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/stutter_map.py b/tools/stutter_map.py new file mode 100644 index 00000000..ae68780c --- /dev/null +++ b/tools/stutter_map.py @@ -0,0 +1,181 @@ +"""The stutter map of a play session, from its log (runtime/stutter_map.c, NX2NAVI_STUTTER_MAP=1), and what a game +folder recorded that a pipeline list lacks. + + python tools/stutter_map.py report LOG [LOG ...] [--out FILE.md] + python tools/stutter_map.py missing GAME_DIR LIST_FILE + +report: each log's last "stutter map" block (the one written at exit, or the last periodic one when the game did not +exit cleanly) gives the areas ranked by their hitches with the causes, and the pipelines draws met missing that cost +the most; the "stutter:" lines add how the hitches spread over the session. Several logs (one session's rotated files, +or several sessions) are reported one after the other, then summed by area. + +missing: the pipeline records of GAME_DIR's pipeline-keys.bin and the shader variants of its shader-answers.bin that +LIST_FILE (tools/pipeline_list.py merge) does not hold: what `pipeline_list.py merge NEW LIST_FILE GAME_DIR` would +add, so the next installs build them before play instead of meeting them in it. The files are derived from game data; +the list stays out of the repository. +""" +import argparse +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pipeline_list # noqa: E402 + +HEAD = re.compile(r'stutter map \(([^)]*)\): ([\d.]+) min of play in (\d+) areas; (\d+) hitches in play and (\d+) after ' + r'area loads; in play by cause:(.*)$') +AREA = re.compile(r'stutter map: (.+?): ([\d.]+) min at (\d+) fps; hitches (\d+) \(([\d.]+) a minute(.*?)\), worst ' + r'([\d.]+) ms \(frame (\d+)\); over 25 ms (\d+); after its loads (\d+); draws that met a pipeline ' + r'missing (\d+), a shader (\d+), ([\d.]+) ms on the replay thread') +CAUSE = re.compile(r'; (\w+) (\d+), ([\d.]+) ms avg') +PIPES = re.compile(r'stutter map: (\d+) pipelines draws met missing: (\d+) that this install had recorded before .*?, ' + r'(\d+) new to it, (\d+) with a shader translated in play, (\d+) still translating when met') +PIPE = re.compile(r'stutter map: (mesh|vertex) (\S+)/(\S+) first met in (.+?): (\d+) draws met it missing, (\d+) its ' + r'shader, ([\d.]+) ms on the replay thread; usable after (\d+) ms at most \((\d+) times\); (.*)$') +HITCH = re.compile(r'stutter: frame (\d+) at ([\d.]+) s, ([\d.]+) ms \(frames about ([\d.]+) ms\) in (.+?)(, after its ' + r'load)?: .*?\(charged to (\w+)\)') + + +def read_log(path): + """(the last map block as (head, areas, pipes summary, pipes), hitch lines) of one log.""" + block, cur, hitches = None, None, [] + for line in open(path, encoding='utf-8', errors='replace'): + m = HEAD.search(line) + if m: + cur = {'head': m.groups(), 'areas': [], 'sum': None, 'pipes': []} + block = cur + continue + if cur is not None: + m = AREA.search(line) + if m: + g = m.groups() + cur['areas'].append({'name': g[0], 'min': float(g[1]), 'fps': int(g[2]), 'hitch': int(g[3]), + 'causes': {c: int(n) for c, n, _ in CAUSE.findall(g[5])}, 'worst': float(g[6]), + 'over25': int(g[8]), 'load': int(g[9]), 'pmiss': int(g[10]), 'smiss': int(g[11]), + 'miss_ms': float(g[12])}) + continue + m = PIPES.search(line) + if m: + cur['sum'] = tuple(int(x) for x in m.groups()) + continue + m = PIPE.search(line) + if m: + g = m.groups() + cur['pipes'].append({'kind': g[0], 'v': g[1], 'f': g[2], 'area': g[3], 'misses': int(g[4]), + 'smiss': int(g[5]), 'ms': float(g[6]), 'ready': int(g[7]), 'state': g[9]}) + continue + m = HITCH.search(line) + if m: + g = m.groups() + hitches.append({'frame': int(g[0]), 't': float(g[1]), 'ms': float(g[2]), 'area': g[4], 'load': bool(g[5]), + 'cause': g[6]}) + return block, hitches + + +def report(logs, out): + lines = ['# Stutter map', ''] + total = {} + for p in logs: + block, hitches = read_log(p) + lines += [f'## {p}', ''] + if not block: + lines += ['No stutter map in this log (stutter_map = 1 in nx2navi.ini turns it on).', ''] + continue + tag, mins, nareas, hitch, load, causes = block['head'] + lines += [f'{float(mins):.1f} min of play in {nareas} areas ({tag}): **{hitch} hitches in play**, {load} right ' + f'after area loads. In play by cause:{causes}.', ''] + lines += ['| Area | Minutes | fps | Hitches | a minute | Causes | Worst ms | Pipelines met missing | Shaders met ' + 'missing |', '|---|---|---|---|---|---|---|---|---|'] + for a in block['areas']: + causes_txt = ', '.join(f'{c} {n}' for c, n in sorted(a['causes'].items(), key=lambda kv: -kv[1])) + lines.append(f"| {a['name']} | {a['min']:.1f} | {a['fps']} | {a['hitch']} | " + f"{a['hitch'] / a['min'] if a['min'] else 0:.2f} | {causes_txt} | {a['worst']:.1f} | " + f"{a['pmiss']} | {a['smiss']} |") + t = total.setdefault(a['name'], {'min': 0.0, 'hitch': 0, 'causes': {}, 'pmiss': 0, 'smiss': 0}) + for k in ('min', 'hitch', 'pmiss', 'smiss'): + t[k] += a[k] + for c, n in a['causes'].items(): + t['causes'][c] = t['causes'].get(c, 0) + n + lines.append('') + if block['sum']: + n, rec, new, fresh, unk = block['sum'] + lines += [f'{n} pipelines draws met missing: {new} new to this install (no list or earlier run named them), ' + f'{rec} recorded but not built yet when met, {fresh} with a shader translated in play, {unk} ' + 'met while their shaders were still translating.', ''] + if block['pipes']: + lines += ['| Pipeline | First met in | Draws met it missing | Its shader | ms on the replay thread | Usable ' + 'after ms | State |', '|---|---|---|---|---|---|---|'] + for q in block['pipes']: + lines.append(f"| {q['kind']} {q['v']}/{q['f']} | {q['area']} | {q['misses']} | {q['smiss']} | " + f"{q['ms']:.1f} | {q['ready']} | {q['state']} |") + lines.append('') + if hitches: + play = [h for h in hitches if not h['load']] + by_min = {} + for h in play: + by_min[int(h['t'] // 60)] = by_min.get(int(h['t'] // 60), 0) + 1 + busiest = sorted(by_min.items(), key=lambda kv: -kv[1])[:5] + lines += [f'{len(hitches)} hitch lines ({len(play)} in play). Busiest minutes of the session: ' + + ', '.join(f'minute {m} ({n})' for m, n in busiest) + '.', ''] + if len(logs) > 1 and total: + lines += ['## All logs, by area', '', '| Area | Minutes | Hitches | a minute | Causes | Pipelines met missing | ' + 'Shaders met missing |', '|---|---|---|---|---|---|---|'] + for name, t in sorted(total.items(), key=lambda kv: -kv[1]['hitch']): + causes_txt = ', '.join(f'{c} {n}' for c, n in sorted(t['causes'].items(), key=lambda kv: -kv[1])) + lines.append(f"| {name} | {t['min']:.1f} | {t['hitch']} | {t['hitch'] / t['min'] if t['min'] else 0:.2f} " + f"| {causes_txt} | {t['pmiss']} | {t['smiss']} |") + lines.append('') + text = '\n'.join(lines) + if out: + Path(out).write_text(text, encoding='utf-8') + print(f'wrote {out}') + else: + print(text) + return 0 + + +def missing(game_dir, list_file): + (magic, version, rec, recs), ans = pipeline_list.read_dir(game_dir) + got = pipeline_list.unpack(Path(list_file).read_bytes()) + if got is None: + print(f'{list_file}: not a pipeline list') + return 1 + l_ans, l_keys = got + lmagic, lversion, lrec, lrecs = pipeline_list.read_keys(l_keys) + if recs and (magic, version, rec) != (lmagic, lversion, lrec): + print(f'{game_dir}: its pipeline-keys.bin has another layout than the list ({magic!r} v{version} {rec} bytes; ' + f'the list {lmagic!r} v{lversion} {lrec} bytes): a merge leaves it out') + return 1 + have = set(lrecs) + new = [r for r in recs if r not in have] + stage = {a[36:68]: a[0:4] for a in ans} + mesh = sum(1 for r in new if stage.get(r[:pipeline_list.ID]) == (pipeline_list.STAGE_MESH).to_bytes(4, 'little')) + l_ids = {a[36:68] for a in l_ans} + new_ans = [a for a in ans if a[36:68] not in l_ids] + by = {} + for a in new_ans: + s = pipeline_list.STAGE_NAMES.get(int.from_bytes(a[0:4], 'little'), 'other') + by[s] = by.get(s, 0) + 1 + print(f'{game_dir}: {len(recs)} pipelines recorded, {len(new)} of them not in {Path(list_file).name} ({mesh} mesh); ' + f'{len(ans)} shader variants recorded, {len(new_ans)} not in the list (' + + ', '.join(f'{n} {s}' for s, n in sorted(by.items())) + ')') + return 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = ap.add_subparsers(dest='cmd', required=True) + r = sub.add_parser('report', help='the stutter map of one or more logs, as Markdown') + r.add_argument('logs', nargs='+', type=Path) + r.add_argument('--out', type=Path) + m = sub.add_parser('missing', help='what a game folder recorded that a pipeline list lacks') + m.add_argument('game_dir', type=Path) + m.add_argument('list_file', type=Path) + a = ap.parse_args() + if a.cmd == 'report': + return report(a.logs, a.out) + return missing(a.game_dir, a.list_file) + + +if __name__ == '__main__': + sys.exit(main())