diff --git a/runtime/host.h b/runtime/host.h index 39ad79e4..88ad5190 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -48,6 +48,15 @@ enum { PERF_CNT_PIPE_ASYNC, PERF_CNT_PIPE_SKIP, PERF_CNT_TEX_UP, PERF_CNT_TEX_RE PERF_CNT_PIPE_WAIT, PERF_CNT_ONESHOT_WAIT, PERF_CNT_PIPE_BUILT, PERF_CNT_TEX_NEW, PERF_CNT_TEX_ALIAS, PERF_CNT_WB, PERF_CNT_WB_SKIP, PERF_CNT_N }; void perf_count(int kind); +// NX2NAVI_TRACE=perf: work on any thread that a frame can wait for besides the render thread's waits above, summed per +// frame and per 60-frame window (perf.c perf_mark_add): RomFS reads, host zstd decodes, pipeline creates (vk_pipeline.c +// pipe_create_in), guest memory maps (heap size, MapPhysicalMemory), NVN memory pool initialize and finalize +// (vk_memory.c vk_pool_init, vk_pool_fini), and device memory VMA allocates (blocks and dedicated memory: count and +// bytes, no time). perf_mark_begin returns 0 while the trace is off, so a normal run pays one branch per site. +enum { PERF_M_FILE, PERF_M_ZSTD, PERF_M_PIPE, PERF_M_GMEM, PERF_M_POOL, PERF_M_DEVMEM, PERF_M_N }; +uint64_t perf_mark_begin(void); +void perf_mark_end(int kind, uint64_t t0, uint64_t bytes); // nothing when t0 is 0 +void perf_mark_add(int kind, uint64_t ns, uint64_t bytes); // nothing while the trace is off // NX2NAVI_REPLAY_TIME=1: how long the replay spends in each kind of op (perf.c perf_replay_op reports it with the // frame-time summary). Draws and dispatches include their bindings and the vkCmd calls they record; binds are the state // and resource binds between them; xfer is copies and clears. diff --git a/runtime/memory.c b/runtime/memory.c index 1f31ee91..3703e792 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -895,6 +895,7 @@ uint32_t mem_map_gen(void) { return __atomic_load_n(&map_gen, __ATOMIC_ACQUIRE); // ---- 0x01 SetHeapSize uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { + uint64_t t0 = perf_mark_begin(); // NX2NAVI_TRACE=perf: the commit or decommit, the wait for the lock included plat_mutex_lock(&lock); if (size > HEAP_CAP || size % 0x200000 || size + image_bytes() > L.pool) { plat_mutex_unlock(&lock); @@ -902,9 +903,11 @@ uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { } if (size > heap_size) commit(L.heap.base + heap_size, size - heap_size, "heap"); if (size < heap_size) decommit(L.heap.base + size, heap_size - size, "heap"); + uint64_t moved = size > heap_size ? size - heap_size : heap_size - size; heap_size = size; __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); + perf_mark_end(PERF_M_GMEM, t0, moved); if (addr) *addr = L.heap.base; return R_OK; } @@ -1061,7 +1064,7 @@ static uint32_t alias_map(int unmap, uint64_t a, uint64_t n) { n = (n + PAGE - 1) & ~(uint64_t)(PAGE - 1); // the SDK asks for byte ranges; a mapping is whole pages if ((a | n) & (PAGE - 1)) return R_INVALID_ADDRESS; if (!inside(a, n, L.alias)) return R_INVALID_REGION; - uint64_t first = (a - L.alias.base) / PAGE, count = n / PAGE; + uint64_t first = (a - L.alias.base) / PAGE, count = n / PAGE, t0 = perf_mark_begin(); plat_mutex_lock(&lock); uint32_t r = R_OK; if (unmap) { @@ -1087,6 +1090,7 @@ static uint32_t alias_map(int unmap, uint64_t a, uint64_t n) { out: __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); + perf_mark_end(PERF_M_GMEM, t0, n); if (r) rt_log("%sMapPhysicalMemory %#llx+%#llx failed %#x (alias %#llx used, heap %#llx)", unmap ? "Un" : "", (unsigned long long)a, (unsigned long long)n, r, (unsigned long long)alias_bytes, (unsigned long long)heap_size); diff --git a/runtime/ov_zstd.c b/runtime/ov_zstd.c index c5853150..c7d0ca3d 100644 --- a/runtime/ov_zstd.c +++ b/runtime/ov_zstd.c @@ -152,7 +152,9 @@ static size_t tracked_decode(Cpu *c, uint64_t dst, uint64_t cap, uint64_t src, u if (d) __atomic_store_n(d, *d + 1, __ATOMIC_RELAXED); int away = size >= AWAY_BYTES ? kern_core_away() : 0; if (away) __atomic_add_fetch(&S.away, 1, __ATOMIC_RELAXED); + uint64_t t0 = perf_mark_begin(); size_t r = host_decode(dst, cap, src, size, dict, dict_size, ddict, ok); + perf_mark_end(PERF_M_ZSTD, t0, *ok && !ZSTD_isError(r) ? r : 0); kern_core_back(&away); if (d) { int n = *d - 1; @@ -372,12 +374,14 @@ static size_t stream_run(Cpu *c, Stream *st, ZSTD_outBuffer *out, ZSTD_inBuffer if (d) __atomic_store_n(d, *d + 1, __ATOMIC_RELAXED); int away = in->size >= AWAY_BYTES ? kern_core_away() : 0; if (away) __atomic_add_fetch(&S.away, 1, __ATOMIC_RELAXED); - size_t r = 1; + size_t r = 1, o_start = out->pos; + uint64_t t0 = perf_mark_begin(); while (in->pos < in->size) { size_t i0 = in->pos, o0 = out->pos; r = ZSTD_decompressStream(st->dctx, out, in); if (ZSTD_isError(r) || !r || (in->pos == i0 && out->pos == o0)) break; } + perf_mark_end(PERF_M_ZSTD, t0, out->pos - o_start); kern_core_back(&away); if (d) { int n = *d - 1; diff --git a/runtime/perf.c b/runtime/perf.c index f801ca97..7f370c8a 100644 --- a/runtime/perf.c +++ b/runtime/perf.c @@ -118,6 +118,56 @@ void perf_count(int kind) { if (__atomic_load_n(&perf_on, __ATOMIC_RELAXED)) plat_inc64(&counts[kind]); } +// ---- NX2NAVI_TRACE=perf work marks (host.h PERF_M_*): any thread adds; the present takes the frame's sums (mark_frame), +// names them on a slow frame's line and adds them to the 60-frame window's line. +static const char *const mark_name[PERF_M_N] = {"file reads", "zstd decodes", "pipeline creates", "guest memory maps", + "pool init/finalize", "device memory allocations"}; +static volatile int64_t mk_ns[PERF_M_N], mk_n[PERF_M_N], mk_bytes[PERF_M_N], mk_max[PERF_M_N]; // since the last present +typedef struct { uint64_t ns[PERF_M_N], n[PERF_M_N], bytes[PERF_M_N], max[PERF_M_N]; } MarkSums; +static MarkSums mk_win; // the 60-frame window's, present thread only + +uint64_t perf_mark_begin(void) { return (rt_trace & RT_TRACE_PERF) ? kern_now_ns() : 0; } + +void perf_mark_add(int kind, uint64_t ns, uint64_t bytes) { + if (!(rt_trace & RT_TRACE_PERF) || (unsigned)kind >= PERF_M_N) return; + plat_add64(&mk_ns[kind], (int64_t)ns); + plat_inc64(&mk_n[kind]); + plat_add64(&mk_bytes[kind], (int64_t)bytes); + int64_t m = __atomic_load_n(&mk_max[kind], __ATOMIC_RELAXED); + while ((int64_t)ns > m && !__atomic_compare_exchange_n(&mk_max[kind], &m, (int64_t)ns, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} +} + +void perf_mark_end(int kind, uint64_t t0, uint64_t bytes) { + if (t0) perf_mark_add(kind, kern_now_ns() - t0, bytes); +} + +// Takes the sums since the last present into *f and adds them to the window. +static void mark_frame(MarkSums *f) { + for (int k = 0; k < PERF_M_N; k++) { + f->ns[k] = (uint64_t)plat_xchg64(&mk_ns[k], 0), f->n[k] = (uint64_t)plat_xchg64(&mk_n[k], 0); + f->bytes[k] = (uint64_t)plat_xchg64(&mk_bytes[k], 0), f->max[k] = (uint64_t)plat_xchg64(&mk_max[k], 0); + mk_win.ns[k] += f->ns[k], mk_win.n[k] += f->n[k], mk_win.bytes[k] += f->bytes[k]; + if (f->max[k] > mk_win.max[k]) mk_win.max[k] = f->max[k]; + } +} + +// "; on any thread: file reads 12.3 ms (40, 5.1 MiB, longest 2.0 ms), ..." for the kinds with work, appended at line + n. +// Device memory allocations carry no time. +static int mark_text(const MarkSums *s, char *line, int n, size_t cap) { + int any = 0; + for (int k = 0; k < PERF_M_N && n >= 0 && n < (int)cap - 100; k++) { + if (!s->n[k]) continue; + n += snprintf(line + n, cap - (size_t)n, "%s %s", any ? "," : "; on any thread:", mark_name[k]); + if (k == PERF_M_DEVMEM) + n += snprintf(line + n, cap - (size_t)n, " %llu (%.1f MiB)", (unsigned long long)s->n[k], s->bytes[k] / 1048576.0); + else + n += snprintf(line + n, cap - (size_t)n, " %.1f ms (%llu, %.1f MiB, longest %.1f ms)", s->ns[k] / 1e6, + (unsigned long long)s->n[k], s->bytes[k] / 1048576.0, s->max[k] / 1e6); + any = 1; + } + return n; +} + // ---- Frame-time summary, always on: every 60 s of play and at exit one line of frame count, mean fps, 1% low and the // frames over 25/50/100 ms, then the worst frames of the period with the render thread's waits in them. Per frame: a // histogram increment, the compare against the worst list, a memset of the wait array. No allocation. @@ -586,15 +636,21 @@ void perf_present(uint64_t frame) { const char *e = cfg_get("NX2NAVI_PERF_SLOW_MS"); slow_ns = (uint64_t)(e && atoi(e) > 0 ? atoi(e) : 50) * 1000000ull; } + MarkSums fm; + mark_frame(&fm); if (last) { sum += d; n++; if (d > max) max = d, memcpy(worst_ns, frame_ns, sizeof worst_ns); - if (d >= slow_ns) - rt_log("perf: slow frame %llu: %.1f ms; blocked pipeline %.1f xlate %.1f texture %.1f astc %.1f fence %.1f pacing %.1f", + if (d >= slow_ns) { + char marks[700]; + marks[0] = 0; + mark_text(&fm, marks, 0, sizeof marks); + rt_log("perf: slow frame %llu: %.1f ms; blocked pipeline %.1f xlate %.1f texture %.1f astc %.1f fence %.1f pacing %.1f%s", (unsigned long long)frame, d / 1e6, frame_ns[PERF_PIPE] / 1e6, frame_ns[PERF_XLATE] / 1e6, frame_ns[PERF_TEX] / 1e6, frame_ns[PERF_ASTC] / 1e6, frame_ns[PERF_FENCE] / 1e6, - frame_ns[PERF_PACE] / 1e6); + frame_ns[PERF_PACE] / 1e6, marks); + } } memset(frame_ns, 0, sizeof frame_ns); last = now; @@ -629,6 +685,9 @@ void perf_present(uint64_t frame) { plat_xchg64(&replay_ns, 0) / 1e6 / n, (unsigned long long)ab_b, (unsigned long long)n, frame_step_world_us() / 1e6, (unsigned long long)dwin[0], dwin[1] / 1e6, (unsigned long long)dwin[2]); if (n && span) perf_gpu_line(frame, sum / 1e6 / n, (double)wait_ns[PERF_PACE] / span, n); + char marks[700]; + if (mark_text(&mk_win, marks, 0, sizeof marks) > 0) rt_log("perf: frame %llu: work in the window%s", (unsigned long long)frame, marks); + memset(&mk_win, 0, sizeof mk_win); sum = max = n = ab_b = 0; memset(wait_ns, 0, sizeof wait_ns); win_start = now; diff --git a/runtime/romfs.c b/runtime/romfs.c index 5316a9c1..7441f9b0 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -1222,7 +1222,7 @@ uint32_t romfs_read(uint64_t *out, uint64_t h, int64_t offset, void *buffer, uin uint64_t base, fsize; if (!out || offset < 0 || !handle_source(h, &f, &base, &fsize)) return FS_R_INVALID; if ((uint64_t)offset > fsize || (!buffer && size)) return FS_R_INVALID; - uint64_t n = size < fsize - (uint64_t)offset ? size : fsize - (uint64_t)offset, done = 0; + uint64_t n = size < fsize - (uint64_t)offset ? size : fsize - (uint64_t)offset, done = 0, t0 = perf_mark_begin(); if (f == image && image_map) { memcpy(buffer, image_map + base + (uint64_t)offset, (size_t)n); done = n; @@ -1233,6 +1233,7 @@ uint32_t romfs_read(uint64_t *out, uint64_t h, int64_t offset, void *buffer, uin if (r <= 0) rt_fatal("RomFS read failure at %#llx", (unsigned long long)(base + offset + done)); done += (uint64_t)r; } + perf_mark_end(PERF_M_FILE, t0, done); *out = done; return 0; } diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c index 811d7cfd..0ae9072f 100644 --- a/runtime/vk_headroom.c +++ b/runtime/vk_headroom.c @@ -83,6 +83,7 @@ uint64_t vk_headroom_held(void) { return (uint64_t)__atomic_load_n(&H.held, __AT static void VKAPI_PTR on_allocate(VmaAllocator a, uint32_t type, VkDeviceMemory m, VkDeviceSize size, void *user) { (void)a, (void)m, (void)size, (void)user; + perf_mark_add(PERF_M_DEVMEM, 0, size); // NX2NAVI_TRACE=perf: every block and dedicated allocation VMA makes if (!H.on || type >= 64) return; if (plat_thread_id() == H.tid) { plat_inc64(&H.made); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 1c17fd59..39de4681 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -271,7 +271,7 @@ static void destroy_rec(VkCtx *c, PoolRec *r) { free(r); } -int vk_pool_init(pool_t *p) { +static int pool_init(pool_t *p) { VkCtx *c = vk_ctx(); if (!c) { rt_log("vk: MemoryPoolInitialize before vk_init"); return 0; } // The GPU never sees a GPU_NO_ACCESS pool, except a physical one (0x189; 1.4.3 also 0x18c, CPU-visible) through the @@ -346,7 +346,7 @@ static uint32_t gpa_cover(uint64_t a) { static void gpa_retire(void *arg) { destroy_rec(vk_ctx(), arg); } -void vk_pool_fini(pool_t *p) { +static void pool_fini(pool_t *p) { VkCtx *c = vk_ctx(); if (!c) return; pidx_ent e; @@ -396,6 +396,21 @@ void vk_pool_fini(pool_t *p) { else destroy_rec(c, r); } +// NX2NAVI_TRACE=perf: a pool's initialize and finalize, with their imports, allocations, uploads and GPU waits, marked +// as work a frame can wait for (perf.c perf_mark_add). +int vk_pool_init(pool_t *p) { + uint64_t t0 = perf_mark_begin(); + int r = pool_init(p); + perf_mark_end(PERF_M_POOL, t0, p->size); + return r; +} + +void vk_pool_fini(pool_t *p) { + uint64_t t0 = perf_mark_begin(), size = p->size; + pool_fini(p); + perf_mark_end(PERF_M_POOL, t0, size); +} + // pool_lock held: the device address table follows a MapVirtual request on [lo, hi) of virtual pool v. Ranges it // covers whole leave the table (their records retire); the pages then name the physical pool's buffer at poff, or none. static void gpa_map_virtual(PoolRec *v, uint64_t lo, uint64_t hi, PoolRec *ph, uint64_t poff) { diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 2cfa06f2..645376fa 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -530,10 +530,12 @@ static volatile uint32_t cache_merges; static VkResult pipe_create_in(int kind, void *ci, VkPipeline *pipe, VkPipelineCache cache) { int shared = cache && !own_cache; if (cache && own_cache) cache = own_cache; + uint64_t t0 = perf_mark_begin(); // NX2NAVI_TRACE=perf: the create on any thread, its wait for cache_rw included if (shared) plat_mutex_lock_shared(&cache_rw); VkResult r = kind == 2 ? vkCreateComputePipelines(P.device, cache, 1, ci, NULL, pipe) : vkCreateGraphicsPipelines(P.device, cache, 1, ci, NULL, pipe); if (shared) plat_mutex_unlock_shared(&cache_rw); + perf_mark_end(PERF_M_PIPE, t0, 0); return r; }