diff --git a/runtime/cfg.c b/runtime/cfg.c index 8bcb63f5..f490535a 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -54,6 +54,7 @@ static const char *const known[] = { "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", "SCENE_SHARE", + "PRESNAP_PLACE", "REPLAY_CHECK", "RESOLVE_CHECK", "RESTART_WAIT", "ROMFS", "ROMFS_INDEX", "ROMFS_MMAP", "ROMFS_MODS", "ROMFS_PATCHES", "ROMFS_PROOF_DIR", "ROMFS_VERIFY", "RSTICK_DEADZONE", "RUMBLE", "RUMBLE_CURVE", "RUMBLE_STRENGTH", "RUMBLE_TRIGGERS", "SHADER_CACHE", "SHADER_CACHE_FLUSH", "SHADER_CARRY", "SHADER_MEMO", "SHADOW_COMPARE", "SHADOW_RANGE", "SHADOW_RES", "SHADOW_SCALE", "SHARED_FONTS", "SHARPNESS", "SHOTS", "SHOTS_DIR", diff --git a/runtime/vk.h b/runtime/vk.h index a124ee7f..81ae6766 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -588,6 +588,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t // batch) without a signal, and returns a new begun one for the rest of the submission. The submission's timeline value // is signalled by its last part, whose signal covers every earlier part in submission order. VkCommandBuffer vk_sync_split(void); +extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) // Runs fn(arg) on the completion thread once fence is done on the GPU (event writes, counter reports, sync state). // Callbacks run in the order they were added, after those of earlier values; a value already done runs at once there. void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg); diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index a2e715c8..953f8842 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -272,8 +272,11 @@ static uint64_t submit_sync(const uint64_t *handles, int n) { return f; } +int vk_sub_in_frame; + static void replay_main(void *arg) { (void)arg; + int sub_idx = 0; // submissions since the last present, in queue order perf_replay_thread(); plat_mutex_lock(&rl); for (;;) { @@ -285,6 +288,7 @@ static void replay_main(void *arg) { plat_cond_broadcast(&rcv); plat_mutex_unlock(&rl); if (it.kind == RI_SUBMIT) { + vk_sub_in_frame = sub_idx++; uint64_t f = submit_sync(it.h, it.n); if (f != it.value) rt_fatal("vk: replayed submission signalled %llu, queued as %llu", (unsigned long long)f, @@ -298,6 +302,7 @@ static void replay_main(void *arg) { uint64_t frame = __atomic_add_fetch(&gfx_counters.frames, 1, __ATOMIC_RELAXED); vk_dlssg_submitted(frame); // the frame's submissions were replayed before its present present_image(it.tex, it.has_crop ? it.crop : NULL, it.at, it.queued, frame, it.value); + sub_idx = 0; } plat_mutex_lock(&rl); if (it.kind == RI_SUBMIT) r_submit_done = it.submit_no; @@ -480,7 +485,10 @@ static void vk_be_program_release(const prog_t *p) { replay_drain(DR_PROG_RELEAS // Asynchronous: the handles (nvn_submit_prepare_async ran under the front end's submit lock) are copied and queued; // the value returned is the one the replay will signal. static uint64_t vk_be_submit(const uint64_t *handles, int n) { - if (!gfx_replay_async()) return submit_sync(handles, n); + if (!gfx_replay_async()) { + vk_sub_in_frame = nvn_submits_in_frame; // the front end counts it up after the submit returns + return submit_sync(handles, n); + } vk_sync_start(); ReplayItem it = {.kind = RI_SUBMIT, .n = n, .submit_no = nvn_submit_count()}; if (n > 0) { diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 537540f8..284c004e 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -63,6 +63,9 @@ typedef struct PoolRec { VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf VmaAllocation malloc_; uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync) + uint32_t pm_frame; // replay: the frame (pframe) of pm_cur + uint8_t pm_cur, pm_prev; // replay: the places in their frame (vk_sub_in_frame, 7 the rest) of the submissions + // that synced it, in frame pm_frame and in the one before (NX2NAVI_PRESNAP_PLACE) uint32_t wb_sub; // replay: the last submission that wrote mirror bytes back to guest memory uint32_t retry_sub; // replay: a mirror refused for the VRAM budget is asked again from here, 0 never uint8_t asked_room; // replay: its refusal already asked the textures for room (once until it gets one) @@ -1328,6 +1331,34 @@ enum { MR_BUDGET, MR_UNTRACKED, MR_OVERLAP, MR_ALLOC, MR_N }; int vk_mirror_on; static int mirror_verify; static int mirror_gap = 2; // pre-snap: hot pools at most this many 2 MiB regions apart share one query +static uint32_t pframe; // replay: frames begun (vk_mem_mirror_begin: a place not after the last one's) +static int pplace = 99; // replay: the place of the last submission vk_mem_mirror_begin saw + +// NX2NAVI_PRESNAP_PLACE=1 (default), 0, ab: a submission's pre-snap (vk_mem_mirror_begin) takes the write-watch +// snapshots of the pools the submission at its place in the last frame synced, not those the submission before it +// synced. In Kokiri Forest the frame's first submission (#0) and its fourth (#3) use no pool, and their pre-snaps +// covered the pools of the submission before them, #5's and #2's: 36.8 queries over 282 MiB a frame, 0.23-0.31 ms; then +// #2 and #5, whose submissions before them use no pool either, queried their pools one at a time at their first use, +// 63.5 queries over 254 MiB. +static int presnap_place(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("PRESNAP_PLACE", "1"); + return RT_KNOB_ON(mode); +} + +static int sub_place(void) { return vk_sub_in_frame >= 0 && vk_sub_in_frame < 7 ? vk_sub_in_frame : 7; } + +static void pm_note(PoolRec *r) { // replay: r synced in the submission being recorded + if (r->pm_frame != pframe) { + r->pm_prev = r->pm_frame + 1 == pframe ? r->pm_cur : 0; + r->pm_cur = 0, r->pm_frame = pframe; + } + r->pm_cur |= (uint8_t)(1u << sub_place()); +} + +static uint8_t pm_last(const PoolRec *r) { // the places that synced r in the last frame + return r->pm_frame == pframe ? r->pm_prev : r->pm_frame + 1 == pframe ? r->pm_cur : 0; +} static PoolRec **mlist; // pool_lock: mirrored pools by import_lo static int nm, mcap; static uint32_t msub; // replay: number of the submission being recorded @@ -1709,6 +1740,7 @@ static void mirror_retire(void *arg) { static void mirror_sync(PoolRec *r) { uint64_t t0 = qpc(); r->acc_sub = msub; + pm_note(r); uint64_t lo = r->import_lo, hi = r->import_hi; uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; int excl = 0; @@ -2191,9 +2223,12 @@ void vk_mem_mirror_begin(void) { plat_mutex_lock(&gen_lock); ww_ep++; uint64_t b0 = ww_bytes; + int pl = sub_place(), pp = presnap_place(); + if (pl <= pplace) pframe++; // a new frame, though its first submissions may have had nothing to record + pplace = pl; for (int i = 0; i < nm; i++) { const PoolRec *r = mlist[i]; - if (r->acc_sub != prev) continue; + if (pp ? !(pm_last(r) >> pl & 1) : r->acc_sub != prev) continue; uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT; if (have && a <= run1 + 1 + (uint64_t)mirror_gap) { if (b > run1) run1 = b;