diff --git a/runtime/boot.c b/runtime/boot.c index 33cc0b55..939f6726 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -367,6 +367,7 @@ static void boot_wait(PlatThread *warm, uint64_t cap_ms) { if (p.phase != last.phase || p.sh_done != last.sh_done || p.pipe_done != last.pipe_done) last = p, moved = now; if ((cap_ms && now - t0 >= cap_ms) || now - moved >= BOOT_STALL_MS) { plat_thread_detach(warm); + vk_pipeline_prewarm_marker_clear(); rt_log("boot: the prewarm goes on in the background after %llu ms (%s)", (unsigned long long)(now - t0), cap_ms && now - t0 >= cap_ms ? "NX2NAVI_PREWARM_WAIT_MS" : "no progress for 60 s"); break; diff --git a/runtime/cfg.c b/runtime/cfg.c index 6973171e..42afeead 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -49,7 +49,7 @@ static const char *const known[] = { "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", - "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", + "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "READBACK_SYNC", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index 7626b8f9..6eb736a2 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -80,6 +80,7 @@ uint64_t nvn_tex_view_offset(const tex_t *t, const view_t *v); // (nvn_pins_sweep). Deferred binds (p_bind_handles.src != 0) hold the // handles read at submit (nvn_submit_prepare). Payloads of the ops nvn_obj.h does not define: typedef struct { volatile uint32_t *val; uint32_t mode, value; } p_signal_event; // OP_SIGNAL_EVENT; mode: NVNeventSignalMode +typedef struct { volatile uint32_t *val; uint32_t mode, value; } p_wait_event; // OP_WAIT_EVENT; mode: NVNeventWaitMode typedef struct { uint64_t type, addr; } p_report_counter; // OP_REPORT_COUNTER: NVNcounterType, NVNcounterData address typedef struct { uint64_t type; } p_reset_counter; // OP_RESET_COUNTER typedef struct { uint64_t src, dst, size; } p_copy_b2b; // OP_COPY_B2B: GPU addresses @@ -113,6 +114,9 @@ const char *nvn_op_name(uint32_t op); // "DRAW_IDX", ...; " // MemoryPoolInvalidateMappedRange: the CPU is about to read pool storage the GPU wrote. Weak no-op in nvn_ext.c; a back // end that renders into host-side copies defines it and writes them back. void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size); +// The range nvn_pool_invalidate is about to read back. Weak no-op in nvn_ext.c; a back end that keeps device-side copies +// of pool memory (vk_memory.c, "read-back ranges") defines it and refreshes the range from guest memory at every use. +void nvn_pool_invalidated(const pool_t *p, uint64_t off, uint64_t size); // MemoryPoolFlushMappedRange: the CPU wrote [mem + off, mem + off + size). Weak no-op in nvn_ext.c; a back end that keeps // device-side copies of pool memory (vk_memory.c's VRAM mirror) defines it and copies the range again. void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size); diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index 34ccf8a1..30714853 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -1160,8 +1160,18 @@ NVN_API(nvnCommandBufferSetBlendColor) { memcpy(REC(cmd_of(c), OP_BLEND_COLOR, p // void nvnCommandBufferWaitSync(NVNcommandBuffer *cmd, const NVNsync *sync), WaitEvent(cmd, const NVNevent *event, // NVNeventWaitMode mode, uint32_t value), SetShaderScratchMemory(cmd, const NVNmemoryPool *pool, ptrdiff_t offset, // size_t size), SetPatchSize(cmd, int size) -NVN_API(nvnCommandBufferWaitSync) { (void)c; } -NVN_API(nvnCommandBufferWaitEvent) { (void)c; } +// WaitSync logs its first call: a game that orders two queues with it replays here in submission order only. +NVN_API(nvnCommandBufferWaitSync) { + static volatile int32_t logged; + if (!plat_xchg32(&logged, 1)) rt_log("nvn: first CommandBufferWaitSync: recorded as nothing (one replay queue)"); + (void)c; +} +// WaitEvent is recorded (OP_WAIT_EVENT) so that the replay can count the waits its submission order would have +// passed before the signal they wait for (vk_exec_xfer.c wait_event); it waits for nothing. +NVN_API(nvnCommandBufferWaitEvent) { + const ev_t *e = arg_p(c, 1); + *REC(cmd_of(c), OP_WAIT_EVENT, p_wait_event) = (p_wait_event){e->val, (uint32_t)arg_x(c, 2), (uint32_t)arg_x(c, 3)}; +} NVN_API(nvnCommandBufferSetShaderScratchMemory) { (void)c; } NVN_API(nvnCommandBufferSetPatchSize) { (void)c; } @@ -1264,6 +1274,7 @@ const char *nvn_op_name(uint32_t op) { "DRAW_IDX", "DISPATCH", "BARRIER", "SET_TEX_POOL", "SET_SAMP_POOL", "COPY_B2T", "COPY_STRIDE", "BLEND_COLOR", "COPY_T2B", "BIND_IMG", "COPY_T2T", "RESET_COUNTER", "CLEAR_TEX", "DEPTH_CLAMP", "RENDER_ENABLE", "RASTER_DISCARD", "RENDER_COND", "PRIM_RESTART", "DRAW_IDX_IND", "DISPATCH_IND", "CLEAR_BUF", "PUSH_CC", "DRAW_MESH", "DRAW_MESH_IND", + "WAIT_EVENT", }; _Static_assert(sizeof names / sizeof *names == OP_COUNT_, "one name per op"); return op < OP_COUNT_ ? names[op] : "?"; diff --git a/runtime/nvn_ext.c b/runtime/nvn_ext.c index a0ce8d60..e1ad5d2c 100644 --- a/runtime/nvn_ext.c +++ b/runtime/nvn_ext.c @@ -16,9 +16,13 @@ void nvn_pool_invalidate(const pool_t *p, uint64_t off, uint64_t size) { if (!p->mem || nvn_gpu_only((uintptr_t)p->mem) || off >= p->size || !size) return; if (size > p->size - off) size = p->size - off; + nvn_pool_invalidated(p, off, size); gfx_backend()->guest_read((uintptr_t)p->mem + off, size); } +__attribute__((weak)) void nvn_pool_invalidated(const pool_t *p, uint64_t off, uint64_t size) { + (void)p, (void)off, (void)size; +} __attribute__((weak)) void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { (void)p, (void)off, (void)size; } // ---------- debug labels, markers, groups, checkpoints: meaningful only to NVN debugging tools diff --git a/runtime/nvn_obj.h b/runtime/nvn_obj.h index 38c775d4..526ea503 100644 --- a/runtime/nvn_obj.h +++ b/runtime/nvn_obj.h @@ -128,7 +128,7 @@ enum { OP_BIND_PROGRAM, OP_BIND_STATE, OP_BIND_VATTR, OP_BIND_VSTREAM, OP_BIND_VBUF, OP_BIND_UBO, OP_BIND_SSBO, OP_BIND_TEX, OP_BIND_SAMP, OP_STENCIL, OP_POLY_OFFSET, OP_DRAW, OP_DRAW_IDX, OP_DISPATCH, OP_BARRIER, - OP_SET_TEX_POOL, OP_SET_SAMP_POOL, OP_COPY_B2T, OP_COPY_STRIDE, OP_BLEND_COLOR, OP_COPY_T2B, OP_BIND_IMG, OP_COPY_T2T, OP_RESET_COUNTER, OP_CLEAR_TEX, OP_DEPTH_CLAMP, OP_RENDER_ENABLE, OP_RASTER_DISCARD, OP_RENDER_COND, OP_PRIM_RESTART, OP_DRAW_IDX_IND, OP_DISPATCH_IND, OP_CLEAR_BUF, OP_PUSH_CC, OP_DRAW_MESH, OP_DRAW_MESH_IND, OP_COUNT_ + OP_SET_TEX_POOL, OP_SET_SAMP_POOL, OP_COPY_B2T, OP_COPY_STRIDE, OP_BLEND_COLOR, OP_COPY_T2B, OP_BIND_IMG, OP_COPY_T2T, OP_RESET_COUNTER, OP_CLEAR_TEX, OP_DEPTH_CLAMP, OP_RENDER_ENABLE, OP_RASTER_DISCARD, OP_RENDER_COND, OP_PRIM_RESTART, OP_DRAW_IDX_IND, OP_DISPATCH_IND, OP_CLEAR_BUF, OP_PUSH_CC, OP_DRAW_MESH, OP_DRAW_MESH_IND, OP_WAIT_EVENT, OP_COUNT_ }; // Payload (immediately follows op_hdr) typedef struct { uint32_t offset, size; uint8_t data[]; } p_push_cc; // OP_PUSH_CC: NVN2 compute push constants diff --git a/runtime/nvn_probe.c b/runtime/nvn_probe.c index c74a0d73..98a32cc5 100644 --- a/runtime/nvn_probe.c +++ b/runtime/nvn_probe.c @@ -403,6 +403,7 @@ static uint64_t record_all(void) { CALL(nvnTextureViewSetLevels, U(&view), 1, 3); REC(OP_SIGNAL_EVENT, CALL(nvnCommandBufferSignalEvent, U(c), U(&ev), NVN_EVENT_SIGNAL_MODE_ADD, NVN_EVENT_SIGNAL_LOCATION_BOTTOM, 0, 0x55)); + REC(OP_WAIT_EVENT, CALL(nvnCommandBufferWaitEvent, U(c), U(&ev), NVN_EVENT_WAIT_MODE_EQUAL, 0x55)); REC(OP_REPORT_COUNTER, CALL(nvnCommandBufferReportCounter, U(c), NVN_COUNTER_TYPE_SAMPLES_PASSED, 0xaabb00)); REC(OP_RESET_COUNTER, CALL(nvnCommandBufferResetCounter, U(c), NVN_COUNTER_TYPE_SAMPLES_PASSED)); REC(OP_COPY_B2B, CALL(nvnCommandBufferCopyBufferToBuffer, U(c), 0x1000, 0x2000, 0x40, 0)); @@ -494,6 +495,11 @@ static void parse_all(uint64_t h) { CHECK(p->val == (volatile uint32_t *)pmem && p->mode == NVN_EVENT_SIGNAL_MODE_ADD && p->value == 0x55, "SIGNAL_EVENT payload"); break; } + case OP_WAIT_EVENT: { + const p_wait_event *p = a; + CHECK(p->val == (volatile uint32_t *)pmem && p->mode == NVN_EVENT_WAIT_MODE_EQUAL && p->value == 0x55, "WAIT_EVENT payload"); + break; + } case OP_REPORT_COUNTER: { const p_report_counter *p = a; CHECK(p->type == NVN_COUNTER_TYPE_SAMPLES_PASSED && p->addr == 0xaabb00, "REPORT_COUNTER payload"); diff --git a/runtime/vk.h b/runtime/vk.h index 205eb09b..dc625bce 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -587,6 +587,9 @@ int vk_pipeline_async_on(void); // 1 after vk_pipeline_async_enable: graphics // cached under the current translator from their stored inputs, and builds every recorded pipeline, on all cores. // No-op without vk_pipeline_async_enable, with NX2NAVI_PIPE_PREBUILD=0 or with NX2NAVI_GPU=null. void vk_pipeline_prewarm(void); +// Ends the boot prebuild's crash marker (vk_pipeline.c, "boot prebuild crash marker"): boot_wait calls it when it lets +// the game start while the prewarm goes on, so a crash in play is not taken for one in the boot's builds. +void vk_pipeline_prewarm_marker_clear(void); typedef struct { int phase; // 0 starting (device, key file), 1 shaders, 2 pipelines, 3 done int64_t sh_done, sh_total, sh_translated; // gfx_shader_prewarm_progress diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 684a37d9..4e51eef7 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -58,10 +58,14 @@ void vk_exec_frame_written(const tex_t *t) { fw[nfw++].n = 1; } +uint64_t vk_exec_submit_seq; + void vk_exec_frame_log(const tex_t *p, uint64_t frame) { vk_exec_cond_frame_end(frame); // a frame boundary in replay order (present_image runs on the replay thread) if (!(rt_trace & RT_TRACE_GFX)) return; VkExecFrame *f = &vk_exec_frame; + uint64_t ew, eu, el, ed; + vk_exec_event_stats(&ew, &eu, &el, &ed); int hit = 0; char list[512] = ""; size_t o = 0; @@ -73,14 +77,16 @@ void vk_exec_frame_log(const tex_t *p, uint64_t frame) { } rt_log("frame %llu: submits %llu passes %llu draws %llu/%llu clears %llu copies %llu dispatches %llu; present %p " "%ux%u/%#x written %d; feedback draws %llu barriers %llu restarts %llu layouts %llu read-before-write %llu; " - "uniform updates %llu (%llu ended a pass); targets%s", + "uniform updates %llu (%llu ended a pass); event waits %llu (%llu before their signal: %llu " + "signalled later, %.1f submits later on average); targets%s", (unsigned long long)frame, (unsigned long long)f->submits, (unsigned long long)f->passes, (unsigned long long)f->draws, (unsigned long long)f->draws_seen, (unsigned long long)f->clears, (unsigned long long)f->copies, (unsigned long long)f->dispatches, p ? (const void *)p->mem : NULL, p ? p->d.width : 0, p ? p->d.height : 0, p ? (uint32_t)p->d.format : 0, hit, (unsigned long long)f->fb_draws, (unsigned long long)f->fb_barriers, (unsigned long long)f->fb_restarts, (unsigned long long)f->fb_layouts, (unsigned long long)f->fb_war_barriers, (unsigned long long)f->ub_updates, - (unsigned long long)f->ub_pass_breaks, list); + (unsigned long long)f->ub_pass_breaks, (unsigned long long)ew, (unsigned long long)eu, (unsigned long long)el, + el ? (double)ed / (double)el : 0.0, list); memset(f, 0, sizeof *f); nfw = 0; } @@ -945,6 +951,7 @@ static void walk(VkExec *ex, const op_hdr *h, int depth) { case OP_CLEAR_BUF: vk_exec_op_xfer(ex, h->op, a); break; + case OP_WAIT_EVENT: vk_exec_wait_event(a); break; default: rt_fatal("vk: op %s has no replay handler", nvn_op_name(h->op)); } if (split_draws >= split_n) maybe_split(ex); @@ -971,6 +978,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t ex->barrier_at = -1; ex->nfbl = 0; // the previous command buffer ended with a full barrier: every image is in GENERAL vk_exec_frame.submits++; + vk_exec_submit_seq++; split_draws = part_start = 0; part_t0 = kern_now_ns(); x_here = 1; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index b10c500d..e499b782 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -200,6 +200,12 @@ VkImageView vk_exec_target_view(const VkTexImage *img, VkFormat format, VkImageV // vk_exec_xfer.c: OP_COPY_B2B, OP_COPY_B2T, OP_COPY_T2B, OP_COPY_T2T, OP_CLEAR_TEX, OP_SIGNAL_EVENT, OP_REPORT_COUNTER, // OP_RESET_COUNTER, OP_RENDER_ENABLE, OP_RENDER_COND, OP_CLEAR_BUF. void vk_exec_op_xfer(VkExec *ex, uint32_t op, const void *payload); +// OP_WAIT_EVENT: records nothing; counts a wait whose signal the replay has not reached yet (vk_exec_xfer.c). +void vk_exec_wait_event(const void *payload); +// Since the last call, which resets them: waits, waits reached before their signal, those of them a later replayed +// signal met, and the sum of how many submissions later it came. +void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs); +extern uint64_t vk_exec_submit_seq; // replay thread: submissions replayed so far // vk_dlaa.c: once DLSS has scaled the scene this frame, passes larger than the scene's light target vk_dlaa_swap_img // sample DLSS's output vk_dlaa_swap_view in its place, so the game's scaling draw scales DLSS's picture. extern const VkTexImage *vk_dlaa_swap_img; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 644c17bb..0a7120d1 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -333,7 +333,65 @@ static void oq_report(VkExec *ex, uint64_t addr) { if (!oq_keep) oq_counting = 0; } +// ---- event order check +// The game orders work between its queues with events: a command buffer waits (WaitEvent) until another's SignalEvent +// has set the value. Every submission replays here on one queue in the order it came, and a wait records nothing, so a +// wait whose signal comes in a later submission does not hold its work back: that work runs before what it waits for. +// The replay keeps the value each event reaches in replay order (ev_seen, the signals recorded so far, on top of the +// value in memory) and counts the waits it reaches unmet; NX2NAVI_TRACE=perf logs the count, and the first few are +// logged as they happen. Replay thread only. +#define EV_SEEN 64 +static struct { volatile uint32_t *val; uint32_t v; } ev_seen[EV_SEEN]; +static int ev_n, ev_next; +static uint64_t ev_waits, ev_unmet, ev_late, ev_late_subs; +// Waits reached unmet, until a replayed signal meets them (ev_late: the signal came that many submissions later, so +// the work after the wait ran before the work it waited for) or they are pushed out by newer ones. +#define EV_OPEN 64 +static struct { volatile uint32_t *val; uint32_t mode, value; uint64_t sub; } ev_open[EV_OPEN]; +static int ev_open_n; + +static uint32_t *ev_slot(volatile uint32_t *val, int create) { + for (int i = 0; i < ev_n; i++) + if (ev_seen[i].val == val) return &ev_seen[i].v; + if (!create) return NULL; + int i = ev_n < EV_SEEN ? ev_n++ : ev_next++ % EV_SEEN; + ev_seen[i].val = val, ev_seen[i].v = __atomic_load_n(val, __ATOMIC_ACQUIRE); + return &ev_seen[i].v; +} + +static int ev_holds(uint32_t have, uint32_t mode, uint32_t want) { + return mode == NVN_EVENT_WAIT_MODE_GEQUAL_WRAP ? (int32_t)(have - want) >= 0 : have == want; +} + +void vk_exec_wait_event(const void *payload) { + const p_wait_event *p = payload; + ev_waits++; + uint32_t mem = __atomic_load_n(p->val, __ATOMIC_ACQUIRE); + const uint32_t *seen = ev_slot(p->val, 0); + if (ev_holds(mem, p->mode, p->value) || (seen && ev_holds(*seen, p->mode, p->value))) return; + ev_unmet++; + if (ev_open_n == EV_OPEN) memmove(ev_open, ev_open + 1, sizeof ev_open - sizeof *ev_open), ev_open_n--; + ev_open[ev_open_n].val = p->val, ev_open[ev_open_n].mode = p->mode, ev_open[ev_open_n].value = p->value; + ev_open[ev_open_n++].sub = vk_exec_submit_seq; + static int logged; + if (logged++ < 8) + rt_log("vk: event wait reached before its signal: event %p, mode %u, wants %#x, memory %#x, replayed signals %s%#x", + (void *)p->val, p->mode, p->value, mem, seen ? "" : "none ", seen ? *seen : 0); +} + +void vk_exec_event_stats(uint64_t *waits, uint64_t *unmet, uint64_t *late, uint64_t *late_subs) { + *waits = ev_waits, *unmet = ev_unmet, *late = ev_late, *late_subs = ev_late_subs; + ev_waits = ev_unmet = ev_late = ev_late_subs = 0; +} + static void signal_event(VkExec *ex, const p_signal_event *p) { + uint32_t *seen = ev_slot(p->val, 1); + *seen = p->mode == NVN_EVENT_SIGNAL_MODE_ADD ? *seen + p->value : p->value; + for (int i = 0; i < ev_open_n;) + if (ev_open[i].val == p->val && ev_holds(*seen, ev_open[i].mode, ev_open[i].value)) { + ev_late++, ev_late_subs += vk_exec_submit_seq - ev_open[i].sub; + ev_open[i] = ev_open[--ev_open_n]; + } else i++; EventWrite *e = malloc(sizeof *e); if (!e) rt_fatal("vk: out of memory"); *e = (EventWrite){p->val, p->mode, p->value}; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 6234e53f..1dc7797a 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -36,6 +36,11 @@ typedef struct { struct PoolRec *phys; uint64_t off; } VPage; // virtual page #define FLUSH_RANGES 32 // flush ranges a mirrored pool keeps between two syncs (flush_add) typedef struct { uint64_t lo, hi; } FlushRange; +// Read-back ranges: small results the CPU invalidates, reads and often writes again (the exposure meter: the GPU adds +// a frame's light into it, the CPU reads the sum and stores 0). mirror_sync copies them from guest memory into the +// mirror at every submission that uses the pool, whether or not a write was seen: see "read-back ranges" below. +#define READBACK_RANGES 4 +#define READBACK_MAX (16u << 10) typedef struct PoolRec { const pool_t *key; @@ -68,6 +73,8 @@ typedef struct PoolRec { FlushRange fl[FLUSH_RANGES + 1]; // flush_lock: nvnMemoryPoolFlushMappedRange pages not copied yet (flush_add) int fl_n; // flush_lock: ranges in fl; read without the lock to skip an empty list uint64_t fl_calls, fl_sum; // flush_lock: the calls and bytes behind fl, for mirror_sync's statistics + FlushRange rb[READBACK_RANGES + 1]; // flush_lock: read-back ranges (nvn_pool_invalidated), guest addresses + int rb_n; // flush_lock: ranges in rb; read without the lock to skip an empty list // Shadow storage (POOL_IMPORT in shadow mode, "shadow storage" below): buf is a host-visible copy of the guest pages. VmaAllocation salloc; uint8_t *smap; // its mapping; offsets as in buf (from import_lo) @@ -1236,6 +1243,7 @@ typedef struct { uint64_t presnap_queries, sync_queries, presnap_t, sync_t, prologues, presnap_bytes; uint64_t snap_bytes, merge_bytes, merge_dispatches, snap_allocs; // write-back merge uint64_t page_bytes, flush_n, flush_bytes, flush_calls, flush_sum; // what mirror_sync queues, by source + uint64_t rb_n, rb_bytes; // read-back ranges mirror_sync queued } MirrorAcc; static MirrorAcc mrep; // replay: this submission static MirrorAcc macc; // mirror_stat_lock: since the last summary @@ -1493,6 +1501,19 @@ static void mirror_sync(PoolRec *r) { (hi - lo) / 1048576.0, n, (unsigned long long)calls, sum / 1048576.0); } } + if (__atomic_load_n(&r->rb_n, __ATOMIC_ACQUIRE)) { + FlushRange rb[READBACK_RANGES]; + plat_mutex_lock(&flush_lock); + int n = r->rb_n; + memcpy(rb, r->rb, (size_t)n * sizeof *rb); + plat_mutex_unlock(&flush_lock); + for (int i = 0; i < n; i++) { + uint64_t a = rb[i].lo > lo ? rb[i].lo : lo, b = rb[i].hi < hi ? rb[i].hi : hi; + if (a >= b) continue; + range_add(&pend, &npend, &cpend, r, a - lo, b - a); + mrep.rb_n++, mrep.rb_bytes += b - a; + } + } mrep.syncs++; mrep.sync_t += qpc() - t0; } @@ -1834,6 +1855,36 @@ void nvn_pool_flushed(const pool_t *p, uint64_t off, uint64_t size) { plat_mutex_unlock_shared(&pool_lock); } +// ---- read-back ranges +// mirror_sync decides at record time which guest pages to copy into the mirror; the copy itself runs when the GPU +// reaches the submission. A CPU write in between (the game's threads overlap the replay) is in guest memory when the +// copy runs but was not seen at record time, so the GPU works on the mirror's older bytes and the write-back puts +// them over the CPU's: the exposure meter added a frame onto the sum the CPU had already read, it read twice the +// light one frame and none the next, and the picture flared or dimmed for a frame. A range the CPU invalidates is a +// result it reads and may reset, so every submission that uses its pool copies it unconditionally, last of the +// uploads, and the GPU adds to what the CPU left there. Small ranges only, READBACK_RANGES per pool (the nearest two +// merge beyond that). NX2NAVI_READBACK_SYNC=0 turns it off. +static int readback_on = -1; +static volatile int64_t m_rb_calls; +void nvn_pool_invalidated(const pool_t *p, uint64_t off, uint64_t size) { + if (!vk_mirror_on || !size || size > READBACK_MAX || !p->mem) return; + if (readback_on < 0) { const char *v = cfg_get("NX2NAVI_READBACK_SYNC"); readback_on = !(v && !strcmp(v, "0")); } + if (!readback_on) return; + plat_inc64(&m_rb_calls); + plat_mutex_lock_shared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)p->mem, 0); + PoolRec *r = e && e->key == p ? e->buf : NULL; + if (r && r->kind == POOL_IMPORT) { + uint64_t a = ((uintptr_t)p->mem + off) & ~0xfffull, b = ((uintptr_t)p->mem + off + size + 0xfff) & ~0xfffull; + plat_mutex_lock(&flush_lock); + int known = 0; + for (int i = 0; i < r->rb_n && !known; i++) known = r->rb[i].lo <= a && b <= r->rb[i].hi; + if (!known) __atomic_store_n(&r->rb_n, flush_add(r->rb, r->rb_n, a, b, READBACK_RANGES), __ATOMIC_RELEASE); + plat_mutex_unlock(&flush_lock); + } + plat_mutex_unlock_shared(&pool_lock); +} + // gen_lock exclusive: freezes up to FREEZE_PER_SUB candidate regions still idle. The query after the // protection collects the writes made before it took effect. static void freeze_pass(void) { @@ -2083,9 +2134,11 @@ static void mirror_log(uint64_t frame) { a.merge_dispatches / f, a.snap_bytes / f / M, a.snap_allocs / f, fc / f, fb / f / M, a.presnap_queries / f, a.presnap_bytes / f / M, a.sync_queries / f, a.prologues / f, qpc_ms(a.presnap_t + a.sync_t) / f, qpc_ms(a.presnap_t) / f, qpc_ms(a.sync_t) / f, ver); + uint64_t rc = (uint64_t)plat_xchg64(&m_rb_calls, 0); rt_log("mem: mirror frame %llu: upload sources per frame, overlaps counted twice: written pages %.3f MiB; flush ranges " - "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB", (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, - a.flush_bytes / f / M, a.flush_calls / f, a.flush_sum / f / M); + "%.2f, %.3f MiB, from %.1f flushes of %.3f MiB; read-back ranges %.2f, %.4f MiB, from %.2f invalidates", + (unsigned long long)frame, a.page_bytes / f / M, a.flush_n / f, a.flush_bytes / f / M, a.flush_calls / f, + a.flush_sum / f / M, a.rb_n / f, a.rb_bytes / f / M, rc / f); } // With NX2NAVI_TRACE=perf every 600 frames: what holds device-local memory, to see what grows over a long session. diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index b7384b57..da67cf46 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2673,6 +2673,8 @@ static void warm_builder(void *arg) { } static void lb_start(const uint32_t pr_hist[16], const uint32_t fs_hist[4]); +static int pm_skipped(void); +static void pm_set(void); // vk_pipeline_prewarm_progress's numbers: the phase, and the pipelines finished (built or failed) of those queued, copied // while prewarm_run waits for its builders. @@ -2756,6 +2758,16 @@ static void prewarm_run(void) { rt_log("vk_pipeline: vertex output unions: %u vertex shaders, %.2f locations each; %llu pairs read %.2f each", VO.n, VO.n ? (double)uni / VO.n : 0.0, (unsigned long long)pairs, pairs ? (double)pair / pairs : 0.0); } + if (pm_skipped()) { // the boot's pipeline builds killed an earlier start on this GPU and driver + plat_mutex_lock(&P.lock); + free(K.file); + K.file = NULL; + plat_mutex_unlock(&P.lock); + free(ids), free(sh); + lb_start(pr_hist, fs_hist); + return; + } + pm_set(); uint32_t queued = 0, dropped = 0, known = 0; plat_mutex_lock(&P.lock); K.t0 = t1; @@ -2820,9 +2832,71 @@ static void prewarm_run(void) { free(K.file); K.file = NULL; plat_mutex_unlock(&P.lock); + vk_pipeline_prewarm_marker_clear(); // the boot's share is done; what the pool builds later runs during play lb_start(pr_hist, fs_hist); } +// ---- boot prebuild crash marker +// The boot builds the recorded pipelines (warm_builder) before the game starts. A driver whose compiler crashes on one +// of them would kill every start at the same point, and the preparation journal above does not cover these builds. +// So /pipeline-prewarm-running.txt exists while they run: written before the first one, deleted when the boot's +// share ends (or when the boot stops waiting and lets the game start). A start that finds it left over records this +// GPU and driver in /pipeline-prewarm-skip.txt, and starts on them build no recorded pipelines ahead of play: +// each one is built at its first draw instead, as with NX2NAVI_PIPE_PREBUILD=0. Another driver version, or deleting +// the file, brings the prebuild back. Only the boot's own share is covered: a crash during play, where the pool also +// builds what the boot left, says nothing about these builds. +static int cache_dir(char *dir, size_t n); +static void gpu_key(char *out, size_t n); +static char PM_run[PLAT_PATH_MAX * 2 + 48]; +static volatile int32_t pm_on; + +static int pm_skipped(void) { + char dir[PLAT_PATH_MAX * 2], sp[sizeof PM_run], key[32]; + if (!cache_dir(dir, sizeof dir)) return 0; + snprintf(PM_run, sizeof PM_run, "%s/pipeline-prewarm-running.txt", dir); + snprintf(sp, sizeof sp, "%s/pipeline-prewarm-skip.txt", dir); + gpu_key(key, sizeof key); + uint64_t n = 0; + uint8_t *left = plat_file_read(PM_run, &n); + if (left) { + char text[256]; + int len = snprintf(text, sizeof text, + "# The boot's pipeline prebuild did not finish on this GPU and driver (runtime/vk_pipeline.c " + "pm_skipped): starts skip it\n# while this line matches. Delete this file to build ahead of play " + "again.\n%s\n", key); + if (plat_file_write_atomic(sp, text, (size_t)len)) plat_file_delete(PM_run); + rt_log("vk_pipeline: the last start died while the boot built pipelines (%s was left); %s now skips that " + "prebuild on this GPU and driver (%s)", PM_run, sp, key); + free(left); + } + uint8_t *skip = plat_file_read(sp, &n); + int on = 0; + for (char *line = (char *)skip, *next; line && *line && !on; line = next) { + next = strchr(line, '\n'); + if (next) *next++ = 0; + size_t l = strlen(line); + while (l && (line[l - 1] == '\r' || line[l - 1] == ' ')) line[--l] = 0; + on = line[0] != '#' && !strcmp(line, key); + } + free(skip); + if (on) + rt_log("vk_pipeline: prewarm: %zu recorded pipelines not built ahead of play: an earlier start died while building " + "them on this GPU and driver (%s lists %s); each builds at its first draw", K.file_n, sp, key); + return on; +} + +static void pm_set(void) { + if (!PM_run[0]) return; + char key[32]; + gpu_key(key, sizeof key); + // straight to the disk: the file has to be there if the driver takes the process down a moment later + if (plat_file_write_atomic(PM_run, key, strlen(key))) plat_xchg32(&pm_on, 1); +} + +void vk_pipeline_prewarm_marker_clear(void) { + if (plat_xchg32(&pm_on, 0)) plat_file_delete(PM_run); +} + // ensure_init's hook: without a prewarm (NX2NAVI_PIPE_PREBUILD=0, or the device came up first elsewhere) the records // are not built. Caller holds no lock. static void prebuild_start(void) {