From 04de04bc28d07075bad842cce3dc86180ec57246 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Mon, 9 Nov 2026 23:01:18 -0500 Subject: [PATCH 1/3] Give a texture bucket the generation its state had before, for releases as well as take-backs. The first version put a bucket's generation back only when a take-back undid the release just before it. A town run with it showed half the take-backs going back and the fullest bucket (1,315 slots) still bumped 7,736 times per 3,600 submissions: the game keeps those textures released and takes them back for a moment, so every release was a new generation and the pool refresh still refilled 5,220 slots per submission. A bucket's state is now named by its epoch (changes other than a release or a take-back) and the XOR of the uids of its released entries. A release or a take-back that brings the bucket to a state it was in before gets that state's generation, whichever way round, and in any order of several entries; a new state gets a new one, never one used. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 6 ++-- runtime/vk_texture.c | 85 +++++++++++++++++++++++++++++--------------- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 5ee3445c..b6a1a00e 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -154,9 +154,9 @@ # made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay. # 2 (diagnosis): the same marks, and such textures give way to stand-ins in the texture pool, but none is freed. # tex_disuse = 1 -# 1: a texture the game releases and takes straight back, with nothing else changed at its address, leaves the texture -# table's slots there as they were, instead of having them all looked up again (large towns release and take back -# thousands a second). 0: every release and take-back counts as a change, as before. +# 1: a texture the game releases or takes back, with nothing else changed at its address, puts the texture table's +# slots there back in a state they were in before, and they are not all looked up again (large towns release and take +# back thousands a second). 0: every release and take-back counts as a new state. # tex_gen_undo = 1 # Host zstd decoder in place of the game's (0 = the game's), and a check that decodes every call both ways and logs # the differences (1 = on; slow). Ocarina of Time 1.1.1 keeps the game's decoder while zstd_native is unset: its host diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 52007d66..e33ab2c1 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -57,7 +57,7 @@ typedef struct Tex { uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it int uploaded, released; uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve) - uint32_t rel_agen, rel_pre; // its address generation after and before the release's bump (tex_gen_unrelease) + uint64_t uid; // names the entry in its bucket's released set (tex_gen_flip); 0 until its first release uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale) uint32_t back_gap; // most presents between a release and its taking back so far (stale_after) uint32_t n_up; // uploads of this entry (vk_texture_debug) @@ -203,13 +203,26 @@ static uint16_t *slot_bucket; static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; } -// tex_gen_bump calls, by site: destroy, create, take-back, release; and the take-backs that undid their release's bump -// instead (tex_gen_unrelease), for vk_texture_bucket_stats +// tex_gen_bump calls, by site: destroy, create, take-back, release; and the releases and take-backs that went back to a +// generation their bucket's state had before (tex_gen_flip), for vk_texture_bucket_stats static uint64_t n_bumps, n_site[4], n_undone; static uint32_t bump_count[AGEN_N]; -// The highest generation each bucket ever had. A bump takes the next one after it, not the one after the current -// value, so a value a bucket went back from (tex_gen_unrelease) is never given to another state. +// The highest generation each bucket ever had. A new state takes the next one after it, not the one after the current +// value, so a value a bucket went back from (tex_gen_flip) is never given to another state. static uint32_t agen_top[AGEN_N]; +// What a slot at an address resolves to depends on the entries there and on which of them are released +// (slot_resolve). Per bucket: epoch counts every change other than a release or a take-back (an entry made, destroyed, +// retired, held or taken back with more than its flag changing), sig is the released entries' uids XORed, which a +// release and a take-back each flip. (bucket, epoch, sig) names the state; gs remembers the generation each state +// got, so a bucket that comes back to a state gets that state's generation back. +static uint32_t bepoch[AGEN_N]; +static uint64_t bsig[AGEN_N]; +#define GS_N 8192 +static struct { uint64_t sig; uint32_t b, epoch, gen, valid; } gs[GS_N]; + +static uint32_t gs_at(uint32_t b) { + return (uint32_t)(((bsig[b] ^ ((uint64_t)b << 32 | bepoch[b])) * 0x9E3779B97F4A7C15ull) >> 51); +} static void tex_gen_bump(uint64_t addr) { n_bumps++; @@ -217,35 +230,51 @@ static void tex_gen_bump(uint64_t addr) { if (!++tex_gen) tex_gen = 1; slots_gen++; uint32_t b = agen_idx(addr); + bepoch[b]++; __atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE); + gs[gs_at(b)] = (typeof(gs[0])){bsig[b], b, bepoch[b], agen[b], 1}; dirty_push(DIRTY_BUCKET | b); } -// NX2NAVI_TEX_GEN_UNDO (default 1; 0: a take-back bumps like any change). Ocarina finalizes and initializes transient -// textures all through a frame: in a large town about 40,000 releases and as many take-backs per 3,600 submissions, -// each a bump of its address bucket, and the buckets with the most slots (1,300 registered over one address) were the -// ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it and spent its 6 ms of a -// frame in half the frames there, which is also when new textures arrive late. -// A release followed by the take-back of the same entry, with no other change at its address between them, leaves every -// slot there resolving as before the release: slot_resolve reads the entries at the address and their released flags, -// and those are as they were. So the take-back puts the bucket's generation back to its value from before the -// release, and whatever was resolved then counts as current again: the pool's tokens, the descriptor memo and the -// slots' own cached entries. What was resolved while the entry was released carries the generation in between, which -// no state gets again (agen_top). The global generations still move, so their readers look, and find nothing changed. -// Not for an entry that was superseded or overwritten while released: its take-back changes more than the flag. -static void tex_gen_unrelease(Tex *e) { +// NX2NAVI_TEX_GEN_UNDO (default 1; 0: every release and take-back is a new generation). Ocarina finalizes and +// initializes transient textures all through a frame: in a large town about 40,000 releases and as many take-backs +// per 3,600 submissions, each a bump of its address bucket, and the buckets with the most slots (1,300 registered over +// one address) were the ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it. +// A release or a take-back changes one entry's released flag and nothing else, so it moves the bucket between states +// it has usually been in before: the textures the game keeps released and takes back for a moment, and those it +// releases and takes straight back. The bucket then gets the generation that state had, and whatever was resolved in +// that state counts as current again: the pool's tokens, the descriptor memo and the slots' own cached entries. A +// state seen for the first time gets a new generation (agen_top), and any other change starts a new epoch, so a +// generation never stands for two states. The global generations still move, so their readers look, and find the +// tokens equal. plain: the take-back of an entry that was superseded or overwritten while released changes more than +// the flag, and is a change like any other. +static void tex_gen_flip(Tex *e, int plain) { static int on = -1; + static uint64_t uids; if (on < 0) on = rt_knob("TEX_GEN_UNDO", "1") == 1; - uint32_t b = agen_idx(e->key.addr), at = e->rel_agen; - e->rel_agen = 0; - if (!on || !at || agen[b] != at || e->superseded || e->overwritten) { + if (!e->uid) { + uint64_t z = ++uids * 0x9E3779B97F4A7C15ull; + z = (z ^ z >> 30) * 0xBF58476D1CE4E5B9ull, z = (z ^ z >> 27) * 0x94D049BB133111EBull; + e->uid = (z ^ z >> 31) | 1; + } + uint32_t b = agen_idx(e->key.addr); + bsig[b] ^= e->uid; + if (!on || plain) { tex_gen_bump(e->key.addr); return; } - n_undone++; if (!++tex_gen) tex_gen = 1; slots_gen++; - __atomic_store_n(&agen[b], e->rel_pre, __ATOMIC_RELEASE); + typeof(gs[0]) *m = &gs[gs_at(b)]; + if (m->valid && m->b == b && m->epoch == bepoch[b] && m->sig == bsig[b]) { + n_undone++; + __atomic_store_n(&agen[b], m->gen, __ATOMIC_RELEASE); + } else { + n_bumps++; + bump_count[b]++; + __atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE); + *m = (typeof(*m)){bsig[b], b, bepoch[b], agen[b], 1}; + } dirty_push(DIRTY_BUCKET | b); } @@ -2134,7 +2163,7 @@ static Tex *entry_of(const tex_t *t) { e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one uint64_t gap = e->samp_frame - e->rel_frame; if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; - e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_unrelease(e); + e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); e->overwritten = 0; if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work e->superseded = 0, e->reused = 1; @@ -2201,9 +2230,7 @@ static void mark_released(Tex *e) { // tex_lock held; e is not released nreleased++; rel_append(e); n_site[3]++; - e->rel_pre = agen[agen_idx(e->key.addr)]; - tex_gen_bump(e->key.addr); - e->rel_agen = agen[agen_idx(e->key.addr)]; + tex_gen_flip(e, 0); } // e, in the GPU-only address window, is written by the GPU in frame fr: the entries its storage runs over are @@ -2473,8 +2500,8 @@ void vk_texture_bucket_stats(char *out, size_t n) { } uint64_t hot = 0; // bumps weighted by the slots they invalidate for (int b = 0; b < AGEN_N; b++) hot += (uint64_t)bump_count[b] * occ[b]; - snprintf(out, n, "%llu bumps (destroy %llu, create %llu, take-back %llu, release %llu; %llu take-backs put their " - "release's generation back instead), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u", + snprintf(out, n, "%llu new generations (destroy %llu, create %llu, take-back %llu, release %llu; %llu releases and " + "take-backs went back to a generation their state had), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u", (unsigned long long)n_bumps, (unsigned long long)n_site[0], (unsigned long long)n_site[1], (unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone, (unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]], From be0a3eda67abfd37801abb05c3abb93e83666ca3 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Mon, 9 Nov 2026 23:05:30 -0500 Subject: [PATCH 2/3] Do not give a texture bucket its old generation for a state that a slot lookup left by changing it. A slot lookup's callers stamp what they keep with the generation read before the lookup. A lookup that takes a released entry back for a slot registered since the release left such a stamp naming the released state, with the entry as its result. When the game released the entry again the bucket came back to that state and got its generation back, the stamp matched, and the slot was handed the released entry although it is stale by then: a re-upload from storage the game may be unmapping. The state such a lookup leaves now loses its remembered generation, so it gets a new one next time and the stamp never matches again. Found in review. Co-Authored-By: Claude Fable 5.2 --- runtime/vk_texture.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index e33ab2c1..e931b411 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -219,6 +219,15 @@ static uint32_t bepoch[AGEN_N]; static uint64_t bsig[AGEN_N]; #define GS_N 8192 static struct { uint64_t sig; uint32_t b, epoch, gen, valid; } gs[GS_N]; +// A slot lookup is resolving (vk_texture_by_id_locked around slot_resolve). The lookup's callers stamp what they keep +// with the generation they read before it: the slot's own cache, the pool's tokens, the descriptor, type and feedback +// memos (vk_exec_draw.c). A lookup that itself changes the state (it takes a released entry back for a slot registered +// since the release) leaves such a stamp naming the state it left, with a result that state would not give: the next +// release of that entry is later than the slot's registration, the slot is stale then (slot_resolve), and the kept +// result would hand out the released entry, whose storage the game may be unmapping. So the state such a lookup +// leaves loses its remembered generation (tex_gen_flip): it gets a new one when the bucket is next in it, and the +// stamp never matches again. +static int in_lookup; static uint32_t gs_at(uint32_t b) { return (uint32_t)(((bsig[b] ^ ((uint64_t)b << 32 | bepoch[b])) * 0x9E3779B97F4A7C15ull) >> 51); @@ -248,6 +257,10 @@ static void tex_gen_bump(uint64_t addr) { // generation never stands for two states. The global generations still move, so their readers look, and find the // tokens equal. plain: the take-back of an entry that was superseded or overwritten while released changes more than // the flag, and is a change like any other. +// One input of slot_resolve is not in the state: the stale rule compares a slot's registration with the entry's +// release (rel_serial), which is later at every release. A slot is stale at every later release once it is at one, so +// what it resolves to in a released state stays right; the one that is not stale takes the entry back, which is the +// lookup-made change in_lookup deals with. static void tex_gen_flip(Tex *e, int plain) { static int on = -1; static uint64_t uids; @@ -258,6 +271,10 @@ static void tex_gen_flip(Tex *e, int plain) { e->uid = (z ^ z >> 31) | 1; } uint32_t b = agen_idx(e->key.addr); + if (in_lookup) { // see in_lookup: the state this lookup leaves is not to get its generation again + typeof(gs[0]) *m0 = &gs[gs_at(b)]; + if (m0->valid && m0->b == b && m0->epoch == bepoch[b] && m0->sig == bsig[b]) m0->valid = 0; + } bsig[b] ^= e->uid; if (!on || plain) { tex_gen_bump(e->key.addr); @@ -2540,7 +2557,9 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { view_t v; int hv; resolve_held = 0; + in_lookup = 1; s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it + in_lookup = 0; s->cview = v, s->chas_view = (uint8_t)hv; s->cgen = g; s->cepoch = 0; From 9728424aec328ad0d15cbe1969ce635e9dba9076 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Mon, 9 Nov 2026 23:42:12 -0500 Subject: [PATCH 3/3] End the wait after a frame's first submission when its passes are through, not its write-backs. Every frame the render thread waited about 5 ms for the GPU after the frame's first submission, with the game CPU-bound in every area. The wait is there so that the GPU has run that submission's passes before the game rewrites what they read. But the submission also ends with the write-backs of what its shaders wrote, 60 MiB merged into guest memory in 3 to 4 ms of GPU time, and the wait covered those too. The write-backs now go into a part of their own. The part before them signals a second timeline, the executed timeline, and the first-submission wait waits on that; the write-back part signals the timeline as before, so every fence the game asks for still covers them, in the same queue order. wb_split = 0 keeps one part and the old wait. Not measured in the game yet. Co-Authored-By: Claude Fable 5.2 --- nx2navi.ini.example | 4 +++ runtime/cfg.c | 6 ++-- runtime/gfx.h | 3 ++ runtime/gfx_null.c | 1 + runtime/nvn_api_core.c | 5 ++- runtime/vk.h | 5 +++ runtime/vk_backend.c | 6 ++++ runtime/vk_exec.c | 7 ++++ runtime/vk_memory.c | 2 ++ runtime/vk_sync.c | 80 ++++++++++++++++++++++++++++++++++++++---- 10 files changed, 108 insertions(+), 11 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index b6a1a00e..9da59324 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -590,6 +590,10 @@ # are made inaccessible until their first read or write, which is logged with its instruction ("mem: write-back # probe"). The bytes are there all along; the probe only watches, at the cost of the faults it takes. # wb_probe = 0 +# 1: what the GPU wrote goes back to the game's memory in a GPU submission of its own, and the wait after each frame's +# first submission ends when that submission's passes are through, without waiting for the copy back (3 to 4 ms of +# every frame). 0: the copy back is part of the submission, and the wait covers it. +# wb_split = 1 # 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every # page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in # Kokiri Forest. diff --git a/runtime/cfg.c b/runtime/cfg.c index c2b024b2..8bcb63f5 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -62,7 +62,7 @@ static const char *const known[] = { "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", - "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", @@ -118,7 +118,7 @@ static const struct { const char *key, *section; } sections[] = { {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, - {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, + {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; @@ -231,7 +231,7 @@ static const CfgMeta metas[] = { {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, - {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, + {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha diff --git a/runtime/gfx.h b/runtime/gfx.h index 2a1b7778..762404de 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -145,6 +145,9 @@ typedef struct { uint64_t (*submit)(const uint64_t *handles, int n); // replays; returns this submission's fence value // (gfx_submit_queue names the game's queue) int (*fence_wait)(uint64_t value, uint64_t timeout_ns); // 1 reached; caller has given its core up + // As fence_wait, but only until the GPU has executed the submission's own commands: its write-backs to guest + // memory and its fence's side effects may follow (vk_sync.c exec_tl). The same as fence_wait where there is no split. + int (*exec_wait)(uint64_t value, uint64_t timeout_ns); uint64_t (*fence_done)(void); // highest value whose side effects are written // Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that // has passed or at_ns is 0. diff --git a/runtime/gfx_null.c b/runtime/gfx_null.c index dd01f281..b73010a4 100644 --- a/runtime/gfx_null.c +++ b/runtime/gfx_null.c @@ -146,6 +146,7 @@ const GfxBackend gfx_null_backend = { .program_release = null_program_release, .submit = null_submit, .fence_wait = null_fence_wait, + .exec_wait = null_fence_wait, .fence_done = null_fence_done, .present = null_present, .finish = null_finish, diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 28aeeb1b..73f5d213 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -660,6 +660,9 @@ NVN_API(nvnQueueSubmitCommands) { // point). Copies of the data at submit made it worse, a mix of two frames' versions. // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits // after every submission, 0 never. + // The wait is for the submission's passes, which is what the rewrite races with: exec_wait leaves out the + // write-backs of what they wrote (vk_sync.c exec_tl), which nothing here needs before the game goes on. The fences + // the game itself waits on cover them as before. static long long finish_mask = -2; if (finish_mask == -2) { const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH"); @@ -667,7 +670,7 @@ NVN_API(nvnQueueSubmitCommands) { if (e && *e && finish_mask == 1 && !(e[0] == '0' && (e[1] == 'x' || e[1] == 'X'))) finish_mask = -1; // "1": all } int idx = nvn_submits_in_frame++; - if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->fence_wait(f, UINT64_MAX); + if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->exec_wait(f, UINT64_MAX); } // The producer (1.2.1 main f_70008c6460) releases the read lock on obj+0x538 at lr 0x70008c70c8; when diff --git a/runtime/vk.h b/runtime/vk.h index 4ca82456..a124ee7f 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -570,6 +570,11 @@ void vk_sync_start(void); // vk_init (fatal without a device) a uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted +// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet. +int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns); +int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own +VkCommandBuffer vk_sync_split_exec(void); // vk_sync_split for the part before a submission's write-backs +int vk_mem_mirror_wb_pending(void); // vk_memory.c: vk_mem_mirror_end has write-backs to record uint64_t vk_sync_fence_done(void); // highest done value uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1) void vk_sync_finish(void); // every submitted value done diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index f5389dfd..a2e715c8 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -183,6 +183,11 @@ int vk_replay_async(void) { return on; } +// With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's. +static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { + return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns); +} + // The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end. int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); } @@ -606,6 +611,7 @@ const GfxBackend gfx_vulkan_backend = { .program_release = vk_be_program_release, .submit = vk_be_submit, .fence_wait = vk_sync_fence_wait, + .exec_wait = vk_be_exec_wait, .fence_done = vk_sync_fence_done, .present = vk_be_present, .frame_start = vk_dlssg_frame_start, diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index bd891bb2..c2f056ae 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1325,6 +1325,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t vk_exec_pass_end(ex); x_here = 0; if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes + // The write-backs in a part of their own, after one that signals the executed timeline (vk_sync.c exec_tl): the + // wait after a frame's first submission then ends when the submission's passes are through, not its write-backs. + if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) { + ex->cb = vk_sync_split_exec(); + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + } if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory if (ex->nviews) { ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ced07ef4..38cfaf51 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2331,6 +2331,8 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) { return j->n + 2; } +int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); } + int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { if (!vk_mirror_on) return 0; int rec = 0; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index cae29b16..6d9e1176 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -62,7 +62,7 @@ static struct { int n; VkSubmitInfo2 si[HOLD_CAP]; VkCommandBufferSubmitInfo cbi[HOLD_CAP][2]; - VkSemaphoreSubmitInfo sig[HOLD_CAP]; + VkSemaphoreSubmitInfo sig[HOLD_CAP][2]; uint64_t last; // the newest timeline value a held submission signals, 0 none uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried } H; @@ -71,6 +71,26 @@ static volatile uint64_t held_t0; // when the oldest held submission was hel static volatile uint64_t queued_v; // the newest timeline value handed to the queue static int hold_max = -1; +// ---- the executed timeline +// A submission's write-backs (vk_memory.c vk_mem_mirror_end: what its shaders wrote into VRAM mirrors goes back to guest +// memory) are the last thing in it, 3 to 4 ms of GPU time a frame in Ocarina, 60 MiB merged. The wait after a frame's +// first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run that submission's passes +// before the game rewrites what they read; it does not need the write-backs, and waited for them too: 5 ms of every +// frame with the render thread idle. +// exec_tl is a second timeline that reaches a submission's value when the GPU has executed its own commands: the +// replay ends the part before the write-backs (vk_exec.c, vk_sync_split_exec), that part signals exec_tl, and the +// write-backs follow in a part of their own, which signals the timeline as before. A submission without that split +// signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among +// them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's. +static VkSemaphore exec_tl; +static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl +static int wb_split = -1; + +int vk_sync_wb_split(void) { + if (wb_split < 0) wb_split = cfg_bool("NX2NAVI_WB_SPLIT"); + return wb_split && exec_tl != VK_NULL_HANDLE; +} + static void hold_flush_locked(VkCtx *c) { if (!H.n) return; VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); @@ -111,7 +131,10 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) { if (si->commandBufferInfoCount) memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount); H.si[i].pCommandBufferInfos = H.cbi[i]; - if (si->signalSemaphoreInfoCount) H.sig[i] = si->pSignalSemaphoreInfos[0], H.si[i].pSignalSemaphoreInfos = &H.sig[i]; + if (si->signalSemaphoreInfoCount) { // at most two: the timeline and the executed timeline (exec_tl) + memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount); + H.si[i].pSignalSemaphoreInfos = H.sig[i]; + } if (value) H.last = value; uint64_t now = kern_now_ns(); if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE); @@ -430,6 +453,12 @@ void vk_sync_start(void) { if (reserved < now) reserved = now; stop = 0; plat_mutex_unlock(&st_lock); + // the executed timeline starts where the timeline is; without it every exec wait is a fence wait + VkSemaphoreTypeCreateInfo et = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO, + .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = now}; + VkSemaphoreCreateInfo ei = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &et}; + if (vkCreateSemaphore(c->device, &ei, NULL, &exec_tl) != VK_SUCCESS) exec_tl = VK_NULL_HANDLE; + exec_sig = now; vk_mem_note_submit(now); completion_thread = plat_thread_new(completion_main, c, "vk completion"); if (!completion_thread) rt_fatal("vk: cannot start the completion thread"); @@ -484,7 +513,12 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { return cb; } -VkCommandBuffer vk_sync_split(void) { +static VkCommandBuffer split_part(int exec); +VkCommandBuffer vk_sync_split(void) { return split_part(0); } +// The part that ends here is the last with the submission's own commands: it signals the executed timeline. +VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); } + +static VkCommandBuffer split_part(int exec) { VkCtx *c = vk_ctx(); VkCommandBuffer cb = slots[cur_slot].cb; vk_gpu_passes_part(cb, 1); @@ -496,6 +530,10 @@ VkCommandBuffer vk_sync_split(void) { {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}}; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1, .pCommandBufferInfos = pro ? cbi : cbi + 1}; + VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, + .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; + if (exec && exec_tl && exec_sig < cur_fence) + si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence; ts_submitted(); queue_submit(c, &si, 0); // held or handed to the queue (submit batching) vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted @@ -556,12 +594,17 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { s = &slots[cur_slot]; VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro}, {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}}; - VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, - .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; + // the timeline, and the executed timeline unless a part of this submission signalled it already (split_part) + VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, + .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}, + {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, + .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}}; + int both = exec_tl && exec_sig < fence; + if (both) exec_sig = fence; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u), - .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1, - .pSignalSemaphoreInfos = &sig}; + .pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = both ? 2u : 1u, + .pSignalSemaphoreInfos = sig}; ts_submitted(); queue_submit(c, &si, fence); // held or handed to the queue (submit batching) s->fence = recorded ? fence : 0; @@ -635,6 +678,27 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) { return ok; } +// Waits until the GPU has executed submission value's own commands (the executed timeline); its write-backs and the +// CPU side effects of its fence may still be to come. 1 when reached. +int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) { + vk_sync_start(); + if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns); + if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; + uint64_t p0 = perf_begin(); + hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now + VkCtx *c = vk_ctx(); + VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1, + .pSemaphores = &exec_tl, .pValues = &value}; + VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns); + if (r != VK_SUCCESS && r != VK_TIMEOUT) { + vk_device_fault_report(r); + rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r); + } + perf_end(PERF_FENCE, p0); + perf_gpu_wait(p0); + return r == VK_SUCCESS; +} + uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); } uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); } @@ -661,6 +725,8 @@ void vk_sync_shutdown(void) { plat_mutex_lock(&sub_lock); VkCtx *c = vk_ctx(); if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL); + if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL); + exec_tl = VK_NULL_HANDLE; perf_gpu_stats = NULL; if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL); ts_pool = VK_NULL_HANDLE;