From 4a954a1c6d401e2ed5c1e8218fdf17d7f89e3e5f Mon Sep 17 00:00:00 2001 From: ordinarybob Date: Tue, 10 Nov 2026 14:09:21 -0500 Subject: [PATCH] Refresh the texture pool from a list of its entries instead of walking every slot, for about 7 % more frames a second where the replay thread is the limit. Each submission's first sync of the bindless texture pool walked every registered slot, 33,000 in Kokiri Forest, to find the few textures the CPU wrote and the few slots that changed. Three parts of that walk change, each with its own switch (pool_fast, pool_presnap, pool_group; on by default, 0 and ab accepted): - The freshness check goes through a list of the entries the slots resolved to, with a check by 2 MiB region first; slots that resolved to nothing are looked up again when the game's memory map changes and every eighth submission. The use stamps the idle eviction reads are left from a count of the slots that hold each entry. - The check takes the write-watch snapshots of its regions first, so unchanged entries pass the region test. - The GPU-written slots of one address bucket with one descriptor form a group, which the dirty walk leaves alone while its first member is unchanged. Measured on an RTX 5090 at 2560x1440, standing in Kokiri Forest, in-run A/B over five runs with all three switched (three of them with the barrier change switched too): 93.6 to 100.7, 96.4 to 103.1, 95.8 to 102.6, 99.3 to 107.1 and 91.9 to 99.9 frames a second, the replay thread 0.7 to 0.8 ms a frame shorter; a control's blocks differ by 0.4 %. On a walk with NX2NAVI_POOL_GROUP_CHECK=1 no group that was left alone had missed a write, and screenshots across the A and B blocks show the same picture. From pull request #390. Not tried: the texture eviction under memory pressure (a 9 GiB budget was never reached on the walk), AMD and Intel drivers, Linux. One of the nine runs with it on ended by itself with exit code 0 and nothing in its log two seconds after the window lost the focus; three repeats of that run did not. Co-Authored-By: Claude Fable 5.2 --- runtime/cfg.c | 1 + runtime/host.h | 1 + runtime/memory.c | 9 ++ runtime/vk.h | 36 ++++- runtime/vk_exec_draw.c | 261 ++++++++++++++++++++++++++++-- runtime/vk_memory.c | 37 +++++ runtime/vk_texture.c | 352 ++++++++++++++++++++++++++++++++++++++--- 7 files changed, 663 insertions(+), 34 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index 115377e2..31cf1cca 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -54,6 +54,7 @@ static const char *const known[] = { "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", + "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", diff --git a/runtime/host.h b/runtime/host.h index e77c8a6f..e8af6dde 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -161,6 +161,7 @@ uint32_t mem_map(uint64_t dst, uint64_t src, uint64_t n); uint32_t mem_unmap(uint64_t dst, uint64_t src, uint64_t n); // 0x05 uint32_t mem_query(uint64_t a, RtMemoryInfo *out); // 0x06 int mem_readable(uint64_t a, uint64_t n); // every page readable +uint32_t mem_map_gen(void); // changes with everything mem_readable answers from (mapping, heap, permissions) int mem_read_hold(uint64_t a, uint64_t n); // mem_readable that keeps the range mapped until mem_read_release void mem_read_release(void); uint32_t mem_map_physical(uint64_t a, uint64_t n); // 0x2C diff --git a/runtime/memory.c b/runtime/memory.c index 21983d6b..c8bff935 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -683,6 +683,11 @@ static uint64_t image_bytes(void) { static int overlaps(uint64_t a, uint64_t n, uint64_t b, uint64_t m) { return a < b + m && b < a + n; } static int inside(uint64_t a, uint64_t n, RtRange r) { return a >= r.base && n <= r.size && a - r.base <= r.size - n; } +// Bumped (lock held exclusively) by every change of what is mapped or readable: vk_exec_draw.c's texture pool looks the +// slots over unreadable storage up again only then (and now and then). +static uint32_t map_gen; +uint32_t mem_map_gen(void) { return __atomic_load_n(&map_gen, __ATOMIC_ACQUIRE); } + // ---- 0x01 SetHeapSize uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { plat_mutex_lock(&lock); @@ -693,6 +698,7 @@ uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { if (size > heap_size) commit(L.heap.base + heap_size, size - heap_size, "heap"); if (size < heap_size) decommit(L.heap.base + size, heap_size - size, "heap"); heap_size = size; + __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); if (addr) *addr = L.heap.base; return R_OK; @@ -749,6 +755,7 @@ uint32_t mem_set_permission(uint64_t a, uint64_t n, uint32_t perm) { } memset(t, (int)perm, n / PAGE); protect(a, n, perm == PERM_RW ? PLAT_PROT_RW : perm == PERM_R ? PLAT_PROT_READ : PLAT_PROT_NONE, "module data"); + __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); rt_log("%s data permission +%#llx size %#llx perm %u", m->name, (unsigned long long)(a - m->base), (unsigned long long)n, perm); @@ -820,6 +827,7 @@ static uint32_t stack_move(int unmap, uint64_t dst, uint64_t src, uint64_t n) { stack_maps[nstack_maps++] = (StackMap){dst, src, n}; } out: + __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); return r; } @@ -872,6 +880,7 @@ static uint32_t alias_map(int unmap, uint64_t a, uint64_t n) { alias_bytes += fresh * PAGE; } out: + __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); if (r) rt_log("%sMapPhysicalMemory %#llx+%#llx failed %#x (alias %#llx used, heap %#llx)", unmap ? "Un" : "", (unsigned long long)a, (unsigned long long)n, r, (unsigned long long)alias_bytes, diff --git a/runtime/vk.h b/runtime/vk.h index 55147b61..a70d5d21 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -565,8 +565,42 @@ uint32_t vk_texture_entry_gen(void); // vk_texture.c: bumped by // Slots whose token may have changed since *pos (0 at first; then the value these calls leave): fn is called for each // (some more than once) and *pos advanced. 0 when too many changes came since: the caller looks at every slot. *pos // from vk_texture_dirty_pos_locked after such a walk. Texture lock held. -int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg); +// gfn, when given, gets each slot group of a bucket in place of fn for its members. +int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void (*gfn)(int g, void *arg), void *arg); +// Slot groups (vk_texture.c): the GPU-written slots of one address bucket with one descriptor hash kh, which resolve +// alike. head the first member, vk_texture_group_next_locked the next (-1: the end), n how many; stamp and fmask the +// texture pool's (vk_exec_draw.c pool_dirty_group: 1 when every member holds the same views, of the kinds fmask; a slot +// that joins goes to the tail, refilled on its own), set 0 by vk_texture_groups_unstamp_locked. Texture lock held. +typedef struct { uint64_t kh; int32_t head, n; uint32_t stamp, fmask; } VkSlotGroup; +VkSlotGroup *vk_texture_group_locked(int g); +int vk_texture_group_next_locked(int id); +void vk_texture_groups_unstamp_locked(void); uint64_t vk_texture_dirty_pos_locked(void); +// The bindless pool's entries (vk_texture.c): after a pool fill of slot id, notes the entry it resolved to (1; 0: none) +// and gives its number for vk_texture_pool_hold_locked in *ent (0: none); a submission's freshness check of every +// noted entry (returns how many). Texture lock held. +int vk_texture_pool_note_locked(int id, uint32_t *ent); +int vk_texture_pool_fresh_locked(void); +// A pool slot that held the entry old (0: none) now holds ent: the entries slots hold get the use stamps a lookup +// leaves (vk_texture.c touch) from the check by list. Texture lock held. +void vk_texture_pool_hold_locked(uint32_t old, uint32_t ent); +// A slot whose last lookup found no entry, looked up again: 1 it resolves now; 0 still none, its storage may become +// readable; -1 none until its token changes (a CPU slot whose texture was released after its registration, no storage). +int vk_texture_slot_retry_locked(int id); +// vk_memory.c, for the pool's freshness check: the generation table's lock, shared; with it held, 1 when every 2 MiB +// region of [addr, addr+size) has this epoch's write-watch snapshot and no page stamped after since (nothing in it can +// have changed), 0 when a closer look is due (vk_mem_changed_snap). +void vk_mem_gen_lock_shared(void); +void vk_mem_gen_unlock_shared(void); +int vk_mem_unchanged_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch); +// The same check's reason when it fails: 1 a region without this epoch's snapshot yet, 2 one untracked, 3 one written +// since, 4 nothing to check (no size, beyond the regions); 0 unchanged. +int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch); +// The lock exclusively, for the snapshot of the regions of [addr, addr+size) that have none in epoch yet (the number of +// write-watch queries it took). +void vk_mem_gen_lock(void); +void vk_mem_gen_unlock(void); +uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch); void vk_sampler_dlss_bias(float bias); // texture LOD bias added for DLSS upscaling (material samplers) void vk_sampler_aniso_live(void); // each present: NX2NAVI_ANISO after a settings change (tiled samplers) uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 0c611fd9..36fd1c4b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1743,6 +1743,9 @@ static int tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int stor // absent from that submit's page tables, a GPU page fault on the 128x128 probe pass of Kokiri Forest's first frame // (RADV's radv_wait_for_vm_map_updates hid it, as did NX2NAVI_SPLIT_DRAWS=0). volatile uint32_t vk_pool_kinds_used; +// Per texture slot, what pool_fill keeps between refills: the descriptor hash of the registration it was read under, +// and the lookup memo stamp its entries were last written from (0: none) with the kinds that write covered. +typedef struct { uint64_t kh; uint32_t kreg, stamp, fmask; } PoolSlot; #define POOL_SETS VK_POOL_SETS #define POOL_LOG (1u << 20) // entries of the change log: binding * VK_POOL_TEX + id, one per write static struct { @@ -1758,7 +1761,24 @@ static struct { uint64_t logn; // writes so far uint64_t *last; // per entry: the log position of its newest write (catch-up writes it once, there) uint64_t switches, waits, catchup, written, rewrites, catch_ns, sync_ns, n_slots, n_refill, n_fresh; + uint64_t n_walk[2], walk_visit[2], walk_refill[2], walk_ns[2]; // dirty walks: at a submission's start, within one + uint64_t n_visit; // slots the dirty walks looked at + uint64_t n_gwalk, n_gskip, n_gskip_slots, n_grep; // POOL_GROUP: groups walked, left alone and their slots, first + // members looked up for the memo + uint64_t n_gchk_writes, n_gchk_groups; // POOL_GROUP_CHECK: writes the groups left alone got all the same, and in how many + uint64_t n_gchk_pending, n_gchk_bad; // of members registered since their last fill (not counted above); this group's + uint64_t n_gnouni, n_gchanged; // groups walked member by member: not known to hold one resolution, first member changed + int gmode; // pool_group() as of the last sync uint64_t n_by_reg, n_by_addr; // refills whose token changed by a registration, by an address generation only + uint64_t full_ns, n_full; // the walks over every slot (a submission's first sync) and their time + uint64_t ep_ns, n_ep, n_ep_ent; // POOL_FAST: a submission's freshness check of the noted entries instead + uint64_t unres_ns, n_unres_walks; // of it: the walks over the slots without an entry + uint32_t unres_mg; // mem_map_gen of the last such walk + int unres_more; // it left a lookup for the creation budget: the next submission walks again + PoolSlot *ps; // [id] + uint8_t *unres_on; // [id]: its last fill resolved to no entry (POOL_FAST: retried each submission) + int *unres, nunres, cunres; // those slots (some may have left: unres_on clear) + uint32_t *ent; // [id]: the entry its last fill resolved to, held there (pool_hold); 0: none uint64_t dirty_pos; // vk_texture_dirty_slots_locked position (0: the next sync walks every slot) int ready, uab, sealed; // sealed: a part of the current submission already reads sets[cur] uint32_t kinds, gen, epoch, sgen; @@ -1836,7 +1856,10 @@ static VkDescriptorSetLayout pool_layout_make(void) { PL.tok = calloc(VK_POOL_TEX, sizeof *PL.tok); PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view); PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp); - if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory"); + PL.ps = calloc(VK_POOL_TEX, sizeof *PL.ps); + PL.unres_on = calloc(VK_POOL_TEX, 1); + PL.ent = calloc(VK_POOL_TEX, sizeof *PL.ent); + if (!PL.tok || !PL.view || !PL.samp || !PL.ps || !PL.unres_on || !PL.ent) rt_fatal("vk: out of memory"); rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)%s", VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP, PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets, @@ -2071,6 +2094,7 @@ uint64_t vk_pool_takes(void) { return __atomic_load_n(&pool_takes, __ATOMIC_ACQU // The current set gets the entries written into other sets in [from, to) of the log, as PL.view / PL.samp hold them now. static uint64_t pool_memo_hits; // pool_fill: slots that shared another slot's lookup (defined with pool_memo below) +static uint64_t pool_memo_same; // of those, slots whose entries held the shared views already static void pool_catch_up(uint64_t from, uint64_t to) { VkDescriptorSet set = PL.set; @@ -2089,23 +2113,47 @@ static void pool_catch_up(uint64_t from, uint64_t to) { pool_flush(); PL.switches++; if (PL.switches % 3600 == 0) { // diagnosis: what invalidates the slots (vk_texture.c vk_texture_bucket_stats) - char bs[320]; + char bs[768]; vk_texture_bucket_stats(bs, sizeof bs); rt_log("vk: texture pool buckets over 3600 switches: %s", bs); } if (PL.switches % 3600 == 0) rt_log("vk: texture pool sets: %llu switches, %llu waited for the GPU, %.1f log entries and %.1f writes per " - "catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.0f slots, %.0f " - "refilled: %.0f registered, %.0f address changes, %.0f shared a lookup; %.0f checked); %llu identical registrations kept; " + "catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.3f ms of it in " + "%.2f walks over every slot, %.3f ms in %.2f checks of %.0f listed entries and %d slots without one " + "(%.3f ms in %.2f walks over those); " + "%.0f slots, %.0f refilled: %.0f registered, %.0f address changes, %.0f " + "shared a lookup, %.0f of them holding its views already; %.0f checked); %llu identical registrations kept; " "%llu refreshes created nothing for their frame's time (pool_create_ms), %llu lookups of slots left in the frame skipped", (unsigned long long)PL.switches, (unsigned long long)PL.waits, (double)PL.catchup / (double)PL.switches, (double)PL.written / (double)PL.switches, (unsigned long long)PL.rewrites, PL.catch_ns / 1e6 / (double)PL.switches, PL.sync_ns / 1e6 / (double)PL.switches, + PL.full_ns / 1e6 / (double)PL.switches, (double)PL.n_full / (double)PL.switches, + PL.ep_ns / 1e6 / (double)PL.switches, (double)PL.n_ep / (double)PL.switches, + PL.n_ep ? (double)PL.n_ep_ent / (double)PL.n_ep : 0.0, PL.nunres, + PL.unres_ns / 1e6 / (double)PL.switches, (double)PL.n_unres_walks / (double)PL.switches, (double)PL.n_slots / (double)PL.switches, (double)PL.n_refill / (double)PL.switches, (double)PL.n_by_reg / (double)PL.switches, (double)PL.n_by_addr / (double)PL.switches, - (double)pool_memo_hits / (double)PL.switches, + (double)pool_memo_hits / (double)PL.switches, (double)pool_memo_same / (double)PL.switches, (double)PL.n_fresh / (double)PL.switches, (unsigned long long)vk_texture_same_regs(), (unsigned long long)PL.n_frame_stops, (unsigned long long)PL.n_defer_skips); + if (PL.switches % 3600 == 0) + rt_log("vk: texture pool dirty walks per switch: %.2f at a submission's start (%.0f slots looked at, %.0f " + "refilled, %.3f ms), %.2f within one (%.0f looked at, %.0f refilled, %.3f ms)", + (double)PL.n_walk[0] / (double)PL.switches, (double)PL.walk_visit[0] / (double)PL.switches, + (double)PL.walk_refill[0] / (double)PL.switches, PL.walk_ns[0] / 1e6 / (double)PL.switches, + (double)PL.n_walk[1] / (double)PL.switches, (double)PL.walk_visit[1] / (double)PL.switches, + (double)PL.walk_refill[1] / (double)PL.switches, PL.walk_ns[1] / 1e6 / (double)PL.switches); + if (PL.switches % 3600 == 0) + rt_log("vk: texture pool slot groups per switch (NX2NAVI_POOL_GROUP): %.0f walked, %.0f left alone with %.0f " + "slots, %.0f first members refilled (%.0f changed), %.0f not known to hold one resolution; check " + "(NX2NAVI_POOL_GROUP_CHECK): %llu writes in %llu groups left alone so far, %llu more to members registered " + "since their last fill", + (double)PL.n_gwalk / (double)PL.switches, (double)PL.n_gskip / (double)PL.switches, + (double)PL.n_gskip_slots / (double)PL.switches, (double)PL.n_grep / (double)PL.switches, + (double)PL.n_gchanged / (double)PL.switches, (double)PL.n_gnouni / (double)PL.switches, + (unsigned long long)PL.n_gchk_writes, (unsigned long long)PL.n_gchk_groups, + (unsigned long long)PL.n_gchk_pending); } // Entry id of every kind in mask: the slot's view of that kind or the kind's stand-in. Texture lock held. 0: the @@ -2151,22 +2199,75 @@ static int pool_on(void) { // transient render target registered over its storage, and every finalize and initialize of that target changes their // tokens, so a refill looked up the same texture ~400 times, ~15,000 lookups a frame in Kokiri Forest. An entry holds the // views one slot got, for the slots with its hash, address generation and entry generation. +// Its stamp changes only when one of its views does (NX2NAVI_POOL_FAST, default on): a slot whose entries were written +// from the entry at the stamp it has now (PoolSlot) holds those views already, so a refill that finds the entry so +// compares and writes nothing. That is most refills: a transient render target released and taken back moves the +// address generation of its storage, ~1,300 slots in Kokiri Forest, and every slot resolves as before (18,500 of 18,800 +// refills a submission shared a lookup, each comparing up to 16 views a quarter megabyte apart: 1.9 ms). #define POOL_MEMO 4096 -static struct { uint64_t kh; uint32_t agen, egen, mask; VkImageView view[VK_POOL_KINDS]; } pool_memo[POOL_MEMO]; -static uint64_t pool_memo_hits; +static struct { uint64_t kh; uint32_t agen, egen, mask, stamp, got, ent; VkImageView view[VK_POOL_KINDS]; } pool_memo[POOL_MEMO]; +static uint32_t pool_stamp; // the last stamp given out +static uint64_t pool_memo_hits, pool_memo_same; + +static int pool_fast(void) { // ab: in the B blocks + static int on = -1; + if (on < 0) on = rt_knob("POOL_FAST", "1"); + return RT_KNOB_ON(on); +} + +// Slot id's last fill resolved to the entry ent (vk_texture_pool_note_locked; 0: none). vk_texture.c counts the slots +// that hold each entry: the entries held are the ones the walk over every slot reached with a lookup in each +// submission, and the check by list leaves their use stamps from that count (vk_texture_pool_hold_locked). +static void pool_hold(uint32_t id, uint32_t ent) { + if (PL.ent[id] == ent) return; + vk_texture_pool_hold_locked(PL.ent[id], ent); + PL.ent[id] = ent; +} + +// Slot id's last fill resolved to no entry (got 0) or to one: the slots without one are looked up again each +// submission (pool_sync_entries), as the walk over every slot did: an entry their storage could not have then may be +// made now, and its address generation then refills them. +static void pool_unres(uint32_t id, int got) { + if (got || PL.unres_on[id]) { + if (got) PL.unres_on[id] = 0; + return; + } + PL.unres_on[id] = 1; + if (PL.nunres == PL.cunres) { + int n = PL.cunres ? PL.cunres * 2 : 1024; + int *p = realloc(PL.unres, sizeof *p * (size_t)n); + if (!p) rt_fatal("vk: out of memory"); + PL.unres = p, PL.cunres = n; + } + PL.unres[PL.nunres++] = (int)id; +} static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) { vk_texture_create_deferred_locked(); - uint64_t kh = tok ? vk_texture_slot_khash_locked((int)id) : 0; + PoolSlot *ps = &PL.ps[id]; + uint64_t kh = 0; + if (tok) { // the descriptor hash changes only with a registration (the token's upper half) + uint32_t reg = (uint32_t)(tok >> 32); + if (ps->kreg != reg || !ps->kh) ps->kh = vk_texture_slot_khash_locked((int)id), ps->kreg = reg; + kh = ps->kh; + } uint32_t ag = (uint32_t)tok; typeof(pool_memo[0]) *m = kh ? &pool_memo[(kh >> 20) % POOL_MEMO] : NULL; if (m && m->kh == kh && m->agen == ag && m->egen == vk_texture_entry_gen() && (m->mask & mask) == mask) { + pool_memo_hits++; + pool_unres(id, (int)m->got); + pool_hold(id, m->ent); + if (ps->stamp && ps->stamp == m->stamp && (ps->fmask & mask) == mask && pool_fast()) { + pool_memo_same++; + return 1; + } for (int k = 0; k < VK_POOL_KINDS; k++) { if (!(mask >> k & 1)) continue; VkImageView *e = &PL.view[(size_t)k * VK_POOL_TEX + id]; if (*e != m->view[k]) *e = m->view[k], pool_write((uint32_t)k, id, m->view[k], VK_NULL_HANDLE); } - pool_memo_hits++; + ps->fmask = ps->stamp == m->stamp ? ps->fmask | mask : mask; + ps->stamp = m->stamp; return 1; } for (int k = 0; k < VK_POOL_KINDS; k++) { @@ -2180,12 +2281,29 @@ static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) { if (*e != ii.imageView) *e = ii.imageView, pool_write((uint32_t)k, id, ii.imageView, VK_NULL_HANDLE); } int ok = !vk_texture_create_deferred_locked(); + uint32_t ent = 0; + int got = vk_texture_pool_note_locked((int)id, &ent); + if (tok) pool_unres(id, got); + pool_hold(id, tok ? ent : 0); + ps->stamp = 0; // its entries match no memo entry, unless the one below if (m && ok) { // the generation after the lookups: an entry they created is part of what they resolved uint32_t eg = vk_texture_entry_gen(); - if (m->kh != kh || m->agen != ag || m->egen != eg) m->kh = kh, m->agen = ag, m->egen = eg, m->mask = 0; + // a view that changes gives the entry a new stamp; one kept outside the mask keeps its value, so a kind that + // comes back with another view is a change too + int changed = !m->stamp; for (int k = 0; k < VK_POOL_KINDS; k++) - if (mask >> k & 1) m->view[k] = PL.view[(size_t)k * VK_POOL_TEX + id]; + if (mask >> k & 1) { + VkImageView v = PL.view[(size_t)k * VK_POOL_TEX + id]; + if (m->view[k] != v) m->view[k] = v, changed = 1; + } + if (m->kh != kh || m->agen != ag || m->egen != eg) m->kh = kh, m->agen = ag, m->egen = eg, m->mask = 0; m->mask |= mask; + m->got = (uint32_t)got, m->ent = ent; + if (changed) { + if (!++pool_stamp) pool_stamp = 1; + m->stamp = pool_stamp; + } + ps->stamp = m->stamp, ps->fmask = mask; } return ok; } @@ -2237,6 +2355,7 @@ static int pool_refill(VkExec *ex, uint32_t id, uint32_t used, uint64_t t, int s static void pool_dirty_slot(int id, void *arg) { // vk_texture_dirty_slots_locked: a slot whose token may have changed PoolDirty *d = arg; + PL.n_visit++; if (id >= d->n) return; uint64_t t = vk_texture_slot_token(id); if (t == PL.tok[id]) return; @@ -2253,6 +2372,76 @@ static int pool_current(void) { vk_texture_slots_gen() == PL.gen && vk_texture_epoch() == PL.epoch && vk_sampler_gen() == PL.sgen; } +// NX2NAVI_POOL_GROUP=1 (default), 0, ab: a dirty bucket's slot groups (vk_texture.c) are walked whole. The members of a +// group resolve alike, so once a walk of the group brought every member to the current resolution (every fill done, an +// entry for each) they all hold the same views, and later the group is left alone when its first member still holds the +// current resolution: its token did not change (its views are what that token resolves to), or its refill wrote nothing. +// A slot that joined since is at the tail and refilled on its own (its registration's entry on the dirty ring). +// The members keep their older tokens: their views are what those tokens resolved to, and what the current ones do. +// (Comparing the memo's stamps instead left half the groups to their members: the memo's entries are shared between +// hashes, so the same views came back under a new stamp.) +static int pool_group(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("POOL_GROUP", "1"); + return RT_KNOB_ON(mode); +} + +// NX2NAVI_POOL_GROUP_CHECK=1: a group left alone is walked member by member all the same, and the writes that walk +// makes are counted (none when the rule holds; the pool log's slot groups line), but for the members registered since +// their last fill: their registration's entry on the dirty ring refills them in the same walk (counted apart). +static int pool_group_check(void) { + static int on = -1; + if (on < 0) on = rt_knob("POOL_GROUP_CHECK", "0") == 1; // a diagnosis: no settings row + return on; +} + +static void pool_dirty_group(int g, void *arg) { + PoolDirty *d = arg; + uint64_t chk = 0; + VkSlotGroup *G = vk_texture_group_locked(g); + int rep = G->head; + PL.n_gwalk++; + if (!G->stamp) PL.n_gnouni++; + if (G->stamp && (G->fmask & d->used) == d->used && rep >= 0 && rep < d->n && !PL.unres_on[rep]) { + uint64_t t = vk_texture_slot_token(rep); + int same = t == PL.tok[rep]; + if (!same) { // the first member's refill: nothing written, an entry found, its token taken + uint64_t l0 = PL.logn; + PL.n_grep++; + pool_dirty_slot(rep, arg); + same = PL.tok[rep] == t && PL.logn == l0 && !PL.unres_on[rep]; + if (!same) PL.n_gchanged++; + } + if (same) { + PL.n_gskip++, PL.n_gskip_slots += (uint64_t)G->n; + if (!pool_group_check()) return; + chk = PL.logn + 1; + } + } + int r0 = d->retry, uni = 1; // member by member: then all hold the current resolution, unless a fill was left + d->retry = 0; + for (int id = G->head; id >= 0; id = vk_texture_group_next_locked(id)) { + if (id >= d->n) continue; // outside the pool + if (chk) { // the check: a member registered since its last fill is refilled by its own entry anyway + uint64_t l0 = PL.logn; + int pending = vk_texture_slot_token(id) >> 32 != PL.tok[id] >> 32; + pool_dirty_slot(id, arg); + if (PL.logn != l0) { + if (pending) PL.n_gchk_pending += PL.logn - l0; + else PL.n_gchk_writes += PL.logn - l0, PL.n_gchk_bad++; + } + } else { + pool_dirty_slot(id, arg); + } + if (PL.unres_on[id]) uni = 0; + } + if (d->retry) uni = 0; + d->retry |= r0; + G = vk_texture_group_locked(g); + G->stamp = (uint32_t)uni, G->fmask = uni ? d->used : 0; + if (chk && PL.n_gchk_bad) PL.n_gchk_groups++, PL.n_gchk_bad = 0; +} + static void pool_sync_entries(VkExec *ex) { uint32_t used = __atomic_load_n(&vk_pool_kinds_used, __ATOMIC_RELAXED); uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); @@ -2277,12 +2466,51 @@ static void pool_sync_entries(VkExec *ex) { vk_texture_create_budget_locked(spent ? 0 : pool_create_budget()); PL.n_frame_stops += (uint64_t)spent; // Only a slot change since the last sync (no new kinds, no new submission, nothing left over): the slots that - // changed (vk_texture_dirty_slots_locked), not the whole table + // changed (vk_texture_dirty_slots_locked), not the whole table. With NX2NAVI_POOL_FAST (default on) a new + // submission too: its freshness check goes through the entries the slots resolved to and the slots that + // resolved to none (vk_texture_pool_fresh_locked, pool_unres), not every slot. PoolDirty pd = {ex, used, n, 0, spent}; - int walked = !fresh && ep == PL.epoch && !PL.retry && PL.dirty_pos && - vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, &pd); + int newep = ep != PL.epoch; + uint64_t v0 = PL.n_visit, f0 = PL.n_refill, w0 = kern_now_ns(); + int grp = pool_group(); + if (grp && !PL.gmode) vk_texture_groups_unstamp_locked(); // members were refilled one by one meanwhile + PL.gmode = grp; + int walked = !fresh && (!newep || pool_fast()) && !PL.retry && PL.dirty_pos && + vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, grp ? pool_dirty_group : NULL, &pd); + if (walked) { + int at = newep ? 0 : 1; + PL.n_walk[at]++, PL.walk_visit[at] += PL.n_visit - v0, PL.walk_refill[at] += PL.n_refill - f0; + PL.walk_ns[at] += kern_now_ns() - w0; + } retry = pd.retry; + if (walked && newep) { + uint64_t te = kern_now_ns(); + PL.n_ep_ent += (uint64_t)vk_texture_pool_fresh_locked(); + // The slots without an entry are over storage the guest has not mapped (stale descriptors, 1,160 in + // Kokiri Forest): looked up again when the mapping changed, while a lookup is left for the creation budget, + // and every 8th submission for the rest (an entry whose creation failed, an image at their address that + // took GPU content since: slot_resolve's alias rule) + uint32_t mg = mem_map_gen(); + uint64_t tu = kern_now_ns(); + int walk_unres = mg != PL.unres_mg || PL.unres_more || !(PL.n_ep & 7); + if (walk_unres) PL.unres_mg = mg, PL.n_unres_walks++, vk_texture_create_deferred_locked(); + for (int i = 0; walk_unres && i < PL.nunres;) { + int id = PL.unres[i]; + uint64_t t = PL.unres_on[id] ? vk_texture_slot_token(id) : 0; + int r = t && t == PL.tok[id] ? vk_texture_slot_retry_locked(id) : -1; + if (r) { // resolves now (an entry it made changes its token: refilled), was refilled since, or + PL.unres_on[id] = 0; // resolves to none until its token changes (the refill lists it again) + PL.unres[i] = PL.unres[--PL.nunres]; + continue; + } + i++; + } + if (walk_unres) PL.unres_more = vk_texture_create_deferred_locked(); + PL.unres_ns += kern_now_ns() - tu; + PL.n_ep++, PL.ep_ns += kern_now_ns() - te; + } if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked(); + uint64_t tw = walked ? 0 : kern_now_ns(); for (int id = 0; id < n && !walked; id++) { // slots past n were never registered: stand-ins since the first sync uint64_t t = vk_texture_slot_token(id); if (t != PL.tok[id]) { @@ -2298,7 +2526,10 @@ static void pool_sync_entries(VkExec *ex) { vk_texture_by_id_locked(id, NULL, NULL); // the submit's freshness check } } - if (!walked) PL.n_slots += (uint64_t)n; + if (!walked) { + PL.n_slots += (uint64_t)n, PL.n_full++, PL.full_ns += kern_now_ns() - tw; + vk_texture_groups_unstamp_locked(); // the walk over every slot refilled members one by one + } vk_texture_create_budget_locked(-1); } PL.retry = retry != 0; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index f2ee73a1..e730abea 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -914,6 +914,43 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e return untracked ? -1 : changed; } +void vk_mem_gen_lock_shared(void) { plat_mutex_lock_shared(&gen_lock); } +void vk_mem_gen_unlock_shared(void) { plat_mutex_unlock_shared(&gen_lock); } + +void vk_mem_gen_lock(void) { plat_mutex_lock(&gen_lock); } +void vk_mem_gen_unlock(void) { plat_mutex_unlock(&gen_lock); } + +int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch) { + if (!size || !epoch) return 4; + uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; + if (r1 >= SNAP_REGIONS) return 4; + for (uint64_t r = r0; r <= r1; r++) { // REG_SHIFT is SNAP_SHIFT: reg_max is the newest stamp in the region + if (snap_at[r] != epoch) return 1; + if (snap_untracked[r]) return 2; + if (reg_max[r] > since) return 3; + } + return 0; +} + +int vk_mem_unchanged_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch) { + return !vk_mem_unchanged_why_locked(addr, size, since, epoch); +} + +uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch) { + if (!size || !epoch) return 0; + uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; + if (r1 >= SNAP_REGIONS) return 0; + for (uint64_t r = r0; r <= r1; r++) + if (snap_at[r] != epoch) { + uint64_t q = snap_regions(r0, r1, epoch); + n_snap_calls += q; + return q; + } + return 0; +} + void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) { *checks = __atomic_exchange_n(&n_snap_checks, 0, __ATOMIC_RELAXED); *calls = __atomic_exchange_n(&n_snap_calls, 0, __ATOMIC_RELAXED); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index e9fc6390..87a9cc4b 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -89,6 +89,8 @@ typedef struct Tex { uint64_t ow_frame; // gfx_counters.frames when overwritten was set uint8_t overwritten; // released, and a texture at another address was drawn over its storage (overwrite_others) uint8_t in_gwin; // in gwin[] + int pidx; // 1 + its index in pent[] (vk_texture_pool_note_locked); 0: not listed + int ph; // 1 + its index in pref[] (pool_ref); 0: none } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the @@ -296,32 +298,135 @@ static void tex_gen_flip(Tex *e, int plain) { dirty_push(DIRTY_BUCKET | b); } -// Moves slot id into bucket b's list (the lists exist once a slot was registered: vk_texture_register). -static void slot_rebucket(int id, uint32_t b) { +// Slot groups: the GPU-written slots of one address bucket that share a descriptor hash (TexSlot.khash) resolve alike +// under the same address and entry generations, as the texture pool's memo has it (vk_exec_draw.c pool_fill). Ocarina +// keeps up to ~400 descriptors of each transient render target registered over its storage, and the fullest bucket, +// 1,281 slots, moves 8 times a submission in Kokiri Forest: 12,800 of the 17,600 slots a submission's dirty walks +// refilled held their views already. The dirty walk can hand each group to the pool whole (vk_texture_dirty_slots_locked's +// gfn), which leaves its members alone when their resolution did not change (NX2NAVI_POOL_GROUP). A slot is in its +// bucket's own list (khash 0: a CPU-written texture) or in its group's member list, through the same links. A slot that +// joins a group goes to its tail and leaves the group's stamp as it was: its registration put it on the dirty ring, so +// the walk that follows refills it on its own, and the head, whose state the pool tests, is a member from before. (The +// game's 118 registrations a submission cleared the stamps of the groups they joined, and the walk at a submission's +// start went member by member as before: 11,300 of 11,550 slots.) +typedef struct { VkSlotGroup pub; int32_t gnext, gprev, tail; uint32_t bucket; } SlotGroup; +static SlotGroup *grp; +static int ngrp, cgrp, grp_free = -1, ngrp_live; +static uint64_t n_grouped; // slots in groups +static int32_t bucket_ghead[AGEN_N]; // per bucket: its first group (-1: none; set with the lists) +static int32_t *slot_grp; // per slot id: its group, -1 none +typedef struct { uint64_t kh, b; } GrpKey; +static nvn_hmap grp_by = {.klen = sizeof(GrpKey)}; // -> group + 1 + +static int grp_get(uint64_t kh, uint32_t b) { + GrpKey k = {kh, b}; + void *v = nvn_hm_get(&grp_by, &k); + if (v) return (int)((intptr_t)v - 1); + int g; + if (grp_free >= 0) { + g = grp_free, grp_free = grp[g].gnext; + } else { + if (ngrp == cgrp) { + int n = cgrp ? cgrp * 2 : 1024; + SlotGroup *p = realloc(grp, sizeof *p * (size_t)n); + if (!p) rt_fatal("vk: out of memory for texture slot groups"); + grp = p, cgrp = n; + } + g = ngrp++; + } + SlotGroup *G = &grp[g]; + memset(G, 0, sizeof *G); + G->pub.kh = kh, G->pub.head = G->tail = -1, G->bucket = b; + G->gprev = -1, G->gnext = bucket_ghead[b]; + if (G->gnext >= 0) grp[G->gnext].gprev = g; + bucket_ghead[b] = g; + nvn_hm_put(&grp_by, &k, (void *)(intptr_t)(g + 1)); + ngrp_live++; + return g; +} + +static void grp_drop(int g) { + SlotGroup *G = &grp[g]; + GrpKey k = {G->pub.kh, G->bucket}; + nvn_hm_del(&grp_by, &k); + if (G->gprev >= 0) grp[G->gprev].gnext = G->gnext; + else bucket_ghead[G->bucket] = G->gnext; + if (G->gnext >= 0) grp[G->gnext].gprev = G->gprev; + G->pub.kh = 0, G->pub.head = G->tail = -1, G->pub.n = 0, G->pub.stamp = 0; + G->gnext = grp_free, grp_free = g; + ngrp_live--; +} + +// Places slot id in bucket b: at the tail of its group when it has a descriptor hash, else at the head of the bucket's +// own list (the lists exist once a slot was registered: vk_texture_register). +static void slot_place(int id, uint32_t b, uint64_t kh) { if (slot_prev[id] != -2) { // out of its old list + int32_t g = slot_grp[id]; + int32_t *head = g >= 0 ? &grp[g].pub.head : &bucket_head[slot_bucket[id]]; if (slot_prev[id] >= 0) slot_next[slot_prev[id]] = slot_next[id]; - else bucket_head[slot_bucket[id]] = slot_next[id]; + else *head = slot_next[id]; if (slot_next[id] >= 0) slot_prev[slot_next[id]] = slot_prev[id]; + else if (g >= 0) grp[g].tail = slot_prev[id]; + slot_grp[id] = -1; + if (g >= 0) { + n_grouped--; + if (!--grp[g].pub.n) grp_drop(g); + } } slot_bucket[id] = (uint16_t)b; + if (kh) { + int g = grp_get(kh, b); // may move grp + SlotGroup *G = &grp[g]; + slot_grp[id] = g; + G->pub.n++, n_grouped++; + slot_next[id] = -1; + slot_prev[id] = G->tail; + if (G->tail >= 0) slot_next[G->tail] = id; + else G->pub.head = id; + G->tail = id; + return; + } slot_prev[id] = -1; slot_next[id] = bucket_head[b]; if (bucket_head[b] >= 0) slot_prev[bucket_head[b]] = id; bucket_head[b] = id; } +VkSlotGroup *vk_texture_group_locked(int g) { return &grp[g].pub; } +int vk_texture_group_next_locked(int id) { return slot_next[id]; } +void vk_texture_groups_unstamp_locked(void) { + for (int g = 0; g < ngrp; g++) grp[g].pub.stamp = 0; +} + uint64_t vk_texture_dirty_pos_locked(void) { return ndirty; } -int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg) { +// A bucket listed several times since *pos is walked once a round: the walk reads its slots' tokens after every change +// listed before the round began (the lock is held). Changes the walk itself makes (an entry a lookup created) are the +// next round's. A bucket's slot groups go to gfn when given, else their members to fn. +int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void (*gfn)(int g, void *arg), void *arg) { if (!slot_next || ndirty - *pos > DIRTY_N) { *pos = ndirty; return 0; } - for (uint64_t q = *pos; q < ndirty; q++) { - uint32_t x = dirty[q % DIRTY_N]; - if (x & DIRTY_BUCKET) - for (int32_t id = bucket_head[x & ~DIRTY_BUCKET]; id >= 0; id = slot_next[id]) fn(id, arg); - else fn((int)x, arg); + static uint32_t seen[AGEN_N], round; + for (uint64_t q = *pos; q < ndirty;) { + uint64_t end = ndirty; + if (!++round) memset(seen, 0, sizeof seen), round = 1; + for (; q < end; q++) { + uint32_t x = dirty[q % DIRTY_N]; + if (x & DIRTY_BUCKET) { + uint32_t b = x & ~DIRTY_BUCKET; + if (seen[b] == round) continue; + seen[b] = round; + for (int32_t id = bucket_head[b]; id >= 0; id = slot_next[id]) fn(id, arg); + for (int32_t g = bucket_ghead[b]; g >= 0; g = grp[g].gnext) { // the walk registers nothing + if (gfn) gfn(g, arg); + else for (int32_t id = grp[g].pub.head; id >= 0; id = slot_next[id]) fn(id, arg); + } + } else { + fn((int)x, arg); + } + } } *pos = ndirty; return 1; @@ -1121,10 +1226,189 @@ static void rel_unlink(Tex *e) { e->rel_prev = e->rel_next = NULL; } +// The entries the bindless texture pool's slots resolved to, for a submission's freshness check (make_fresh_snap once +// per entry), which looked at every slot instead: 33,000 slots, 1.5 ms of the replay thread a submission in Kokiri +// Forest. vk_exec_draw.c pool_fill notes a slot's entry after it resolved the slot; only entries whose content the CPU +// provides are listed (the check does nothing for the others; one the GPU took over, gpu_owned, never uploads again +// and leaves), and an entry stays listed until it is destroyed: the list holds every entry a slot resolves to, and +// some that no slot does any more. Each keeps its storage range and the page generation of its last check here, so +// that the check of an entry whose 2 MiB regions took no write since (vk_mem_unchanged_locked, one shared lock for the +// whole list) touches neither the entry nor the lock: 5,600 entries took 0.77 ms a submission one by one. Texture lock +// held. +typedef struct { Tex *e; uint64_t addr, size; uint32_t since; int ok; } PEnt; // ok: uploaded as of since +static PEnt *pent; +static int npent, cpent, *pq, cpq; +static uint64_t n_pool_checks, n_pool_skips; +// The region check's misses (vk_mem_unchanged_why_locked): not uploaded, a region without this epoch's snapshot yet, one +// untracked, one written since, nothing to check; and the snapshot queries its own pass made (NX2NAVI_POOL_PRESNAP). +static uint64_t n_pool_miss[5], n_presnap_q; +static void make_fresh_snap(Tex *e); + +static void pool_unlist(Tex *e) { + if (!e->pidx) return; + int i = e->pidx - 1; + pent[i] = pent[--npent]; + pent[i].e->pidx = i + 1; + e->pidx = 0; +} + +// The entries the pool's slots hold: for each, how many slots' last fill resolved to it (vk_exec_draw.c pool_hold; a fill +// that shared another slot's lookup counts for the entry that lookup found). The walk over every slot looked each slot +// up in each submission, which left its entry's use stamp (touch): the idle clock reads it, and start_at's alias rule +// reads the frame of an entry with GPU content. The check by list is no lookup, so it leaves the stamps from these +// counts (pool_touch): in each submission for the GPU-owned entries, in turn for the others, each once in +// POOL_TOUCH_DIV submissions (IDLE_ROUNDS is 240 of them). An entry keeps its place until it is destroyed and no slot +// holds it any more: a destroyed one is refilled out of its slots by the bump of its bucket. +#define POOL_TOUCH_DIV 64 +typedef struct { Tex *e; uint32_t refs; int next; uint8_t gown; } PRef; // e NULL: destroyed, or free (refs 0, next) +static PRef *pref; +static int npref, cpref, pref_free; // pref_free: 1 + the first free place +static void touch(Tex *e); + +static uint32_t pool_ref(Tex *e) { + if (e->ph) return (uint32_t)e->ph; + int i; + if (pref_free) { + i = pref_free - 1, pref_free = pref[i].next; + } else { + if (npref == cpref) { + int n = cpref ? cpref * 2 : 1024; + PRef *q = realloc(pref, sizeof *q * (size_t)n); + if (!q) rt_fatal("vk: out of memory for the texture pool's entry counts"); + pref = q, cpref = n; + } + i = npref++; + } + pref[i] = (PRef){e, 0, 0, e->pub.gpu_owned}; + return (uint32_t)(e->ph = i + 1); +} + +void vk_texture_pool_hold_locked(uint32_t old, uint32_t ent) { + if (ent) pref[ent - 1].refs++; + if (old && !--pref[old - 1].refs && !pref[old - 1].e) pref[old - 1].next = pref_free, pref_free = (int)old; +} + +static void pool_forget(Tex *e) { // destroy_tex + if (!e->ph) return; + PRef *q = &pref[e->ph - 1]; + q->e = NULL; + if (!q->refs) q->next = pref_free, pref_free = e->ph; + e->ph = 0; +} + +static void pool_touch(void) { + static int at; + for (int i = 0; i < npref; i++) + if (pref[i].gown && pref[i].refs && pref[i].e) touch(pref[i].e); + for (int k = (npref + POOL_TOUCH_DIV - 1) / POOL_TOUCH_DIV; k-- > 0;) { + if (at >= npref) at = 0; + const PRef *q = &pref[at++]; + if (!q->gown && q->refs && q->e) touch(q->e); + } +} + +// e becomes GPU-owned: the list's check does nothing for it any more, and its use stamp is left in each submission. +static void pool_gpu_owned(Tex *e) { + pool_unlist(e); + if (e->ph) pref[e->ph - 1].gown = 1; +} + +int vk_texture_pool_note_locked(int id, uint32_t *ent) { + *ent = 0; + if (id < 0 || id >= ntslots || !tslots[id].used) return 0; + // The caller's lookups left the slot resolved under its bucket's generation, unless it made none (a fill without a + // token or a kind): an entry cached under another generation may be destroyed, so the slot is looked up first. + if (tslots[id].ce && tslots[id].cgen != agen[sinfo[id] & 0xffff]) vk_texture_by_id_locked(id, NULL, NULL); + Tex *e = tslots[id].ce; + if (!e) return 0; + if (!e->pidx && e->pub.mirrored && !e->pub.gpu_owned) { + if (npent == cpent) { + int n = cpent ? cpent * 2 : 1024; + PEnt *p = realloc(pent, sizeof *p * (size_t)n); + int *q = realloc(pq, sizeof *q * (size_t)n); + if (!p || !q) rt_fatal("vk: out of memory for the texture pool's entry list"); + pent = p, pq = q, cpent = cpq = n; + } + pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0}; + e->pidx = npent; + } + *ent = pool_ref(e); + return 1; +} + +// Released entries are left out: a slot resolves to none of those whose content the CPU provides (slot_resolve's stale +// rule), and the game may be unmapping their storage. +// NX2NAVI_POOL_PRESNAP=1 (default), 0, ab: the check takes the write-watch snapshot of an entry's regions that have +// none in this submission yet before it looks at them (one exclusive hold of the generation table's lock for the list), +// so an unchanged entry in such a region passes there. Without it 97 % of the 5,600 entries missed the region check for +// want of a snapshot and went the long way, one entry and one lock at a time. With it 97 % pass, 3.5 snapshot queries +// a submission: the check 0.50 to 0.19 ms a submission, the replay thread -0.37 ms against the control. +static int pool_presnap(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("POOL_PRESNAP", "1"); + return RT_KNOB_ON(mode); +} + +// The submission being recorded checked the pool's entries by this list: for the idle clock it walked the pool +// (vk_texture_new_submit). +static int pool_by_list; + +int vk_texture_pool_fresh_locked(void) { + int nq = 0, ps = pool_presnap(); + pool_by_list = 1; + if (ps) vk_mem_gen_lock(); + else vk_mem_gen_lock_shared(); + for (int i = 0; i < npent; i++) { + const PEnt *p = &pent[i]; + if (p->ok) { + if (ps) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch); + int why = vk_mem_unchanged_why_locked(p->addr, p->size, p->since, submit_epoch); + if (!why) continue; + n_pool_miss[why]++; + } else { + n_pool_miss[0]++; + } + pq[nq++] = i; + } + if (ps) vk_mem_gen_unlock(); + else vk_mem_gen_unlock_shared(); + n_pool_skips += (uint64_t)(npent - nq); + for (int k = nq; k-- > 0;) { // from the end: an entry leaving swaps the last one into its place + int i = pq[k]; + PEnt *p = &pent[i]; + Tex *e = p->e; + if (e->pub.gpu_owned) { + pool_unlist(e); + continue; + } + if (e->released) continue; + if (!e->uploaded || e->checked != submit_epoch) { + make_fresh_snap(e); + e->checked = submit_epoch; + n_pool_checks++; + } + p->since = e->wgen, p->ok = e->uploaded; + } + pool_touch(); + return npent; +} + +// slot_resolve's last answer was none because of its stale rule (vk_texture_slot_retry_locked) +static int resolve_stale; + +int vk_texture_slot_retry_locked(int id) { + if (id < 0 || id >= ntslots || !tslots[id].used || !tslots[id].t.mem) return -1; + resolve_stale = 0; + if (vk_texture_by_id_locked(id, NULL, NULL)) return 1; + return resolve_stale ? -1 : 0; +} + static void retire_remove(Tex *e); static void destroy_tex(Tex *e) { if (e->retiring) retire_remove(e); + pool_unlist(e); + pool_forget(e); own_remove(e); gwin_remove(e); n_site[0]++; @@ -1186,6 +1470,10 @@ static int comes_back_intact(const Tex *e) { return e->pub.mirrored && !e->pub.g // texture object, nothing samples it. The clock is look_round, which only a submission whose slot checks reached half // of the registered pool slots moves on (vk_texture_new_submit): counted in submissions, a pause screen or a fade that // draws nothing through the pool made the whole world's textures idle within seconds. +// With NX2NAVI_POOL_FAST the refresh checks the entries by list instead (vk_texture_pool_fresh_locked), which is no +// lookup. Such a submission counts for the clock all the same, and the entries the pool's slots hold get their stamp +// from the count of those slots (pool_touch), each at least once in POOL_TOUCH_DIV submissions: an entry a slot names +// never comes near IDLE_ROUNDS. #define IDLE_ROUNDS 240 static uint32_t look_round = 1; // submissions whose lookups walked the texture pool static uint32_t round_checks; // slot checks of the submission being recorded (vk_texture_by_id_locked) @@ -1198,6 +1486,7 @@ static void touch(Tex *e) { } void vk_texture_pool_ctx(int on) { pool_ctx = on; } + static int idle(const Tex *e, uint64_t now); // NX2NAVI_TEX_RELEASED_MB (default a thirty-second of the installed memory, 512 MiB to 2 GiB; 0 = no limit) bounds the @@ -1552,6 +1841,7 @@ static uint64_t evict_disused(uint64_t want) { } retiring[nretiring++] = e; e->retiring = 1, e->retire_takes = takes, e->retire_due = 0; + pool_unlist(e); // no slot resolves to it in the pool refresh; taken back, the refill of its slots lists it again retiring_bytes += e->bytes; freed += e->bytes; tex_gen_bump(e->key.addr); @@ -2336,6 +2626,7 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen); e->uploaded = 1; + pool_gpu_owned(e); e->pub.gpu_owned = 1; } e->chash_ok = 0; @@ -2491,6 +2782,7 @@ void vk_texture_gpu_written(const tex_t *t) { e->scan_frame = now + 1; overwrite_others(e, now); } + pool_gpu_owned(e); e->pub.gpu_owned = 1; e->chash_ok = 0; // the image now holds the GPU's content: matching storage bytes no longer mean "unchanged" if (e->pub.skip) { // texture detail leaves out what the GPU can write (detail_skip); said if it ever does @@ -2578,10 +2870,11 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { if (!sinfo) { uint64_t *p = calloc(1 << 20, sizeof *p); slot_next = malloc(sizeof *slot_next << 20), slot_prev = malloc(sizeof *slot_prev << 20); + slot_grp = malloc(sizeof *slot_grp << 20); slot_bucket = calloc(1 << 20, sizeof *slot_bucket); - if (!p || !slot_next || !slot_prev || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens"); - for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1; - for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1; + if (!p || !slot_next || !slot_prev || !slot_grp || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens"); + for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1, slot_grp[i] = -1; + for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1, bucket_ghead[i] = -1; __atomic_store_n(&sinfo, p, __ATOMIC_RELEASE); } TexSlot *s = &tslots[id]; @@ -2607,7 +2900,6 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { slots_gen++; if (!++reg_serial) reg_serial = 1; __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); - slot_rebucket(id, agen_idx((uintptr_t)t->mem)); dirty_push((uint32_t)id); s->has_view = v != NULL; if (v) s->v = *v; @@ -2623,6 +2915,7 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { for (size_t i = 0; i < sizeof s->v; i++) h = (h ^ b[i]) * 0x100000001b3ull; s->khash = (h ^ s->has_view) | 1; } + slot_place(id, agen_idx((uintptr_t)t->mem), s->khash); unlock(); } @@ -2689,6 +2982,7 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { else if (!e && !cpu_slot) e = own && own->superseded ? NULL : own; // a superseded one: entry_of takes it back else if (!e && cpu_slot && s->finalized) stale = 1; } + resolve_stale = stale; if (!e && s->t.mem && !stale) { // evicted by disuse: the pool refresh keeps the stand-in, without spending the creation budget on it if (pool_ctx && holds.n && nvn_hm_get(&holds, &k)) resolve_held = 1; @@ -2757,7 +3051,27 @@ void vk_texture_bucket_stats(char *out, size_t n) { (unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone, (unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]], bump_count[top[0]], bump_count[top[1]], bump_count[top[2]], bump_count[top[3]]); - n_bumps = n_undone = 0; + size_t used = strlen(out); + if (used < n) + snprintf(out + used, n - used, "; pool freshness check: %llu entries checked, %llu passed by their regions (missed: " + "%llu not uploaded, %llu a region without a snapshot yet, %llu untracked, %llu written since, %llu " + "nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP)", + (unsigned long long)n_pool_checks, (unsigned long long)n_pool_skips, + (unsigned long long)n_pool_miss[0], (unsigned long long)n_pool_miss[1], + (unsigned long long)n_pool_miss[2], (unsigned long long)n_pool_miss[3], + (unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q); + used = strlen(out); + if (used < n) + snprintf(out + used, n - used, "; %d slot groups holding %llu slots", ngrp_live, (unsigned long long)n_grouped); + int held = 0, gown = 0; + for (int i = 0; i < npref; i++) + if (pref[i].refs && pref[i].e) held++, gown += pref[i].gown; + used = strlen(out); + if (used < n) + snprintf(out + used, n - used, "; pool slots hold %d entries, %d of them GPU-owned (stamped in each submission " + "checked by list)", held, gown); + n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = 0; + memset(n_pool_miss, 0, sizeof n_pool_miss); memset(n_site, 0, sizeof n_site); memset(bump_count, 0, sizeof bump_count); unlock(); @@ -2800,8 +3114,9 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { s->cheld = (uint8_t)(!s->ce && resolve_held); } Tex *e = s->ce; - // a draw binding: the texture is in use, whether or not the pool walk checked it first in this submission - if (e && !pool_ctx) e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + // a draw binding: the texture is in use, whether or not the pool's refresh checked it first in this submission + // (by list that check is no lookup, vk_texture_pool_fresh_locked, so the stamp of the use is left here too) + if (e && !pool_ctx) touch(e); // Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a // replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not // mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A @@ -2858,8 +3173,9 @@ uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC void vk_texture_new_submit(void) { lock(); - if (round_checks && (uint64_t)round_checks * 2 >= pool_slots) look_round++; // the last one walked the pool - round_checks = 0; + // the last one walked the pool, or checked its entries by list + if (pool_by_list || (round_checks && (uint64_t)round_checks * 2 >= pool_slots)) look_round++; + round_checks = 0, pool_by_list = 0; if (!++submit_epoch) submit_epoch = 1; rate_tick(); sweep_pressure();