diff --git a/runtime/cfg.c b/runtime/cfg.c index 115377e2..31cf1cca 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -54,6 +54,7 @@ static const char *const known[] = { "MV_SURVEY", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", + "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", diff --git a/runtime/host.h b/runtime/host.h index e77c8a6f..e8af6dde 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -161,6 +161,7 @@ uint32_t mem_map(uint64_t dst, uint64_t src, uint64_t n); uint32_t mem_unmap(uint64_t dst, uint64_t src, uint64_t n); // 0x05 uint32_t mem_query(uint64_t a, RtMemoryInfo *out); // 0x06 int mem_readable(uint64_t a, uint64_t n); // every page readable +uint32_t mem_map_gen(void); // changes with everything mem_readable answers from (mapping, heap, permissions) int mem_read_hold(uint64_t a, uint64_t n); // mem_readable that keeps the range mapped until mem_read_release void mem_read_release(void); uint32_t mem_map_physical(uint64_t a, uint64_t n); // 0x2C diff --git a/runtime/memory.c b/runtime/memory.c index 21983d6b..c8bff935 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -683,6 +683,11 @@ static uint64_t image_bytes(void) { static int overlaps(uint64_t a, uint64_t n, uint64_t b, uint64_t m) { return a < b + m && b < a + n; } static int inside(uint64_t a, uint64_t n, RtRange r) { return a >= r.base && n <= r.size && a - r.base <= r.size - n; } +// Bumped (lock held exclusively) by every change of what is mapped or readable: vk_exec_draw.c's texture pool looks the +// slots over unreadable storage up again only then (and now and then). +static uint32_t map_gen; +uint32_t mem_map_gen(void) { return __atomic_load_n(&map_gen, __ATOMIC_ACQUIRE); } + // ---- 0x01 SetHeapSize uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { plat_mutex_lock(&lock); @@ -693,6 +698,7 @@ uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) { if (size > heap_size) commit(L.heap.base + heap_size, size - heap_size, "heap"); if (size < heap_size) decommit(L.heap.base + size, heap_size - size, "heap"); heap_size = size; + __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); if (addr) *addr = L.heap.base; return R_OK; @@ -749,6 +755,7 @@ uint32_t mem_set_permission(uint64_t a, uint64_t n, uint32_t perm) { } memset(t, (int)perm, n / PAGE); protect(a, n, perm == PERM_RW ? PLAT_PROT_RW : perm == PERM_R ? PLAT_PROT_READ : PLAT_PROT_NONE, "module data"); + __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); rt_log("%s data permission +%#llx size %#llx perm %u", m->name, (unsigned long long)(a - m->base), (unsigned long long)n, perm); @@ -820,6 +827,7 @@ static uint32_t stack_move(int unmap, uint64_t dst, uint64_t src, uint64_t n) { stack_maps[nstack_maps++] = (StackMap){dst, src, n}; } out: + __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); return r; } @@ -872,6 +880,7 @@ static uint32_t alias_map(int unmap, uint64_t a, uint64_t n) { alias_bytes += fresh * PAGE; } out: + __atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE); plat_mutex_unlock(&lock); if (r) rt_log("%sMapPhysicalMemory %#llx+%#llx failed %#x (alias %#llx used, heap %#llx)", unmap ? "Un" : "", (unsigned long long)a, (unsigned long long)n, r, (unsigned long long)alias_bytes, diff --git a/runtime/vk.h b/runtime/vk.h index 55147b61..a70d5d21 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -565,8 +565,42 @@ uint32_t vk_texture_entry_gen(void); // vk_texture.c: bumped by // Slots whose token may have changed since *pos (0 at first; then the value these calls leave): fn is called for each // (some more than once) and *pos advanced. 0 when too many changes came since: the caller looks at every slot. *pos // from vk_texture_dirty_pos_locked after such a walk. Texture lock held. -int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg); +// gfn, when given, gets each slot group of a bucket in place of fn for its members. +int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void (*gfn)(int g, void *arg), void *arg); +// Slot groups (vk_texture.c): the GPU-written slots of one address bucket with one descriptor hash kh, which resolve +// alike. head the first member, vk_texture_group_next_locked the next (-1: the end), n how many; stamp and fmask the +// texture pool's (vk_exec_draw.c pool_dirty_group: 1 when every member holds the same views, of the kinds fmask; a slot +// that joins goes to the tail, refilled on its own), set 0 by vk_texture_groups_unstamp_locked. Texture lock held. +typedef struct { uint64_t kh; int32_t head, n; uint32_t stamp, fmask; } VkSlotGroup; +VkSlotGroup *vk_texture_group_locked(int g); +int vk_texture_group_next_locked(int id); +void vk_texture_groups_unstamp_locked(void); uint64_t vk_texture_dirty_pos_locked(void); +// The bindless pool's entries (vk_texture.c): after a pool fill of slot id, notes the entry it resolved to (1; 0: none) +// and gives its number for vk_texture_pool_hold_locked in *ent (0: none); a submission's freshness check of every +// noted entry (returns how many). Texture lock held. +int vk_texture_pool_note_locked(int id, uint32_t *ent); +int vk_texture_pool_fresh_locked(void); +// A pool slot that held the entry old (0: none) now holds ent: the entries slots hold get the use stamps a lookup +// leaves (vk_texture.c touch) from the check by list. Texture lock held. +void vk_texture_pool_hold_locked(uint32_t old, uint32_t ent); +// A slot whose last lookup found no entry, looked up again: 1 it resolves now; 0 still none, its storage may become +// readable; -1 none until its token changes (a CPU slot whose texture was released after its registration, no storage). +int vk_texture_slot_retry_locked(int id); +// vk_memory.c, for the pool's freshness check: the generation table's lock, shared; with it held, 1 when every 2 MiB +// region of [addr, addr+size) has this epoch's write-watch snapshot and no page stamped after since (nothing in it can +// have changed), 0 when a closer look is due (vk_mem_changed_snap). +void vk_mem_gen_lock_shared(void); +void vk_mem_gen_unlock_shared(void); +int vk_mem_unchanged_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch); +// The same check's reason when it fails: 1 a region without this epoch's snapshot yet, 2 one untracked, 3 one written +// since, 4 nothing to check (no size, beyond the regions); 0 unchanged. +int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch); +// The lock exclusively, for the snapshot of the regions of [addr, addr+size) that have none in epoch yet (the number of +// write-watch queries it took). +void vk_mem_gen_lock(void); +void vk_mem_gen_unlock(void); +uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch); void vk_sampler_dlss_bias(float bias); // texture LOD bias added for DLSS upscaling (material samplers) void vk_sampler_aniso_live(void); // each present: NX2NAVI_ANISO after a settings change (tiled samplers) uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 0c611fd9..36fd1c4b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1743,6 +1743,9 @@ static int tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int stor // absent from that submit's page tables, a GPU page fault on the 128x128 probe pass of Kokiri Forest's first frame // (RADV's radv_wait_for_vm_map_updates hid it, as did NX2NAVI_SPLIT_DRAWS=0). volatile uint32_t vk_pool_kinds_used; +// Per texture slot, what pool_fill keeps between refills: the descriptor hash of the registration it was read under, +// and the lookup memo stamp its entries were last written from (0: none) with the kinds that write covered. +typedef struct { uint64_t kh; uint32_t kreg, stamp, fmask; } PoolSlot; #define POOL_SETS VK_POOL_SETS #define POOL_LOG (1u << 20) // entries of the change log: binding * VK_POOL_TEX + id, one per write static struct { @@ -1758,7 +1761,24 @@ static struct { uint64_t logn; // writes so far uint64_t *last; // per entry: the log position of its newest write (catch-up writes it once, there) uint64_t switches, waits, catchup, written, rewrites, catch_ns, sync_ns, n_slots, n_refill, n_fresh; + uint64_t n_walk[2], walk_visit[2], walk_refill[2], walk_ns[2]; // dirty walks: at a submission's start, within one + uint64_t n_visit; // slots the dirty walks looked at + uint64_t n_gwalk, n_gskip, n_gskip_slots, n_grep; // POOL_GROUP: groups walked, left alone and their slots, first + // members looked up for the memo + uint64_t n_gchk_writes, n_gchk_groups; // POOL_GROUP_CHECK: writes the groups left alone got all the same, and in how many + uint64_t n_gchk_pending, n_gchk_bad; // of members registered since their last fill (not counted above); this group's + uint64_t n_gnouni, n_gchanged; // groups walked member by member: not known to hold one resolution, first member changed + int gmode; // pool_group() as of the last sync uint64_t n_by_reg, n_by_addr; // refills whose token changed by a registration, by an address generation only + uint64_t full_ns, n_full; // the walks over every slot (a submission's first sync) and their time + uint64_t ep_ns, n_ep, n_ep_ent; // POOL_FAST: a submission's freshness check of the noted entries instead + uint64_t unres_ns, n_unres_walks; // of it: the walks over the slots without an entry + uint32_t unres_mg; // mem_map_gen of the last such walk + int unres_more; // it left a lookup for the creation budget: the next submission walks again + PoolSlot *ps; // [id] + uint8_t *unres_on; // [id]: its last fill resolved to no entry (POOL_FAST: retried each submission) + int *unres, nunres, cunres; // those slots (some may have left: unres_on clear) + uint32_t *ent; // [id]: the entry its last fill resolved to, held there (pool_hold); 0: none uint64_t dirty_pos; // vk_texture_dirty_slots_locked position (0: the next sync walks every slot) int ready, uab, sealed; // sealed: a part of the current submission already reads sets[cur] uint32_t kinds, gen, epoch, sgen; @@ -1836,7 +1856,10 @@ static VkDescriptorSetLayout pool_layout_make(void) { PL.tok = calloc(VK_POOL_TEX, sizeof *PL.tok); PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view); PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp); - if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory"); + PL.ps = calloc(VK_POOL_TEX, sizeof *PL.ps); + PL.unres_on = calloc(VK_POOL_TEX, 1); + PL.ent = calloc(VK_POOL_TEX, sizeof *PL.ent); + if (!PL.tok || !PL.view || !PL.samp || !PL.ps || !PL.unres_on || !PL.ent) rt_fatal("vk: out of memory"); rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)%s", VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP, PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets, @@ -2071,6 +2094,7 @@ uint64_t vk_pool_takes(void) { return __atomic_load_n(&pool_takes, __ATOMIC_ACQU // The current set gets the entries written into other sets in [from, to) of the log, as PL.view / PL.samp hold them now. static uint64_t pool_memo_hits; // pool_fill: slots that shared another slot's lookup (defined with pool_memo below) +static uint64_t pool_memo_same; // of those, slots whose entries held the shared views already static void pool_catch_up(uint64_t from, uint64_t to) { VkDescriptorSet set = PL.set; @@ -2089,23 +2113,47 @@ static void pool_catch_up(uint64_t from, uint64_t to) { pool_flush(); PL.switches++; if (PL.switches % 3600 == 0) { // diagnosis: what invalidates the slots (vk_texture.c vk_texture_bucket_stats) - char bs[320]; + char bs[768]; vk_texture_bucket_stats(bs, sizeof bs); rt_log("vk: texture pool buckets over 3600 switches: %s", bs); } if (PL.switches % 3600 == 0) rt_log("vk: texture pool sets: %llu switches, %llu waited for the GPU, %.1f log entries and %.1f writes per " - "catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.0f slots, %.0f " - "refilled: %.0f registered, %.0f address changes, %.0f shared a lookup; %.0f checked); %llu identical registrations kept; " + "catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.3f ms of it in " + "%.2f walks over every slot, %.3f ms in %.2f checks of %.0f listed entries and %d slots without one " + "(%.3f ms in %.2f walks over those); " + "%.0f slots, %.0f refilled: %.0f registered, %.0f address changes, %.0f " + "shared a lookup, %.0f of them holding its views already; %.0f checked); %llu identical registrations kept; " "%llu refreshes created nothing for their frame's time (pool_create_ms), %llu lookups of slots left in the frame skipped", (unsigned long long)PL.switches, (unsigned long long)PL.waits, (double)PL.catchup / (double)PL.switches, (double)PL.written / (double)PL.switches, (unsigned long long)PL.rewrites, PL.catch_ns / 1e6 / (double)PL.switches, PL.sync_ns / 1e6 / (double)PL.switches, + PL.full_ns / 1e6 / (double)PL.switches, (double)PL.n_full / (double)PL.switches, + PL.ep_ns / 1e6 / (double)PL.switches, (double)PL.n_ep / (double)PL.switches, + PL.n_ep ? (double)PL.n_ep_ent / (double)PL.n_ep : 0.0, PL.nunres, + PL.unres_ns / 1e6 / (double)PL.switches, (double)PL.n_unres_walks / (double)PL.switches, (double)PL.n_slots / (double)PL.switches, (double)PL.n_refill / (double)PL.switches, (double)PL.n_by_reg / (double)PL.switches, (double)PL.n_by_addr / (double)PL.switches, - (double)pool_memo_hits / (double)PL.switches, + (double)pool_memo_hits / (double)PL.switches, (double)pool_memo_same / (double)PL.switches, (double)PL.n_fresh / (double)PL.switches, (unsigned long long)vk_texture_same_regs(), (unsigned long long)PL.n_frame_stops, (unsigned long long)PL.n_defer_skips); + if (PL.switches % 3600 == 0) + rt_log("vk: texture pool dirty walks per switch: %.2f at a submission's start (%.0f slots looked at, %.0f " + "refilled, %.3f ms), %.2f within one (%.0f looked at, %.0f refilled, %.3f ms)", + (double)PL.n_walk[0] / (double)PL.switches, (double)PL.walk_visit[0] / (double)PL.switches, + (double)PL.walk_refill[0] / (double)PL.switches, PL.walk_ns[0] / 1e6 / (double)PL.switches, + (double)PL.n_walk[1] / (double)PL.switches, (double)PL.walk_visit[1] / (double)PL.switches, + (double)PL.walk_refill[1] / (double)PL.switches, PL.walk_ns[1] / 1e6 / (double)PL.switches); + if (PL.switches % 3600 == 0) + rt_log("vk: texture pool slot groups per switch (NX2NAVI_POOL_GROUP): %.0f walked, %.0f left alone with %.0f " + "slots, %.0f first members refilled (%.0f changed), %.0f not known to hold one resolution; check " + "(NX2NAVI_POOL_GROUP_CHECK): %llu writes in %llu groups left alone so far, %llu more to members registered " + "since their last fill", + (double)PL.n_gwalk / (double)PL.switches, (double)PL.n_gskip / (double)PL.switches, + (double)PL.n_gskip_slots / (double)PL.switches, (double)PL.n_grep / (double)PL.switches, + (double)PL.n_gchanged / (double)PL.switches, (double)PL.n_gnouni / (double)PL.switches, + (unsigned long long)PL.n_gchk_writes, (unsigned long long)PL.n_gchk_groups, + (unsigned long long)PL.n_gchk_pending); } // Entry id of every kind in mask: the slot's view of that kind or the kind's stand-in. Texture lock held. 0: the @@ -2151,22 +2199,75 @@ static int pool_on(void) { // transient render target registered over its storage, and every finalize and initialize of that target changes their // tokens, so a refill looked up the same texture ~400 times, ~15,000 lookups a frame in Kokiri Forest. An entry holds the // views one slot got, for the slots with its hash, address generation and entry generation. +// Its stamp changes only when one of its views does (NX2NAVI_POOL_FAST, default on): a slot whose entries were written +// from the entry at the stamp it has now (PoolSlot) holds those views already, so a refill that finds the entry so +// compares and writes nothing. That is most refills: a transient render target released and taken back moves the +// address generation of its storage, ~1,300 slots in Kokiri Forest, and every slot resolves as before (18,500 of 18,800 +// refills a submission shared a lookup, each comparing up to 16 views a quarter megabyte apart: 1.9 ms). #define POOL_MEMO 4096 -static struct { uint64_t kh; uint32_t agen, egen, mask; VkImageView view[VK_POOL_KINDS]; } pool_memo[POOL_MEMO]; -static uint64_t pool_memo_hits; +static struct { uint64_t kh; uint32_t agen, egen, mask, stamp, got, ent; VkImageView view[VK_POOL_KINDS]; } pool_memo[POOL_MEMO]; +static uint32_t pool_stamp; // the last stamp given out +static uint64_t pool_memo_hits, pool_memo_same; + +static int pool_fast(void) { // ab: in the B blocks + static int on = -1; + if (on < 0) on = rt_knob("POOL_FAST", "1"); + return RT_KNOB_ON(on); +} + +// Slot id's last fill resolved to the entry ent (vk_texture_pool_note_locked; 0: none). vk_texture.c counts the slots +// that hold each entry: the entries held are the ones the walk over every slot reached with a lookup in each +// submission, and the check by list leaves their use stamps from that count (vk_texture_pool_hold_locked). +static void pool_hold(uint32_t id, uint32_t ent) { + if (PL.ent[id] == ent) return; + vk_texture_pool_hold_locked(PL.ent[id], ent); + PL.ent[id] = ent; +} + +// Slot id's last fill resolved to no entry (got 0) or to one: the slots without one are looked up again each +// submission (pool_sync_entries), as the walk over every slot did: an entry their storage could not have then may be +// made now, and its address generation then refills them. +static void pool_unres(uint32_t id, int got) { + if (got || PL.unres_on[id]) { + if (got) PL.unres_on[id] = 0; + return; + } + PL.unres_on[id] = 1; + if (PL.nunres == PL.cunres) { + int n = PL.cunres ? PL.cunres * 2 : 1024; + int *p = realloc(PL.unres, sizeof *p * (size_t)n); + if (!p) rt_fatal("vk: out of memory"); + PL.unres = p, PL.cunres = n; + } + PL.unres[PL.nunres++] = (int)id; +} static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) { vk_texture_create_deferred_locked(); - uint64_t kh = tok ? vk_texture_slot_khash_locked((int)id) : 0; + PoolSlot *ps = &PL.ps[id]; + uint64_t kh = 0; + if (tok) { // the descriptor hash changes only with a registration (the token's upper half) + uint32_t reg = (uint32_t)(tok >> 32); + if (ps->kreg != reg || !ps->kh) ps->kh = vk_texture_slot_khash_locked((int)id), ps->kreg = reg; + kh = ps->kh; + } uint32_t ag = (uint32_t)tok; typeof(pool_memo[0]) *m = kh ? &pool_memo[(kh >> 20) % POOL_MEMO] : NULL; if (m && m->kh == kh && m->agen == ag && m->egen == vk_texture_entry_gen() && (m->mask & mask) == mask) { + pool_memo_hits++; + pool_unres(id, (int)m->got); + pool_hold(id, m->ent); + if (ps->stamp && ps->stamp == m->stamp && (ps->fmask & mask) == mask && pool_fast()) { + pool_memo_same++; + return 1; + } for (int k = 0; k < VK_POOL_KINDS; k++) { if (!(mask >> k & 1)) continue; VkImageView *e = &PL.view[(size_t)k * VK_POOL_TEX + id]; if (*e != m->view[k]) *e = m->view[k], pool_write((uint32_t)k, id, m->view[k], VK_NULL_HANDLE); } - pool_memo_hits++; + ps->fmask = ps->stamp == m->stamp ? ps->fmask | mask : mask; + ps->stamp = m->stamp; return 1; } for (int k = 0; k < VK_POOL_KINDS; k++) { @@ -2180,12 +2281,29 @@ static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) { if (*e != ii.imageView) *e = ii.imageView, pool_write((uint32_t)k, id, ii.imageView, VK_NULL_HANDLE); } int ok = !vk_texture_create_deferred_locked(); + uint32_t ent = 0; + int got = vk_texture_pool_note_locked((int)id, &ent); + if (tok) pool_unres(id, got); + pool_hold(id, tok ? ent : 0); + ps->stamp = 0; // its entries match no memo entry, unless the one below if (m && ok) { // the generation after the lookups: an entry they created is part of what they resolved uint32_t eg = vk_texture_entry_gen(); - if (m->kh != kh || m->agen != ag || m->egen != eg) m->kh = kh, m->agen = ag, m->egen = eg, m->mask = 0; + // a view that changes gives the entry a new stamp; one kept outside the mask keeps its value, so a kind that + // comes back with another view is a change too + int changed = !m->stamp; for (int k = 0; k < VK_POOL_KINDS; k++) - if (mask >> k & 1) m->view[k] = PL.view[(size_t)k * VK_POOL_TEX + id]; + if (mask >> k & 1) { + VkImageView v = PL.view[(size_t)k * VK_POOL_TEX + id]; + if (m->view[k] != v) m->view[k] = v, changed = 1; + } + if (m->kh != kh || m->agen != ag || m->egen != eg) m->kh = kh, m->agen = ag, m->egen = eg, m->mask = 0; m->mask |= mask; + m->got = (uint32_t)got, m->ent = ent; + if (changed) { + if (!++pool_stamp) pool_stamp = 1; + m->stamp = pool_stamp; + } + ps->stamp = m->stamp, ps->fmask = mask; } return ok; } @@ -2237,6 +2355,7 @@ static int pool_refill(VkExec *ex, uint32_t id, uint32_t used, uint64_t t, int s static void pool_dirty_slot(int id, void *arg) { // vk_texture_dirty_slots_locked: a slot whose token may have changed PoolDirty *d = arg; + PL.n_visit++; if (id >= d->n) return; uint64_t t = vk_texture_slot_token(id); if (t == PL.tok[id]) return; @@ -2253,6 +2372,76 @@ static int pool_current(void) { vk_texture_slots_gen() == PL.gen && vk_texture_epoch() == PL.epoch && vk_sampler_gen() == PL.sgen; } +// NX2NAVI_POOL_GROUP=1 (default), 0, ab: a dirty bucket's slot groups (vk_texture.c) are walked whole. The members of a +// group resolve alike, so once a walk of the group brought every member to the current resolution (every fill done, an +// entry for each) they all hold the same views, and later the group is left alone when its first member still holds the +// current resolution: its token did not change (its views are what that token resolves to), or its refill wrote nothing. +// A slot that joined since is at the tail and refilled on its own (its registration's entry on the dirty ring). +// The members keep their older tokens: their views are what those tokens resolved to, and what the current ones do. +// (Comparing the memo's stamps instead left half the groups to their members: the memo's entries are shared between +// hashes, so the same views came back under a new stamp.) +static int pool_group(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("POOL_GROUP", "1"); + return RT_KNOB_ON(mode); +} + +// NX2NAVI_POOL_GROUP_CHECK=1: a group left alone is walked member by member all the same, and the writes that walk +// makes are counted (none when the rule holds; the pool log's slot groups line), but for the members registered since +// their last fill: their registration's entry on the dirty ring refills them in the same walk (counted apart). +static int pool_group_check(void) { + static int on = -1; + if (on < 0) on = rt_knob("POOL_GROUP_CHECK", "0") == 1; // a diagnosis: no settings row + return on; +} + +static void pool_dirty_group(int g, void *arg) { + PoolDirty *d = arg; + uint64_t chk = 0; + VkSlotGroup *G = vk_texture_group_locked(g); + int rep = G->head; + PL.n_gwalk++; + if (!G->stamp) PL.n_gnouni++; + if (G->stamp && (G->fmask & d->used) == d->used && rep >= 0 && rep < d->n && !PL.unres_on[rep]) { + uint64_t t = vk_texture_slot_token(rep); + int same = t == PL.tok[rep]; + if (!same) { // the first member's refill: nothing written, an entry found, its token taken + uint64_t l0 = PL.logn; + PL.n_grep++; + pool_dirty_slot(rep, arg); + same = PL.tok[rep] == t && PL.logn == l0 && !PL.unres_on[rep]; + if (!same) PL.n_gchanged++; + } + if (same) { + PL.n_gskip++, PL.n_gskip_slots += (uint64_t)G->n; + if (!pool_group_check()) return; + chk = PL.logn + 1; + } + } + int r0 = d->retry, uni = 1; // member by member: then all hold the current resolution, unless a fill was left + d->retry = 0; + for (int id = G->head; id >= 0; id = vk_texture_group_next_locked(id)) { + if (id >= d->n) continue; // outside the pool + if (chk) { // the check: a member registered since its last fill is refilled by its own entry anyway + uint64_t l0 = PL.logn; + int pending = vk_texture_slot_token(id) >> 32 != PL.tok[id] >> 32; + pool_dirty_slot(id, arg); + if (PL.logn != l0) { + if (pending) PL.n_gchk_pending += PL.logn - l0; + else PL.n_gchk_writes += PL.logn - l0, PL.n_gchk_bad++; + } + } else { + pool_dirty_slot(id, arg); + } + if (PL.unres_on[id]) uni = 0; + } + if (d->retry) uni = 0; + d->retry |= r0; + G = vk_texture_group_locked(g); + G->stamp = (uint32_t)uni, G->fmask = uni ? d->used : 0; + if (chk && PL.n_gchk_bad) PL.n_gchk_groups++, PL.n_gchk_bad = 0; +} + static void pool_sync_entries(VkExec *ex) { uint32_t used = __atomic_load_n(&vk_pool_kinds_used, __ATOMIC_RELAXED); uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); @@ -2277,12 +2466,51 @@ static void pool_sync_entries(VkExec *ex) { vk_texture_create_budget_locked(spent ? 0 : pool_create_budget()); PL.n_frame_stops += (uint64_t)spent; // Only a slot change since the last sync (no new kinds, no new submission, nothing left over): the slots that - // changed (vk_texture_dirty_slots_locked), not the whole table + // changed (vk_texture_dirty_slots_locked), not the whole table. With NX2NAVI_POOL_FAST (default on) a new + // submission too: its freshness check goes through the entries the slots resolved to and the slots that + // resolved to none (vk_texture_pool_fresh_locked, pool_unres), not every slot. PoolDirty pd = {ex, used, n, 0, spent}; - int walked = !fresh && ep == PL.epoch && !PL.retry && PL.dirty_pos && - vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, &pd); + int newep = ep != PL.epoch; + uint64_t v0 = PL.n_visit, f0 = PL.n_refill, w0 = kern_now_ns(); + int grp = pool_group(); + if (grp && !PL.gmode) vk_texture_groups_unstamp_locked(); // members were refilled one by one meanwhile + PL.gmode = grp; + int walked = !fresh && (!newep || pool_fast()) && !PL.retry && PL.dirty_pos && + vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, grp ? pool_dirty_group : NULL, &pd); + if (walked) { + int at = newep ? 0 : 1; + PL.n_walk[at]++, PL.walk_visit[at] += PL.n_visit - v0, PL.walk_refill[at] += PL.n_refill - f0; + PL.walk_ns[at] += kern_now_ns() - w0; + } retry = pd.retry; + if (walked && newep) { + uint64_t te = kern_now_ns(); + PL.n_ep_ent += (uint64_t)vk_texture_pool_fresh_locked(); + // The slots without an entry are over storage the guest has not mapped (stale descriptors, 1,160 in + // Kokiri Forest): looked up again when the mapping changed, while a lookup is left for the creation budget, + // and every 8th submission for the rest (an entry whose creation failed, an image at their address that + // took GPU content since: slot_resolve's alias rule) + uint32_t mg = mem_map_gen(); + uint64_t tu = kern_now_ns(); + int walk_unres = mg != PL.unres_mg || PL.unres_more || !(PL.n_ep & 7); + if (walk_unres) PL.unres_mg = mg, PL.n_unres_walks++, vk_texture_create_deferred_locked(); + for (int i = 0; walk_unres && i < PL.nunres;) { + int id = PL.unres[i]; + uint64_t t = PL.unres_on[id] ? vk_texture_slot_token(id) : 0; + int r = t && t == PL.tok[id] ? vk_texture_slot_retry_locked(id) : -1; + if (r) { // resolves now (an entry it made changes its token: refilled), was refilled since, or + PL.unres_on[id] = 0; // resolves to none until its token changes (the refill lists it again) + PL.unres[i] = PL.unres[--PL.nunres]; + continue; + } + i++; + } + if (walk_unres) PL.unres_more = vk_texture_create_deferred_locked(); + PL.unres_ns += kern_now_ns() - tu; + PL.n_ep++, PL.ep_ns += kern_now_ns() - te; + } if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked(); + uint64_t tw = walked ? 0 : kern_now_ns(); for (int id = 0; id < n && !walked; id++) { // slots past n were never registered: stand-ins since the first sync uint64_t t = vk_texture_slot_token(id); if (t != PL.tok[id]) { @@ -2298,7 +2526,10 @@ static void pool_sync_entries(VkExec *ex) { vk_texture_by_id_locked(id, NULL, NULL); // the submit's freshness check } } - if (!walked) PL.n_slots += (uint64_t)n; + if (!walked) { + PL.n_slots += (uint64_t)n, PL.n_full++, PL.full_ns += kern_now_ns() - tw; + vk_texture_groups_unstamp_locked(); // the walk over every slot refilled members one by one + } vk_texture_create_budget_locked(-1); } PL.retry = retry != 0; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index f2ee73a1..e730abea 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -914,6 +914,43 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e return untracked ? -1 : changed; } +void vk_mem_gen_lock_shared(void) { plat_mutex_lock_shared(&gen_lock); } +void vk_mem_gen_unlock_shared(void) { plat_mutex_unlock_shared(&gen_lock); } + +void vk_mem_gen_lock(void) { plat_mutex_lock(&gen_lock); } +void vk_mem_gen_unlock(void) { plat_mutex_unlock(&gen_lock); } + +int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch) { + if (!size || !epoch) return 4; + uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; + if (r1 >= SNAP_REGIONS) return 4; + for (uint64_t r = r0; r <= r1; r++) { // REG_SHIFT is SNAP_SHIFT: reg_max is the newest stamp in the region + if (snap_at[r] != epoch) return 1; + if (snap_untracked[r]) return 2; + if (reg_max[r] > since) return 3; + } + return 0; +} + +int vk_mem_unchanged_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch) { + return !vk_mem_unchanged_why_locked(addr, size, since, epoch); +} + +uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch) { + if (!size || !epoch) return 0; + uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; + uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT; + if (r1 >= SNAP_REGIONS) return 0; + for (uint64_t r = r0; r <= r1; r++) + if (snap_at[r] != epoch) { + uint64_t q = snap_regions(r0, r1, epoch); + n_snap_calls += q; + return q; + } + return 0; +} + void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) { *checks = __atomic_exchange_n(&n_snap_checks, 0, __ATOMIC_RELAXED); *calls = __atomic_exchange_n(&n_snap_calls, 0, __ATOMIC_RELAXED); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index e9fc6390..87a9cc4b 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -89,6 +89,8 @@ typedef struct Tex { uint64_t ow_frame; // gfx_counters.frames when overwritten was set uint8_t overwritten; // released, and a texture at another address was drawn over its storage (overwrite_others) uint8_t in_gwin; // in gwin[] + int pidx; // 1 + its index in pent[] (vk_texture_pool_note_locked); 0: not listed + int ph; // 1 + its index in pref[] (pool_ref); 0: none } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the @@ -296,32 +298,135 @@ static void tex_gen_flip(Tex *e, int plain) { dirty_push(DIRTY_BUCKET | b); } -// Moves slot id into bucket b's list (the lists exist once a slot was registered: vk_texture_register). -static void slot_rebucket(int id, uint32_t b) { +// Slot groups: the GPU-written slots of one address bucket that share a descriptor hash (TexSlot.khash) resolve alike +// under the same address and entry generations, as the texture pool's memo has it (vk_exec_draw.c pool_fill). Ocarina +// keeps up to ~400 descriptors of each transient render target registered over its storage, and the fullest bucket, +// 1,281 slots, moves 8 times a submission in Kokiri Forest: 12,800 of the 17,600 slots a submission's dirty walks +// refilled held their views already. The dirty walk can hand each group to the pool whole (vk_texture_dirty_slots_locked's +// gfn), which leaves its members alone when their resolution did not change (NX2NAVI_POOL_GROUP). A slot is in its +// bucket's own list (khash 0: a CPU-written texture) or in its group's member list, through the same links. A slot that +// joins a group goes to its tail and leaves the group's stamp as it was: its registration put it on the dirty ring, so +// the walk that follows refills it on its own, and the head, whose state the pool tests, is a member from before. (The +// game's 118 registrations a submission cleared the stamps of the groups they joined, and the walk at a submission's +// start went member by member as before: 11,300 of 11,550 slots.) +typedef struct { VkSlotGroup pub; int32_t gnext, gprev, tail; uint32_t bucket; } SlotGroup; +static SlotGroup *grp; +static int ngrp, cgrp, grp_free = -1, ngrp_live; +static uint64_t n_grouped; // slots in groups +static int32_t bucket_ghead[AGEN_N]; // per bucket: its first group (-1: none; set with the lists) +static int32_t *slot_grp; // per slot id: its group, -1 none +typedef struct { uint64_t kh, b; } GrpKey; +static nvn_hmap grp_by = {.klen = sizeof(GrpKey)}; // -> group + 1 + +static int grp_get(uint64_t kh, uint32_t b) { + GrpKey k = {kh, b}; + void *v = nvn_hm_get(&grp_by, &k); + if (v) return (int)((intptr_t)v - 1); + int g; + if (grp_free >= 0) { + g = grp_free, grp_free = grp[g].gnext; + } else { + if (ngrp == cgrp) { + int n = cgrp ? cgrp * 2 : 1024; + SlotGroup *p = realloc(grp, sizeof *p * (size_t)n); + if (!p) rt_fatal("vk: out of memory for texture slot groups"); + grp = p, cgrp = n; + } + g = ngrp++; + } + SlotGroup *G = &grp[g]; + memset(G, 0, sizeof *G); + G->pub.kh = kh, G->pub.head = G->tail = -1, G->bucket = b; + G->gprev = -1, G->gnext = bucket_ghead[b]; + if (G->gnext >= 0) grp[G->gnext].gprev = g; + bucket_ghead[b] = g; + nvn_hm_put(&grp_by, &k, (void *)(intptr_t)(g + 1)); + ngrp_live++; + return g; +} + +static void grp_drop(int g) { + SlotGroup *G = &grp[g]; + GrpKey k = {G->pub.kh, G->bucket}; + nvn_hm_del(&grp_by, &k); + if (G->gprev >= 0) grp[G->gprev].gnext = G->gnext; + else bucket_ghead[G->bucket] = G->gnext; + if (G->gnext >= 0) grp[G->gnext].gprev = G->gprev; + G->pub.kh = 0, G->pub.head = G->tail = -1, G->pub.n = 0, G->pub.stamp = 0; + G->gnext = grp_free, grp_free = g; + ngrp_live--; +} + +// Places slot id in bucket b: at the tail of its group when it has a descriptor hash, else at the head of the bucket's +// own list (the lists exist once a slot was registered: vk_texture_register). +static void slot_place(int id, uint32_t b, uint64_t kh) { if (slot_prev[id] != -2) { // out of its old list + int32_t g = slot_grp[id]; + int32_t *head = g >= 0 ? &grp[g].pub.head : &bucket_head[slot_bucket[id]]; if (slot_prev[id] >= 0) slot_next[slot_prev[id]] = slot_next[id]; - else bucket_head[slot_bucket[id]] = slot_next[id]; + else *head = slot_next[id]; if (slot_next[id] >= 0) slot_prev[slot_next[id]] = slot_prev[id]; + else if (g >= 0) grp[g].tail = slot_prev[id]; + slot_grp[id] = -1; + if (g >= 0) { + n_grouped--; + if (!--grp[g].pub.n) grp_drop(g); + } } slot_bucket[id] = (uint16_t)b; + if (kh) { + int g = grp_get(kh, b); // may move grp + SlotGroup *G = &grp[g]; + slot_grp[id] = g; + G->pub.n++, n_grouped++; + slot_next[id] = -1; + slot_prev[id] = G->tail; + if (G->tail >= 0) slot_next[G->tail] = id; + else G->pub.head = id; + G->tail = id; + return; + } slot_prev[id] = -1; slot_next[id] = bucket_head[b]; if (bucket_head[b] >= 0) slot_prev[bucket_head[b]] = id; bucket_head[b] = id; } +VkSlotGroup *vk_texture_group_locked(int g) { return &grp[g].pub; } +int vk_texture_group_next_locked(int id) { return slot_next[id]; } +void vk_texture_groups_unstamp_locked(void) { + for (int g = 0; g < ngrp; g++) grp[g].pub.stamp = 0; +} + uint64_t vk_texture_dirty_pos_locked(void) { return ndirty; } -int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg) { +// A bucket listed several times since *pos is walked once a round: the walk reads its slots' tokens after every change +// listed before the round began (the lock is held). Changes the walk itself makes (an entry a lookup created) are the +// next round's. A bucket's slot groups go to gfn when given, else their members to fn. +int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void (*gfn)(int g, void *arg), void *arg) { if (!slot_next || ndirty - *pos > DIRTY_N) { *pos = ndirty; return 0; } - for (uint64_t q = *pos; q < ndirty; q++) { - uint32_t x = dirty[q % DIRTY_N]; - if (x & DIRTY_BUCKET) - for (int32_t id = bucket_head[x & ~DIRTY_BUCKET]; id >= 0; id = slot_next[id]) fn(id, arg); - else fn((int)x, arg); + static uint32_t seen[AGEN_N], round; + for (uint64_t q = *pos; q < ndirty;) { + uint64_t end = ndirty; + if (!++round) memset(seen, 0, sizeof seen), round = 1; + for (; q < end; q++) { + uint32_t x = dirty[q % DIRTY_N]; + if (x & DIRTY_BUCKET) { + uint32_t b = x & ~DIRTY_BUCKET; + if (seen[b] == round) continue; + seen[b] = round; + for (int32_t id = bucket_head[b]; id >= 0; id = slot_next[id]) fn(id, arg); + for (int32_t g = bucket_ghead[b]; g >= 0; g = grp[g].gnext) { // the walk registers nothing + if (gfn) gfn(g, arg); + else for (int32_t id = grp[g].pub.head; id >= 0; id = slot_next[id]) fn(id, arg); + } + } else { + fn((int)x, arg); + } + } } *pos = ndirty; return 1; @@ -1121,10 +1226,189 @@ static void rel_unlink(Tex *e) { e->rel_prev = e->rel_next = NULL; } +// The entries the bindless texture pool's slots resolved to, for a submission's freshness check (make_fresh_snap once +// per entry), which looked at every slot instead: 33,000 slots, 1.5 ms of the replay thread a submission in Kokiri +// Forest. vk_exec_draw.c pool_fill notes a slot's entry after it resolved the slot; only entries whose content the CPU +// provides are listed (the check does nothing for the others; one the GPU took over, gpu_owned, never uploads again +// and leaves), and an entry stays listed until it is destroyed: the list holds every entry a slot resolves to, and +// some that no slot does any more. Each keeps its storage range and the page generation of its last check here, so +// that the check of an entry whose 2 MiB regions took no write since (vk_mem_unchanged_locked, one shared lock for the +// whole list) touches neither the entry nor the lock: 5,600 entries took 0.77 ms a submission one by one. Texture lock +// held. +typedef struct { Tex *e; uint64_t addr, size; uint32_t since; int ok; } PEnt; // ok: uploaded as of since +static PEnt *pent; +static int npent, cpent, *pq, cpq; +static uint64_t n_pool_checks, n_pool_skips; +// The region check's misses (vk_mem_unchanged_why_locked): not uploaded, a region without this epoch's snapshot yet, one +// untracked, one written since, nothing to check; and the snapshot queries its own pass made (NX2NAVI_POOL_PRESNAP). +static uint64_t n_pool_miss[5], n_presnap_q; +static void make_fresh_snap(Tex *e); + +static void pool_unlist(Tex *e) { + if (!e->pidx) return; + int i = e->pidx - 1; + pent[i] = pent[--npent]; + pent[i].e->pidx = i + 1; + e->pidx = 0; +} + +// The entries the pool's slots hold: for each, how many slots' last fill resolved to it (vk_exec_draw.c pool_hold; a fill +// that shared another slot's lookup counts for the entry that lookup found). The walk over every slot looked each slot +// up in each submission, which left its entry's use stamp (touch): the idle clock reads it, and start_at's alias rule +// reads the frame of an entry with GPU content. The check by list is no lookup, so it leaves the stamps from these +// counts (pool_touch): in each submission for the GPU-owned entries, in turn for the others, each once in +// POOL_TOUCH_DIV submissions (IDLE_ROUNDS is 240 of them). An entry keeps its place until it is destroyed and no slot +// holds it any more: a destroyed one is refilled out of its slots by the bump of its bucket. +#define POOL_TOUCH_DIV 64 +typedef struct { Tex *e; uint32_t refs; int next; uint8_t gown; } PRef; // e NULL: destroyed, or free (refs 0, next) +static PRef *pref; +static int npref, cpref, pref_free; // pref_free: 1 + the first free place +static void touch(Tex *e); + +static uint32_t pool_ref(Tex *e) { + if (e->ph) return (uint32_t)e->ph; + int i; + if (pref_free) { + i = pref_free - 1, pref_free = pref[i].next; + } else { + if (npref == cpref) { + int n = cpref ? cpref * 2 : 1024; + PRef *q = realloc(pref, sizeof *q * (size_t)n); + if (!q) rt_fatal("vk: out of memory for the texture pool's entry counts"); + pref = q, cpref = n; + } + i = npref++; + } + pref[i] = (PRef){e, 0, 0, e->pub.gpu_owned}; + return (uint32_t)(e->ph = i + 1); +} + +void vk_texture_pool_hold_locked(uint32_t old, uint32_t ent) { + if (ent) pref[ent - 1].refs++; + if (old && !--pref[old - 1].refs && !pref[old - 1].e) pref[old - 1].next = pref_free, pref_free = (int)old; +} + +static void pool_forget(Tex *e) { // destroy_tex + if (!e->ph) return; + PRef *q = &pref[e->ph - 1]; + q->e = NULL; + if (!q->refs) q->next = pref_free, pref_free = e->ph; + e->ph = 0; +} + +static void pool_touch(void) { + static int at; + for (int i = 0; i < npref; i++) + if (pref[i].gown && pref[i].refs && pref[i].e) touch(pref[i].e); + for (int k = (npref + POOL_TOUCH_DIV - 1) / POOL_TOUCH_DIV; k-- > 0;) { + if (at >= npref) at = 0; + const PRef *q = &pref[at++]; + if (!q->gown && q->refs && q->e) touch(q->e); + } +} + +// e becomes GPU-owned: the list's check does nothing for it any more, and its use stamp is left in each submission. +static void pool_gpu_owned(Tex *e) { + pool_unlist(e); + if (e->ph) pref[e->ph - 1].gown = 1; +} + +int vk_texture_pool_note_locked(int id, uint32_t *ent) { + *ent = 0; + if (id < 0 || id >= ntslots || !tslots[id].used) return 0; + // The caller's lookups left the slot resolved under its bucket's generation, unless it made none (a fill without a + // token or a kind): an entry cached under another generation may be destroyed, so the slot is looked up first. + if (tslots[id].ce && tslots[id].cgen != agen[sinfo[id] & 0xffff]) vk_texture_by_id_locked(id, NULL, NULL); + Tex *e = tslots[id].ce; + if (!e) return 0; + if (!e->pidx && e->pub.mirrored && !e->pub.gpu_owned) { + if (npent == cpent) { + int n = cpent ? cpent * 2 : 1024; + PEnt *p = realloc(pent, sizeof *p * (size_t)n); + int *q = realloc(pq, sizeof *q * (size_t)n); + if (!p || !q) rt_fatal("vk: out of memory for the texture pool's entry list"); + pent = p, pq = q, cpent = cpq = n; + } + pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0}; + e->pidx = npent; + } + *ent = pool_ref(e); + return 1; +} + +// Released entries are left out: a slot resolves to none of those whose content the CPU provides (slot_resolve's stale +// rule), and the game may be unmapping their storage. +// NX2NAVI_POOL_PRESNAP=1 (default), 0, ab: the check takes the write-watch snapshot of an entry's regions that have +// none in this submission yet before it looks at them (one exclusive hold of the generation table's lock for the list), +// so an unchanged entry in such a region passes there. Without it 97 % of the 5,600 entries missed the region check for +// want of a snapshot and went the long way, one entry and one lock at a time. With it 97 % pass, 3.5 snapshot queries +// a submission: the check 0.50 to 0.19 ms a submission, the replay thread -0.37 ms against the control. +static int pool_presnap(void) { + static int mode = -1; + if (mode < 0) mode = rt_knob("POOL_PRESNAP", "1"); + return RT_KNOB_ON(mode); +} + +// The submission being recorded checked the pool's entries by this list: for the idle clock it walked the pool +// (vk_texture_new_submit). +static int pool_by_list; + +int vk_texture_pool_fresh_locked(void) { + int nq = 0, ps = pool_presnap(); + pool_by_list = 1; + if (ps) vk_mem_gen_lock(); + else vk_mem_gen_lock_shared(); + for (int i = 0; i < npent; i++) { + const PEnt *p = &pent[i]; + if (p->ok) { + if (ps) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch); + int why = vk_mem_unchanged_why_locked(p->addr, p->size, p->since, submit_epoch); + if (!why) continue; + n_pool_miss[why]++; + } else { + n_pool_miss[0]++; + } + pq[nq++] = i; + } + if (ps) vk_mem_gen_unlock(); + else vk_mem_gen_unlock_shared(); + n_pool_skips += (uint64_t)(npent - nq); + for (int k = nq; k-- > 0;) { // from the end: an entry leaving swaps the last one into its place + int i = pq[k]; + PEnt *p = &pent[i]; + Tex *e = p->e; + if (e->pub.gpu_owned) { + pool_unlist(e); + continue; + } + if (e->released) continue; + if (!e->uploaded || e->checked != submit_epoch) { + make_fresh_snap(e); + e->checked = submit_epoch; + n_pool_checks++; + } + p->since = e->wgen, p->ok = e->uploaded; + } + pool_touch(); + return npent; +} + +// slot_resolve's last answer was none because of its stale rule (vk_texture_slot_retry_locked) +static int resolve_stale; + +int vk_texture_slot_retry_locked(int id) { + if (id < 0 || id >= ntslots || !tslots[id].used || !tslots[id].t.mem) return -1; + resolve_stale = 0; + if (vk_texture_by_id_locked(id, NULL, NULL)) return 1; + return resolve_stale ? -1 : 0; +} + static void retire_remove(Tex *e); static void destroy_tex(Tex *e) { if (e->retiring) retire_remove(e); + pool_unlist(e); + pool_forget(e); own_remove(e); gwin_remove(e); n_site[0]++; @@ -1186,6 +1470,10 @@ static int comes_back_intact(const Tex *e) { return e->pub.mirrored && !e->pub.g // texture object, nothing samples it. The clock is look_round, which only a submission whose slot checks reached half // of the registered pool slots moves on (vk_texture_new_submit): counted in submissions, a pause screen or a fade that // draws nothing through the pool made the whole world's textures idle within seconds. +// With NX2NAVI_POOL_FAST the refresh checks the entries by list instead (vk_texture_pool_fresh_locked), which is no +// lookup. Such a submission counts for the clock all the same, and the entries the pool's slots hold get their stamp +// from the count of those slots (pool_touch), each at least once in POOL_TOUCH_DIV submissions: an entry a slot names +// never comes near IDLE_ROUNDS. #define IDLE_ROUNDS 240 static uint32_t look_round = 1; // submissions whose lookups walked the texture pool static uint32_t round_checks; // slot checks of the submission being recorded (vk_texture_by_id_locked) @@ -1198,6 +1486,7 @@ static void touch(Tex *e) { } void vk_texture_pool_ctx(int on) { pool_ctx = on; } + static int idle(const Tex *e, uint64_t now); // NX2NAVI_TEX_RELEASED_MB (default a thirty-second of the installed memory, 512 MiB to 2 GiB; 0 = no limit) bounds the @@ -1552,6 +1841,7 @@ static uint64_t evict_disused(uint64_t want) { } retiring[nretiring++] = e; e->retiring = 1, e->retire_takes = takes, e->retire_due = 0; + pool_unlist(e); // no slot resolves to it in the pool refresh; taken back, the refill of its slots lists it again retiring_bytes += e->bytes; freed += e->bytes; tex_gen_bump(e->key.addr); @@ -2336,6 +2626,7 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen); e->uploaded = 1; + pool_gpu_owned(e); e->pub.gpu_owned = 1; } e->chash_ok = 0; @@ -2491,6 +2782,7 @@ void vk_texture_gpu_written(const tex_t *t) { e->scan_frame = now + 1; overwrite_others(e, now); } + pool_gpu_owned(e); e->pub.gpu_owned = 1; e->chash_ok = 0; // the image now holds the GPU's content: matching storage bytes no longer mean "unchanged" if (e->pub.skip) { // texture detail leaves out what the GPU can write (detail_skip); said if it ever does @@ -2578,10 +2870,11 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { if (!sinfo) { uint64_t *p = calloc(1 << 20, sizeof *p); slot_next = malloc(sizeof *slot_next << 20), slot_prev = malloc(sizeof *slot_prev << 20); + slot_grp = malloc(sizeof *slot_grp << 20); slot_bucket = calloc(1 << 20, sizeof *slot_bucket); - if (!p || !slot_next || !slot_prev || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens"); - for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1; - for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1; + if (!p || !slot_next || !slot_prev || !slot_grp || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens"); + for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1, slot_grp[i] = -1; + for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1, bucket_ghead[i] = -1; __atomic_store_n(&sinfo, p, __ATOMIC_RELEASE); } TexSlot *s = &tslots[id]; @@ -2607,7 +2900,6 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { slots_gen++; if (!++reg_serial) reg_serial = 1; __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); - slot_rebucket(id, agen_idx((uintptr_t)t->mem)); dirty_push((uint32_t)id); s->has_view = v != NULL; if (v) s->v = *v; @@ -2623,6 +2915,7 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { for (size_t i = 0; i < sizeof s->v; i++) h = (h ^ b[i]) * 0x100000001b3ull; s->khash = (h ^ s->has_view) | 1; } + slot_place(id, agen_idx((uintptr_t)t->mem), s->khash); unlock(); } @@ -2689,6 +2982,7 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { else if (!e && !cpu_slot) e = own && own->superseded ? NULL : own; // a superseded one: entry_of takes it back else if (!e && cpu_slot && s->finalized) stale = 1; } + resolve_stale = stale; if (!e && s->t.mem && !stale) { // evicted by disuse: the pool refresh keeps the stand-in, without spending the creation budget on it if (pool_ctx && holds.n && nvn_hm_get(&holds, &k)) resolve_held = 1; @@ -2757,7 +3051,27 @@ void vk_texture_bucket_stats(char *out, size_t n) { (unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone, (unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]], bump_count[top[0]], bump_count[top[1]], bump_count[top[2]], bump_count[top[3]]); - n_bumps = n_undone = 0; + size_t used = strlen(out); + if (used < n) + snprintf(out + used, n - used, "; pool freshness check: %llu entries checked, %llu passed by their regions (missed: " + "%llu not uploaded, %llu a region without a snapshot yet, %llu untracked, %llu written since, %llu " + "nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP)", + (unsigned long long)n_pool_checks, (unsigned long long)n_pool_skips, + (unsigned long long)n_pool_miss[0], (unsigned long long)n_pool_miss[1], + (unsigned long long)n_pool_miss[2], (unsigned long long)n_pool_miss[3], + (unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q); + used = strlen(out); + if (used < n) + snprintf(out + used, n - used, "; %d slot groups holding %llu slots", ngrp_live, (unsigned long long)n_grouped); + int held = 0, gown = 0; + for (int i = 0; i < npref; i++) + if (pref[i].refs && pref[i].e) held++, gown += pref[i].gown; + used = strlen(out); + if (used < n) + snprintf(out + used, n - used, "; pool slots hold %d entries, %d of them GPU-owned (stamped in each submission " + "checked by list)", held, gown); + n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = 0; + memset(n_pool_miss, 0, sizeof n_pool_miss); memset(n_site, 0, sizeof n_site); memset(bump_count, 0, sizeof bump_count); unlock(); @@ -2800,8 +3114,9 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { s->cheld = (uint8_t)(!s->ce && resolve_held); } Tex *e = s->ce; - // a draw binding: the texture is in use, whether or not the pool walk checked it first in this submission - if (e && !pool_ctx) e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + // a draw binding: the texture is in use, whether or not the pool's refresh checked it first in this submission + // (by list that check is no lookup, vk_texture_pool_fresh_locked, so the stamp of the use is left here too) + if (e && !pool_ctx) touch(e); // Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a // replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not // mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A @@ -2858,8 +3173,9 @@ uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC void vk_texture_new_submit(void) { lock(); - if (round_checks && (uint64_t)round_checks * 2 >= pool_slots) look_round++; // the last one walked the pool - round_checks = 0; + // the last one walked the pool, or checked its entries by list + if (pool_by_list || (round_checks && (uint64_t)round_checks * 2 >= pool_slots)) look_round++; + round_checks = 0, pool_by_list = 0; if (!++submit_epoch) submit_epoch = 1; rate_tick(); sweep_pressure();