// Vulkan textures of the graphics layer: a VkImage cache keyed by storage // GPU address plus descriptor, uploads through staging buffers with block-linear deswizzle and ASTC decode (vk_astc.c), // re-uploads when page write generations (vk_memory.c) show the storage written, ReadTexels / WriteTexels, the texture // and sampler descriptor slots, and VkSamplers. // // A texture that mirrors its pool memory (tex_mirrored) stops re-uploading once the GPU rendered into it (the // "GPU-owned" rule); a slot of a finalized texture follows the live texture at its storage address. // Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md). // // Every image lives in VK_IMAGE_LAYOUT_GENERAL. Uploads are recorded into the upload batch of vk_memory.c; the replay // submits it (vk_upload_flush) before a submission that samples the textures. #include #include #include #include #include #include "cfg.h" #include "gfx.h" #include "host.h" #include "nvn_api.h" #include "nvn_hmap.h" #include "nvn_tex.h" #include "tex.h" #include "vk.h" #include "vk_dlaa.h" #include "vk_exec.h" #include "prof.h" #include "platform/platform.h" typedef struct { uint64_t addr; nvn_texdesc d; uint8_t has_layout, layout[8], pad[7]; } TexKey; // What makes two images interchangeable: the parameters create_tex makes them with (img_key_of). Image reuse and the // memory requirement cache match on it. typedef struct { uint32_t flags, type, format, w, h, d, levels, layers, samples, usage; } ImgKey; typedef struct Tex { VkTexImage pub; // first: vk_texture_get hands out &pub TexKey key; tex_t t; // the newest texture object seen with this key TexVkFormat f; VmaAllocation alloc; // own memory; NULL when bound into pool memory uint64_t bytes; // the size of alloc (0 without own memory), for the released entries' total VmaAllocation bound_to; // the pool memory it is bound into uint32_t wgen; // page generation of the last upload check uint32_t checked; // submit_epoch of the last check from a draw binding uint32_t use_round; // look_round when a lookup last reached it (touch) uint64_t use_frame; // gfx_counters.frames then uint64_t samp_frame; // gfx_counters.frames when it was made, a marked shader last sampled it, or a lookup // other than the texture pool refresh reached it ("disuse") uint8_t retiring; // evicted by disuse, in retiring[], not destroyed yet: the pool refresh leaves it out uint64_t retire_takes; // vk_pool_takes when it was retired uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it int uploaded, released; uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve) uint64_t uid; // names the entry in its bucket's released set (tex_gen_flip); 0 until its first release uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale) uint32_t back_gap; // most presents between a release and its taking back so far (stale_after) uint32_t n_up; // uploads of this entry (vk_texture_debug) int last_ch; // last vk_mem_changed result of make_fresh: 1 written, 0 clean, -1 not tracked struct Tex *next_at; // other entries at the same storage address uint8_t gpu_dirty; // mirrored, and the GPU wrote the image after the last write-back to the storage uint32_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes) uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one) uint64_t run_frame; // mirrored: the frame of the newest GPU write uint8_t run; // frames in a row up to run_frame with a GPU write, counted up to WB_RUN uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear, // storage binding); ReadTexels waits for it instead of draining the replay queue uint8_t listed; // in owned[] uint8_t *snap; // a texture that fills no page: its storage bytes when the GPU write was recorded uint32_t wt_n; // WriteTexels into the storage since the last upload uint64_t chash; // content_hash of the storage at its last upload (small textures, CHASH_MAX) uint8_t chash_ok; // chash holds it struct Tex *rel_prev, *rel_next; // while released: the list of released entries, oldest first ImgKey ik; // the image's creation parameters uint8_t reusable; // its image may go to a new texture once it is released (own memory, GPU-only content) uint8_t reused; // its image may still be in use from before (a released entry's, or its own restarted by // start_at): the first barrier waits for all earlier work uint8_t ru_on; // in the reuse list of ik uint8_t superseded; // released, and another descriptor's entry was made at its address since (start_at) uint32_t rel_epoch; // submit_epoch when it was released struct Tex *ru_prev, *ru_next; // the reuse list of ik, newest release first uint64_t gw_frame; // gfx_counters.frames at its creation or newest GPU write (overwrite_others) uint64_t scan_frame; // one more than the frame vk_texture_gpu_written last looked for overwritten entries uint64_t ow_frame; // gfx_counters.frames when overwritten was set uint8_t overwritten; // released, and a texture at another address was drawn over its storage (overwrite_others) uint8_t in_gwin; // in gwin[] } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the // entry (ce), valid while the generation of its address (agen) is still cgen, the view and has_view handed out with it, // and the submit_epoch of the last freshness check through the slot (cepoch). Hot fields first: a lookup that hits // touches one or two cache lines. typedef struct { struct Tex *ce; // NULL: nothing cached (registered since, or never resolved) uint32_t cgen; // agen of the slot's address when ce was resolved uint32_t cepoch; view_t cview; uint8_t chas_view, used, has_view; uint8_t cheld; // ce is NULL as the texture was evicted by disuse, valid while cgen is (held_out) uint8_t cwant; // a shader sampled its stand-in (vk_pool_marks_read): slot_resolve makes it at once uint64_t khash; // hash of the descriptor key and view, 0 for CPU-written textures (vk_texture_slot_khash_locked) view_t v; tex_t t; } TexSlot; typedef struct { samp_par p; VkSampler s, sp, sc; uint8_t used; } SampSlot; // s: vk_sampler_for(&p) once looked up; sp: // its point-sampled form (vk_sampler_point_by_id_locked), // sc its comparing form (vk_sampler_compare_by_id_locked) static PlatRMutex tex_lock; static PlatOnce tex_once = PLAT_ONCE_INIT; static nvn_hmap by_key = {.klen = sizeof(TexKey)}; static nvn_hmap by_addr = {.klen = 8}; static TexSlot *tslots; static int ntslots; static SampSlot *sslots; static int nsslots; static nvn_hmap samplers = {.klen = sizeof(samp_par)}; static int nreleased; static Tex *rel_head, *rel_tail; // released entries in release order (sweep_released drops from the head) static uint64_t n_uploads, n_reuploads; // Re-uploads since the last perf line (tex_lock): all, and those of textures a WriteTexels wrote since their last upload. static uint64_t ru_n, ru_bytes, ru_ns, ru_wt_n, ru_wt_bytes, ru_wt_ns; static uint64_t ru_same_n, ru_same_bytes; // re-uploads skipped: the pages were written, the texture's bytes were not // NX2NAVI_TEX_ALIAS (default on; =0: a new image over live storage starts empty): a new descriptor over the storage of a // live image that the GPU wrote gets that image's content (alias_copy). Counts since the last log line. static int tex_alias = -1; static uint64_t n_alias_copied, n_alias_upload, n_alias_uninit, n_alias_kept, n_alias_released; static uint32_t submit_epoch = 1; // bumped per queue submit: draw bindings check a texture's storage once per submit // The thresholds further down that count presents (STALE_FRAMES, GPU_STALE_FRAMES, OVERWRITTEN_KEEP, DISUSE_FRAMES) // were set at 60 presents a second. Without a limit the game presents 85 to 240 a second (a menu), and "120 presents" // was then half a second and "30 s without a sample" 7 to 20 s: under memory pressure textures went three times as // readily as meant, and the ones still on screen showed their dark stand-ins until they were made again. // tex_rate is the presents per second as a multiple of 60 in 1/256 (256 at 60 or fewer), the highest of the last RATE_N // seconds so that one slow second does not shorten the thresholds; PRESENTS(n) is n presents at 60 in today's presents. #define RATE_N 8 static uint32_t tex_rate = 256; #define PRESENTS(n) ((uint64_t)(n) * tex_rate >> 8) static void rate_tick(void) { // each submission, tex_lock held static uint64_t t0, f0; static uint32_t ring[RATE_N], at; uint64_t t = kern_now_ns(), f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (!t0) t0 = t, f0 = f; if (t - t0 < 1000000000ull) return; uint64_t r = (f - f0) * 256ull * 1000000000ull / ((t - t0) * 60); ring[at++ % RATE_N] = r < 256 ? 256 : r > 2048 ? 2048 : (uint32_t)r; t0 = t, f0 = f; uint32_t m = 256; for (int i = 0; i < RATE_N; i++) if (ring[i] > m) m = ring[i]; tex_rate = m; } // Mirrored textures the GPU wrote into, sorted by address (vk_texture_cpu_stale, vk_texture_write_back). Changed under // tex_lock and owned_lock; read under either. static PlatMutex owned_lock = PLAT_MUTEX_INIT; static Tex **owned; static int nowned, cowned; static uint64_t max_owned; // largest storage size in owned[] static uint64_t gpu_writes; // GPU writes into mirrored textures so far: Tex.gpu_seq // Bumped whenever the entry a slot resolves to can change: an entry created or destroyed, released or taken back into // use (the slot lookup and the follow of a finalized slot read only by_key, by_addr and the released flags). static uint32_t tex_gen = 1; // Bumped with tex_gen and whenever a slot is registered: what vk_texture_by_id returns for any slot may have changed. static uint32_t slots_gen = 1; static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line // tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so // a slot's resolution stays valid while its address's generation does. NX2navi finalizes and initializes transient // textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws, // which emptied every cache keyed on the global generations. 65536 buckets, the most the 16-bit index of sinfo, // slot_bucket and the dirty ring's bucket entries holds: with 4096, about 8 slots shared each bucket, and every // transient texture made or released at one address made the texture pool refresh (vk_exec_draw.c pool_sync_entries) // refill its neighbours too, most of its 3-3.7 ms per frame in Kokiri Forest. #define AGEN_N 65536 static uint32_t agen[AGEN_N]; // Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered) // and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are // 20 bits), unlike tslots, which moves when it grows. static uint64_t *sinfo; static uint32_t reg_serial; static uint64_t n_same_regs; // vk_texture_register calls that changed nothing (vk_texture_same_regs) static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 48); } // Slots that may resolve differently since a position, for the texture pool descriptors (vk_exec_draw.c // pool_sync_entries), which otherwise looked at every registered slot after each change: Ocarina registers slots all // through a submission, and the refresh walked the whole table about twenty times per submission (3.5 ms). A ring of // changes, each a slot id (registered) or an address bucket (an entry at that agen index changed: every slot registered // over it, kept in per-bucket lists). Lock held for all of it. #define DIRTY_N (1u << 16) #define DIRTY_BUCKET 0x80000000u static uint32_t dirty[DIRTY_N]; static uint64_t ndirty; static int32_t bucket_head[AGEN_N]; static int32_t *slot_next, *slot_prev; // per slot id: its bucket list links (slot_prev -2: in no list) static uint16_t *slot_bucket; static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; } // tex_gen_bump calls, by site: destroy, create, take-back, release; and the releases and take-backs that went back to a // generation their bucket's state had before (tex_gen_flip), for vk_texture_bucket_stats static uint64_t n_bumps, n_site[4], n_undone; static uint32_t bump_count[AGEN_N]; // The highest generation each bucket ever had. A new state takes the next one after it, not the one after the current // value, so a value a bucket went back from (tex_gen_flip) is never given to another state. static uint32_t agen_top[AGEN_N]; // What a slot at an address resolves to depends on the entries there and on which of them are released // (slot_resolve). Per bucket: epoch counts every change other than a release or a take-back (an entry made, destroyed, // retired, held or taken back with more than its flag changing), sig is the released entries' uids XORed, which a // release and a take-back each flip. (bucket, epoch, sig) names the state; gs remembers the generation each state // got, so a bucket that comes back to a state gets that state's generation back. static uint32_t bepoch[AGEN_N]; static uint64_t bsig[AGEN_N]; #define GS_N 8192 static struct { uint64_t sig; uint32_t b, epoch, gen, valid; } gs[GS_N]; // A slot lookup is resolving (vk_texture_by_id_locked around slot_resolve). The lookup's callers stamp what they keep // with the generation they read before it: the slot's own cache, the pool's tokens, the descriptor, type and feedback // memos (vk_exec_draw.c). A lookup that itself changes the state (it takes a released entry back for a slot registered // since the release) leaves such a stamp naming the state it left, with a result that state would not give: the next // release of that entry is later than the slot's registration, the slot is stale then (slot_resolve), and the kept // result would hand out the released entry, whose storage the game may be unmapping. So the state such a lookup // leaves loses its remembered generation (tex_gen_flip): it gets a new one when the bucket is next in it, and the // stamp never matches again. static int in_lookup; static uint32_t gs_at(uint32_t b) { return (uint32_t)(((bsig[b] ^ ((uint64_t)b << 32 | bepoch[b])) * 0x9E3779B97F4A7C15ull) >> 51); } static void tex_gen_bump(uint64_t addr) { n_bumps++; bump_count[agen_idx(addr)]++; if (!++tex_gen) tex_gen = 1; slots_gen++; uint32_t b = agen_idx(addr); bepoch[b]++; __atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE); gs[gs_at(b)] = (typeof(gs[0])){bsig[b], b, bepoch[b], agen[b], 1}; dirty_push(DIRTY_BUCKET | b); } // NX2NAVI_TEX_GEN_UNDO (default 1; 0: every release and take-back is a new generation). Ocarina finalizes and // initializes transient textures all through a frame: in a large town about 40,000 releases and as many take-backs // per 3,600 submissions, each a bump of its address bucket, and the buckets with the most slots (1,300 registered over // one address) were the ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it. // A release or a take-back changes one entry's released flag and nothing else, so it moves the bucket between states // it has usually been in before: the textures the game keeps released and takes back for a moment, and those it // releases and takes straight back. The bucket then gets the generation that state had, and whatever was resolved in // that state counts as current again: the pool's tokens, the descriptor memo and the slots' own cached entries. A // state seen for the first time gets a new generation (agen_top), and any other change starts a new epoch, so a // generation never stands for two states. The global generations still move, so their readers look, and find the // tokens equal. plain: the take-back of an entry that was superseded or overwritten while released changes more than // the flag, and is a change like any other. // One input of slot_resolve is not in the state: the stale rule compares a slot's registration with the entry's // release (rel_serial), which is later at every release. A slot is stale at every later release once it is at one, so // what it resolves to in a released state stays right; the one that is not stale takes the entry back, which is the // lookup-made change in_lookup deals with. static void tex_gen_flip(Tex *e, int plain) { static int on = -1; static uint64_t uids; if (on < 0) on = rt_knob("TEX_GEN_UNDO", "1") == 1; if (!e->uid) { uint64_t z = ++uids * 0x9E3779B97F4A7C15ull; z = (z ^ z >> 30) * 0xBF58476D1CE4E5B9ull, z = (z ^ z >> 27) * 0x94D049BB133111EBull; e->uid = (z ^ z >> 31) | 1; } uint32_t b = agen_idx(e->key.addr); if (in_lookup) { // see in_lookup: the state this lookup leaves is not to get its generation again typeof(gs[0]) *m0 = &gs[gs_at(b)]; if (m0->valid && m0->b == b && m0->epoch == bepoch[b] && m0->sig == bsig[b]) m0->valid = 0; } bsig[b] ^= e->uid; if (!on || plain) { tex_gen_bump(e->key.addr); return; } if (!++tex_gen) tex_gen = 1; slots_gen++; typeof(gs[0]) *m = &gs[gs_at(b)]; if (m->valid && m->b == b && m->epoch == bepoch[b] && m->sig == bsig[b]) { n_undone++; __atomic_store_n(&agen[b], m->gen, __ATOMIC_RELEASE); } else { n_bumps++; bump_count[b]++; __atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE); *m = (typeof(*m)){bsig[b], b, bepoch[b], agen[b], 1}; } dirty_push(DIRTY_BUCKET | b); } // Moves slot id into bucket b's list (the lists exist once a slot was registered: vk_texture_register). static void slot_rebucket(int id, uint32_t b) { if (slot_prev[id] != -2) { // out of its old list if (slot_prev[id] >= 0) slot_next[slot_prev[id]] = slot_next[id]; else bucket_head[slot_bucket[id]] = slot_next[id]; if (slot_next[id] >= 0) slot_prev[slot_next[id]] = slot_prev[id]; } slot_bucket[id] = (uint16_t)b; slot_prev[id] = -1; slot_next[id] = bucket_head[b]; if (bucket_head[b] >= 0) slot_prev[bucket_head[b]] = id; bucket_head[b] = id; } uint64_t vk_texture_dirty_pos_locked(void) { return ndirty; } int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg) { if (!slot_next || ndirty - *pos > DIRTY_N) { *pos = ndirty; return 0; } for (uint64_t q = *pos; q < ndirty; q++) { uint32_t x = dirty[q % DIRTY_N]; if (x & DIRTY_BUCKET) for (int32_t id = bucket_head[x & ~DIRTY_BUCKET]; id >= 0; id = slot_next[id]) fn(id, arg); else fn((int)x, arg); } *pos = ndirty; return 1; } static volatile int32_t tex_ready; static void tex_init(void *p) { (void)p; __atomic_store_n(&tex_ready, 1, __ATOMIC_RELEASE); return; } // NX2NAVI_LOCK_TIME: the outermost hold of the lock is timed from here, and one over 1 ms is reported with the code // that took it (perf_lock_hold); lock is not inlined so that its return address names that code. static uint64_t hold_t0; static void *hold_site; __attribute__((noinline)) static void lock(void) { if (!__atomic_load_n(&tex_ready, __ATOMIC_ACQUIRE)) plat_once(&tex_once, tex_init, NULL); plat_rmutex_lock_at(&tex_lock, __builtin_return_address(0)); if (tex_lock.depth == 1 && perf_lock_holds_on()) hold_t0 = plat_now_ns(), hold_site = __builtin_return_address(0); } static void unlock(void) { if (tex_lock.depth == 1 && hold_t0) { uint64_t d = plat_now_ns() - hold_t0; hold_t0 = 0; if (d > 1000000) perf_lock_hold(hold_site, d); } plat_rmutex_unlock(&tex_lock); } static uint32_t umax(uint32_t a, uint32_t b) { return a > b ? a : b; } static uint32_t umin(uint32_t a, uint32_t b) { return a < b ? a : b; } static uint32_t mip(uint32_t v, unsigned lv) { return umax(v >> lv, 1); } static void make_key(const tex_t *t, TexKey *k) { memset(k, 0, sizeof *k); k->addr = (uintptr_t)t->mem; k->d = t->d; k->has_layout = t->has_layout; memcpy(k->layout, t->layout, 8); } // Keys equal apart from the descriptor's swizzle selectors: the same storage, format and geometry. static int same_but_swizzle(const TexKey *a, const TexKey *b) { TexKey x = *a, y = *b; x.d.swizzle_r = x.d.swizzle_g = x.d.swizzle_b = x.d.swizzle_a = 0; y.d.swizzle_r = y.d.swizzle_g = y.d.swizzle_b = y.d.swizzle_a = 0; return !memcmp(&x, &y, sizeof x); } // Textures whose content the CPU provides in pool memory. Display, compressible, zcull, // depth, multisample and GPU-only storage stays GPU-owned. static int tex_mirrored(const tex_t *t) { if (!t->mem || nvn_gpu_only((uintptr_t)t->mem)) return 0; if (t->has_layout) return 1; if (!(t->pool_flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) return 0; if (t->d.flags & (NVN_TEXTURE_FLAGS_DISPLAY | NVN_TEXTURE_FLAGS_COMPRESSIBLE | NVN_TEXTURE_FLAGS_ADAPTIVE_ZCULL | NVN_TEXTURE_FLAGS_DEPTH16_PREFER_FAST_CLEAR | NVN_TEXTURE_FLAGS_ZCULL_SUPPORT_STENCIL)) return 0; if (t->d.samples > 1 || t->d.target == NVN_TEXTURE_TARGET_BUFFER) return 0; if (t->d.format >= 0x31 && t->d.format <= 0x36) return 0; // stencil / depth formats return 1; } static void log_limited(volatile int32_t *n, const char *fmt, ...) __attribute__((format(printf, 2, 3))); static void log_limited(volatile int32_t *n, const char *fmt, ...) { if (plat_inc32(n) > 16) return; char buf[512]; va_list ap; va_start(ap, fmt); vsnprintf(buf, sizeof buf, fmt, ap); va_end(ap); rt_log("vk: %s", buf); } static void barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst, VkAccessFlags2 da) { VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa, .dstStageMask = dst, .dstAccessMask = da}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); } #define ALL_RW VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT // ---------------------------------------------------------------- images static VkFormatFeatureFlags features_of(VkFormat f) { VkFormatProperties p; vkGetPhysicalDeviceFormatProperties(vk_ctx()->phys, f, &p); return p.optimalTilingFeatures; } static VkImageAspectFlags aspect_of(const TexVkFormat *f) { VkImageAspectFlags a = 0; if (f->depth) a |= VK_IMAGE_ASPECT_DEPTH_BIT; if (f->stencil) a |= VK_IMAGE_ASPECT_STENCIL_BIT; return a ? a : VK_IMAGE_ASPECT_COLOR_BIT; } // Binds img into the pool memory behind its GPU address (physical pools, virtual pools through the page table) when // the memory type, alignment and contiguous size allow; 0 otherwise. // A depth / stencil image that needs more bytes than its NVN storage (guest_size) is not bound: AMD's RDNA2 driver // (26.10.1) puts the depth compression metadata after the depth data, and above 1080p that tail lies over the next // texture of the game's heap; each depth write then scrambled it (garbage band at the top, pink glow, blocks in the // grass, a blue checkered scene in the menu). Colour images keep their place: the RX 9070 also sizes some of them // above their storage, and nothing there showed that the GPU writes the excess. static int format_compressed(VkFormat f) { // vkCmdClearColorImage takes no block-compressed format return (f >= VK_FORMAT_BC1_RGB_UNORM_BLOCK && f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK) || (f >= VK_FORMAT_ASTC_4x4_SFLOAT_BLOCK && f <= VK_FORMAT_ASTC_12x12_SFLOAT_BLOCK); } // Image reuse (NX2NAVI_TEX_REUSE, default on): a GPU-only texture that needs an image of its own takes the image of a // released entry made with the same parameters (ImgKey) instead of a new one, which then starts over as a new image // would (init_image: undefined content, cleared). Ocarina places its per-frame render targets with a linear allocator // behind small targets whose size follows an effect on screen (77x77 to 97x97), so every few frames the targets after // them move to new addresses, new descriptors whose images were made again: vkCreateImage, the memory requirements and // the binding of a dedicated allocation cost 8% of the replay in Kokiri Forest. The released entries of each parameter // set are kept in a list, newest release first; create_tex takes the one nearest in address of those released at least // two frames earlier, never during the current submission's replay, whose later parts may still use it, and an // overwritten one (below) before any other. static nvn_hmap reuse = {.klen = sizeof(ImgKey)}; // ImgKey -> the newest released reusable entry static int tex_reuse = -1; static uint64_t n_reused, n_reuse_new; // since the last perf line: images taken from a released entry, made anew static uint64_t n_restarts; // superseded entries taken back (start_at) // Moved render targets. Ocarina keeps about a thousand texture objects for its per-frame render targets and lays the // targets out anew every frame with a linear allocator in one GPU-only pool. A target early in that layout changes its // size with an effect on screen (811x811, then 778x778 a frame later in Kakariko Village), and every target after it // then lies about 100 KB further on: eleven targets, 55 MiB of images, at new addresses in each frame the size // changes, each a new entry with an image of its own, as few of them can lie in the pool memory itself (pool_fits). // The game finalizes an old texture object only when it needs the object again, 90 to 300 frames later, and a // descriptor in the texture pool goes on naming it, so the entries were neither released nor idle: after 6,000 frames // of walking through the village 1,043 of them held 6.1 GiB of device memory, against 233 MiB in the targets still // drawn, and once released they waited GPU_STALE_FRAMES more. A driver that charges device memory to the system's // commit kept the commit low from then on, and the idle textures of the area went and came back (commit_evict). // // On the console those targets are views of one pool's memory: when another target is drawn over a target's bytes, its // content is gone. overwrite_others says so here. When the GPU writes a texture in the GPU-only address window // (nvn_gpu_only), each entry at another address whose storage overlaps it is marked overwritten and released as if // the game had finalized it. Its image then goes to the next texture made with the same parameters (create_tex), or // away OVERWRITTEN_KEEP presents later (drop_overwritten), or as early as an intact entry's while memory is short // (drop_stale). Left alone are an entry at the same address (start_at decides about those), an entry the written // texture lies inside (a level or a layer of it, which a game draws through a texture of its own) and an entry // written in this frame or the one before: two layouts that take turns within a frame keep their images. The mark // goes when the game takes the entry back (entry_of). // gwin: the entries this applies to (reusable, in the window), sorted by address; gwin_span: the largest storage size // one of them had, which bounds how far below an address an overlapping entry can start. #define OVERWRITE_FRAMES 2 #define OVERWRITTEN_KEEP PRESENTS(600) static struct Tex **gwin; static int ngwin, cgwin; static uint64_t gwin_span; static uint64_t n_overwritten, n_over_drops; // since the last perf line: entries marked, dropped by drop_overwritten static int gwin_lower(uint64_t a) { int lo = 0, hi = ngwin; while (lo < hi) { int m = (lo + hi) / 2; if (gwin[m]->key.addr < a) lo = m + 1; else hi = m; } return lo; } static void gwin_add(Tex *e) { if (ngwin == cgwin) { cgwin = cgwin ? 2 * cgwin : 256; if (!(gwin = realloc(gwin, sizeof *gwin * (size_t)cgwin))) rt_fatal("vk: out of memory for the render target list"); } int i = gwin_lower(e->key.addr); memmove(gwin + i + 1, gwin + i, sizeof *gwin * (size_t)(ngwin - i)); gwin[i] = e, ngwin++; if (e->t.L.size > gwin_span) gwin_span = e->t.L.size; e->in_gwin = 1; } static void gwin_remove(Tex *e) { if (!e->in_gwin) return; for (int i = gwin_lower(e->key.addr); i < ngwin && gwin[i]->key.addr == e->key.addr; i++) if (gwin[i] == e) { memmove(gwin + i, gwin + i + 1, sizeof *gwin * (size_t)(ngwin - i - 1)); ngwin--; break; } e->in_gwin = 0; } static int tex_reuse_on(void) { if (tex_reuse < 0) { tex_reuse = cfg_bool("NX2NAVI_TEX_REUSE"); rt_log("vk: texture image reuse %s (NX2NAVI_TEX_REUSE)", tex_reuse ? "on" : "off"); } return tex_reuse; } static void destroy_tex(Tex *e); // Whether an image with these requirements can lie in the pool memory at e's address (*sp: that memory). static int pool_fits(VkCtx *c, const Tex *e, const VkMemoryRequirements *req, uint64_t guest_size, VkMemSpan *sp) { if ((e->f.depth || e->f.stencil) && req->size > guest_size) return 0; if (!req->size) return 0; // an image the device could not lay out (image_size) if (!vk_mem_lookup(e->pub.addr, req->size, sp) || !sp->alloc || sp->alloc_offset % req->alignment) return 0; VmaAllocationInfo info; vmaGetAllocationInfo(c->vma, sp->alloc, &info); return req->memoryTypeBits >> info.memoryType & 1; } // The memory requirements of each parameter set (ImgKey) an image was made with; images made with the same parameters // have the same requirements. With them create_tex makes no image that pool_fits would turn down: Ocarina's moving // render targets mostly miss the device's alignment, and making the image only to read its requirements // (vkGetImageMemoryRequirements does most of the driver's layout work) cost 5% of the replay in Kokiri Forest. static nvn_hmap img_reqs = {.klen = sizeof(ImgKey)}; // ImgKey -> VkMemoryRequirements (malloc'd) static uint64_t n_bind_skips; // since the last perf line #define DEDICATED_MIN ((VkDeviceSize)32 << 20) // create_tex: images this large get memory of their own static int bind_pool_memory(VkCtx *c, Tex *e, VkImage img, uint64_t guest_size) { VkMemoryRequirements req; vkGetImageMemoryRequirements(c->device, img, &req); if (!nvn_hm_get(&img_reqs, &e->ik)) { VkMemoryRequirements *q = malloc(sizeof *q); if (!q) rt_fatal("vk: out of memory"); *q = req; nvn_hm_put(&img_reqs, &e->ik, q); } VkMemSpan sp; if (!pool_fits(c, e, &req, guest_size, &sp)) return 0; if (vmaBindImageMemory2(c->vma, sp.alloc, sp.alloc_offset, img, NULL) != VK_SUCCESS) return 0; e->bound_to = sp.alloc; return 1; } static void img_key_of(const VkImageCreateInfo *ci, ImgKey *k) { memset(k, 0, sizeof *k); k->flags = ci->flags, k->type = ci->imageType, k->format = ci->format, k->w = ci->extent.width; k->h = ci->extent.height, k->d = ci->extent.depth, k->levels = ci->mipLevels, k->layers = ci->arrayLayers; k->samples = ci->samples, k->usage = ci->usage; } // agl's BC4 encoder draws a BC texture's blocks through an RGBA16UI view, one texel per 8-byte block. // Vulkan does it with an uncompressed // view of a BLOCK_TEXEL_VIEW_COMPATIBLE image; EXTENDED_USAGE lets the BC image carry the COLOR_ATTACHMENT usage that // only such views support. Whether the device takes that combination for this format and shape: static int block_views_ok(const VkImageCreateInfo *ci) { VkPhysicalDeviceImageFormatInfo2 fi = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, .format = ci->format, .type = ci->imageType, .tiling = ci->tiling, .usage = ci->usage, .flags = ci->flags}; VkImageFormatProperties2 p = {.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2}; if (vkGetPhysicalDeviceImageFormatProperties2(vk_ctx()->phys, &fi, &p) != VK_SUCCESS) return 0; const VkImageFormatProperties *q = &p.imageFormatProperties; return ci->extent.width <= q->maxExtent.width && ci->extent.height <= q->maxExtent.height && ci->extent.depth <= q->maxExtent.depth && ci->mipLevels <= q->maxMipLevels && ci->arrayLayers <= q->maxArrayLayers; } // The memory the device wants for an image made from ci: 0 when it cannot lay it out. RADV on GFX9 (Vega, Renoir) // hands back a 0-byte image for a mipmapped block-compressed 3D image at least as deep as it is wide in blocks, with // COLOR_ATTACHMENT usage and no STORAGE: that usage asks for a thin 3D layout (Mesa radv_image.c), for which the GFX9 // address library has no swizzle mode left (gfx9addrlib.cpp), and the error is dropped; VMA then refuses the 0-byte // allocation. The format properties call it supported, so only the size shows it. static VkDeviceSize image_size(const VkImageCreateInfo *ci) { VkDeviceImageMemoryRequirements q = {.sType = VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS, .pCreateInfo = ci}; VkMemoryRequirements2 r = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2}; vkGetDeviceImageMemoryRequirements(vk_ctx()->device, &q, &r); return r.memoryRequirements.size; } // UNDEFINED -> GENERAL; new depth / stencil images start cleared (LAS textures.c: the game copies depth // without stencil, and the stencil of those images must not be whatever the memory held). // A reused image (create_tex) was another texture's: the transition waits for every earlier command, which may still // read or write it. static void init_image(Tex *e, VkCommandBuffer cb) { VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = e->reused ? VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT : VK_PIPELINE_STAGE_2_NONE, .srcAccessMask = e->reused ? VK_ACCESS_2_MEMORY_WRITE_BIT : 0, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = e->pub.image, .subresourceRange = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; vkCmdPipelineBarrier2(cb, &di); if (e->f.depth || e->f.stencil) { VkClearDepthStencilValue zero = {0.0f, 0}; VkImageSubresourceRange rg = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; vkCmdClearDepthStencilImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &rg); barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); } else if (vk_fix_zero() && !e->bound_to && !e->pub.mirrored && !format_compressed(e->f.vk)) { // GPU-only storage that got memory of its own starts at zero, as the pool it stands for (vk_memory.c // alloc_physical). Images aliasing pool memory are not cleared: that would erase what other textures rendered // there; mirrored images are uploaded from their storage on first use. VkClearColorValue zero = {0}; VkImageSubresourceRange rg = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; vkCmdClearColorImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &rg); barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); } } // A new entry's first transition. A reused image goes through it at the replay's current position: earlier commands // of the part being recorded may still use the image as the texture it was, and the upload batch runs before the part. static void init_new(Tex *e) { if (e->reused) { int reopen; VkCommandBuffer cb = vk_exec_inline_begin(&reopen); if (cb) { init_image(e, cb); vk_exec_inline_end(reopen); return; } } init_image(e, vk_upload_begin()); vk_upload_end(); } // NX2NAVI_SHADOW_SCALE (1 to 4): the game's shadow maps are made this many times larger on each side. NX2navi renders its // sun cascades into a 960x960 DEPTH16 2D array of 4 layers and its static shadows into 1440x1440 and 720x720 arrays of 2, // whatever the output resolution, and shrinks the cascades under load (872, 848, 800, 480 and so on). Those are made // larger here: render passes into them scale the game's viewports and scissors (vk_exec.c resolve_rt, pass_scale), // copies scale their regions (vk_exec_xfer.c sub_resolve), and shaders sample them with normalised coordinates, so the // game sees nothing of it. A map whose storage the CPU writes (mirrored) or that aliases pool memory keeps its size, // as its texels must match the guest layout. static int shadow_scale(void) { static int scale = -1; if (scale < 0) { int s = cfg_int("NX2NAVI_SHADOW_SCALE"); s = s < 1 ? 1 : s > 4 ? 4 : s; rt_log("vk: shadow map scale %d", s); scale = s; } return scale; } static int shadow_scale_of(const nvn_texdesc *d, const TexVkFormat *f, uint32_t samples, int mirrored) { int scale = shadow_scale(); if (scale == 1 || mirrored || samples > 1 || d->format != NVN_FORMAT_DEPTH16 || !f->depth) return 1; if (d->target != NVN_TEXTURE_TARGET_2D_ARRAY || d->width != d->height || d->width < 256 || d->depth < 2 || d->depth > 4) return 1; uint32_t most = vk_ctx()->props.limits.maxImageDimension2D; return d->width * (uint32_t)scale <= most ? scale : 1; } // NX2NAVI_ASTC_BC_PROBE=1 (astc_bc_probe in nx2navi.ini): measures, without changing any format, how much of the ASTC // set could be compressed if the alpha rule were data-driven instead of footprint-driven: the opaque part could be BC1, // half the size of the BC3 every decoded ASTC texture is now. Each texture's source blocks are decoded on the CPU with // tex_astc.c astc_decode_block (up to 512 spread over the level) and counted by whether any block has alpha below 250. #define ASTC_BC_PROBES 512 static uint64_t n_probe_calls, n_probe_alpha, n_probe_opaque, n_probe_bytes_alpha, n_probe_bytes_opaque; static void astc_probe_alpha(const tex_t *t, const TexVkFormat *f) { uintptr_t mem = (uintptr_t)t->mem; // only a real host pointer can be read: 0 and the GPU address window are not readable from here if (!mem || nvn_gpu_only(mem) || mem < 0x10000 || mem > 0x7ffffffeffffull || !f->astc_bw || !f->astc_bh) return; uint32_t bw = f->astc_bw, bh = f->astc_bh; uint64_t w = t->d.width ? t->d.width : 1, h = t->d.height ? t->d.height : 1; uint64_t bx = (w + bw - 1) / bw, by = (h + bh - 1) / bh, n = bx * by; if (!n) return; uint64_t probes = n < ASTC_BC_PROBES ? n : ASTC_BC_PROBES, bytes = n * 16u; uint8_t rgba[16 * 16 * 4]; // a block decodes to bw x bh texels, up to 12 x 12 (astc_decode_block writes them all) int opaque = 1; for (uint64_t s = 0; s < probes && opaque; s++) { uint64_t i = probes == n ? s : (s * n) / probes; if (!astc_decode_block((const uint8_t *)mem + i * 16u, bw, bh, f->astc_srgb != 0, rgba)) continue; for (int k = 0; k < (int)(bw * bh); k++) if (rgba[k * 4 + 3] < 250) { opaque = 0; break; } } n_probe_calls++; if (opaque) n_probe_opaque++, n_probe_bytes_opaque += bytes; else n_probe_alpha++, n_probe_bytes_alpha += bytes; if (n_probe_calls == 1 || n_probe_calls % 512 == 0) rt_log("vk: ASTC alpha probe: %llu textures, %llu opaque (%.1f MiB) and %llu with alpha (%.1f MiB) so far", (unsigned long long)n_probe_calls, (unsigned long long)n_probe_opaque, n_probe_bytes_opaque / 1048576.0, (unsigned long long)n_probe_alpha, n_probe_bytes_alpha / 1048576.0); } // defer_init: the caller records the image's first transition (init_image or alias_copy). static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { static volatile int32_t n_unmapped, n_unsupported, n_failed, n_unbound, n_thick; VkCtx *c = vk_ctx(); const nvn_texdesc *d = &t->d; if (!c || d->target == NVN_TEXTURE_TARGET_BUFFER) return NULL; TexVkFormat f; if (!tex_vk_format(d->format, &f)) { log_limited(&n_unmapped, "texture format %#x has no Vulkan format: %s", d->format, f.reason); return NULL; } if (!tex_vk_format_supported(&f)) { log_limited(&n_unsupported, "texture format %#x: VkFormat %d lacks features %#x", d->format, (int)f.vk, (unsigned)f.need); return NULL; } // NX2NAVI_ASTC_BC_PROBE (diagnostics only): measure what a data-driven alpha rule would compress static int bc_probe = -1; // read once: this runs for every texture created if (bc_probe < 0) bc_probe = cfg_bool("NX2NAVI_ASTC_BC_PROBE"); if (f.astc_bw && bc_probe) astc_probe_alpha(t, &f); nvn_texlayout L; nvn_tex_layout(d, &L); uint32_t w = umax(d->width, 1), h = umax(d->height, 1), depth = 1, layers = 1; VkImageType type = VK_IMAGE_TYPE_2D; switch (d->target) { case NVN_TEXTURE_TARGET_1D: type = VK_IMAGE_TYPE_1D; h = 1; break; case NVN_TEXTURE_TARGET_1D_ARRAY: type = VK_IMAGE_TYPE_1D; h = 1; layers = L.layers; break; case NVN_TEXTURE_TARGET_3D: type = VK_IMAGE_TYPE_3D; depth = umax(d->depth, 1); break; case NVN_TEXTURE_TARGET_2D_ARRAY: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY: case NVN_TEXTURE_TARGET_CUBEMAP: case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: layers = L.layers; break; default: break; } int scale = shadow_scale_of(d, &f, d->samples > 1 ? (uint32_t)d->samples : 1, tex_mirrored(t)); w *= (uint32_t)scale, h *= (uint32_t)scale; uint32_t levels = umax(d->levels, 1), most = 1; for (uint32_t s = umax(umax(w, h), depth); s > 1; s >>= 1) most++; if (levels > most) levels = most; uint32_t samples = d->samples > 1 ? (uint32_t)d->samples : 1; if (samples > 1) levels = 1; VkFormatFeatureFlags ff = features_of(f.vk); VkImageUsageFlags usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; int ds = f.depth || f.stencil; if (ds && (ff & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT; if (!ds && (ff & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; if ((d->flags & NVN_TEXTURE_FLAGS_IMAGE) && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1) usage |= VK_IMAGE_USAGE_STORAGE_BIT; // NVN2 titles (API 200+) number the texture flags differently: Ocarina's compute targets (the title movie's RGBA8 // frames) carry 0x1, NVN1's DISPLAY. Every single-sample colour texture whose format allows it can be a storage // image there, so a compute write into one is never dropped. if (rt_nvn_api_version[0] >= 200 && !ds && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1) usage |= VK_IMAGE_USAGE_STORAGE_BIT; // contact shadows (vk_contact.c) darken the game's B10G11R11 scene colour in place from a compute shader if (f.vk == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) && samples == 1 && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && w >= 640 && vk_contact_strength()) usage |= VK_IMAGE_USAGE_STORAGE_BIT; VkImageCreateFlags flags = ds ? 0 : VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; // format-reinterpreting views // Cube compatible only for cube targets: the replay makes cube views of nothing else (vk_exec_draw.c get_view checks // the image's own view type). The flag on every square array of 6+ layers changed how AMD's Windows driver lays such // an image out, and terrain material arrays sampled through it came out in rainbow blocks that moved with the mip // level (seen on an RX 580 and a 7800 XT). if (type == VK_IMAGE_TYPE_2D && layers >= 6 && w == h && samples == 1 && (d->target == NVN_TEXTURE_TARGET_CUBEMAP || d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY)) flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT; // NVN renders into a 3D texture slice by slice (a view's min layer is the depth plane, nvn_mtl.m), e.g. the // grading LUT of the menu background blur. Vulkan needs a 2D view of the slice for that. if (type == VK_IMAGE_TYPE_3D && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT; VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = flags, .imageType = type, .format = f.vk, .extent = {w, h, depth}, .mipLevels = levels, .arrayLayers = layers, .samples = (VkSampleCountFlagBits)samples, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = usage, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; int bw, bh, block_views = 0; // NX2NAVI_BLOCK_VIEWS_3D (default 0): a 3D BC image also gets block views. On RDNA2 cards (driver 26.10.1, issue #5) // the 32x32x32 BC4 volume the game creates at the logo, made BLOCK_TEXEL_VIEW_COMPATIBLE and 2D_ARRAY_COMPATIBLE, // took a GPU write just past the end of its memory: device lost at the first submit after it when the image ended // its memory block. Draws into a 3D BC image go through a stand-in instead (vk_exec.c bc_target). static int bv3d = -1; if (bv3d < 0) { bv3d = rt_knob("BLOCK_VIEWS_3D", "0"); rt_log("vk: 3D BC textures %s (NX2NAVI_BLOCK_VIEWS_3D=%d)", bv3d ? "get block views (renderable)" : "have no block views", bv3d); } if (!ds && samples == 1 && f.vk == f.ideal && nvn_fmt_bpb(d->format, &bw, &bh) && bw == 4 && bh == 4 && (type != VK_IMAGE_TYPE_3D || bv3d)) { VkImageCreateInfo bc = ci; // BC formats: renderable through views of their blocks (block_views_ok) bc.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; bc.flags |= VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; if (type == VK_IMAGE_TYPE_3D) bc.flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT; VkImageUsageFlags hidden = 0; // creation usage the replay never uses (views restrict theirs, vk_exec.c) int ok = block_views_ok(&bc); if (ok && type == VK_IMAGE_TYPE_3D && !image_size(&bc)) { // STORAGE usage keeps RADV on GFX9 from the thin layout (image_size), so agl's BC4 encoder still renders // into the volume; else the image is made without block views and its encoder passes are left out bc.usage |= VK_IMAGE_USAGE_STORAGE_BIT, hidden = VK_IMAGE_USAGE_STORAGE_BIT; ok = block_views_ok(&bc) && image_size(&bc); log_limited(&n_thick, "texture fmt %#x %ux%ux%u: the device cannot lay out a 3D image with block views; %s", d->format, w, h, depth, ok ? "made with storage usage (a thick layout)" : "made without them"); } if (ok) ci = bc, usage = bc.usage & ~hidden, flags = bc.flags, block_views = 1; } // Feedback loops (vk.h VkCtx.feedback): every attachable image may be sampled by a draw of a pass that renders it, // so it gets the usage now; adding it when a loop shows up would mean recreating the image (that lost the // grass tile maps once). int fb_usage = c->feedback == VK_FEEDBACK_LAYOUT && !block_views && (usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)); if (fb_usage) ci.usage = usage |= VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; Tex *e = calloc(1, sizeof *e); if (!e) rt_fatal("vk: out of memory for a texture"); e->key = *k; e->t = *t; e->f = f; e->pub.addr = k->addr; img_key_of(&ci, &e->ik); VkImage img = VK_NULL_HANDLE; if (scale == 1 && nvn_gpu_only(k->addr)) { // storage in a physical or virtual pool: alias the pool memory (section 3.2) const VkMemoryRequirements *q = nvn_hm_get(&img_reqs, &e->ik); VkMemSpan sp; if (q && !pool_fits(c, e, q, L.size, &sp)) { // it would not bind there (img_reqs) n_bind_skips++; log_limited(&n_unbound, "texture %#llx (%ux%u fmt %#x) cannot alias its pool memory; own memory instead", (unsigned long long)k->addr, w, h, d->format); } else if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS && !bind_pool_memory(c, e, img, L.size)) { vkDestroyImage(c->device, img, NULL); img = VK_NULL_HANDLE; log_limited(&n_unbound, "texture %#llx (%ux%u fmt %#x) cannot alias its pool memory; own memory instead", (unsigned long long)k->addr, w, h, d->format); } } // A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: // the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load // (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that // 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom // thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference. enum { SUBALLOC_MAX = 32 << 20 }; if (!img && tex_mirrored(t) && samples == 1) { if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS) { // an image the driver requires (not only prefers) dedicated memory for keeps it: vmaCreateImage below VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; VkMemoryRequirements2 mr2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr}; VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; vkGetImageMemoryRequirements2(c->device, &ri, &mr2); VkMemoryRequirements mr = mr2.memoryRequirements; VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && vmaAllocateMemory(c->vma, &mr, &ai, &e->alloc, NULL) == VK_SUCCESS; if (ok && vmaBindImageMemory(c->vma, e->alloc, img) != VK_SUCCESS) { vmaFreeMemory(c->vma, e->alloc); ok = 0; } if (!ok) { vkDestroyImage(c->device, img, NULL); img = VK_NULL_HANDLE, e->alloc = NULL; } } else { img = VK_NULL_HANDLE; } } int gpu_only = !tex_mirrored(t); if (!img && gpu_only && tex_reuse_on()) { // the image of a released entry with these parameters (reuse, above) // Not one released in this frame or the one before: Ocarina releases its render targets and registers them // again every frame, and DLSS's scene target (2560x1440 at a fixed address in Kokiri Forest) lost its image // to a moving target of the same size in between, and DLSS its history (vk_dlaa.c evaluate), several times a // second. Not one at this address either: its own descriptor comes back for it (start_at). Of the others the // one nearest in address, so a target the allocator moves keeps the image it had; an overwritten one first // (overwrite_others): nothing can ask for its content again. uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); Tex *a = NULL, *kept = NULL; uint64_t best = UINT64_MAX, best_kept = UINT64_MAX; int n = 0; for (Tex *q = nvn_hm_get(&reuse, &e->ik); q && n < 128; q = q->ru_next, n++) { if (q->rel_epoch == submit_epoch || now - q->rel_frame < 2 || q->key.addr == k->addr) continue; uint64_t dist = q->key.addr > k->addr ? q->key.addr - k->addr : k->addr - q->key.addr; if (q->overwritten) { if (dist < best) best = dist, a = q; } else if (dist < best_kept) best_kept = dist, kept = q; } if (!a) a = kept; if (a) { img = a->pub.image, e->alloc = a->alloc; vk_exec_image_gone(img); // the views and records of the texture it was a->pub.image = VK_NULL_HANDLE, a->alloc = NULL; destroy_tex(a); e->reused = 1; n_reused++; } } if (!img) { if (gpu_only) n_reuse_new++; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; // An image of DEDICATED_MIN or more gets memory of its own: freed inside one of VMA's 256 MiB blocks it left a hole // the driver went on counting, with the rest of the block, against the budget and the system's commit (0.5 to 1.8 // GiB of such free space in a session at 5120x2160), until the block emptied; its own memory goes back when it does. const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, &e->ik); if ((known ? known->size : image_size(&ci)) >= DEDICATED_MIN) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); if (r != VK_SUCCESS) { log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r, d->format, w, h, depth, d->target, levels, layers); free(e); return NULL; } } if (e->alloc) { VmaAllocationInfo own; vmaGetAllocationInfo(c->vma, e->alloc, &own); e->bytes = own.size; } e->reusable = e->alloc && gpu_only; e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (e->reusable && nvn_gpu_only(k->addr)) gwin_add(e); // A new image's handle can be one a destroyed image had. Views are purged with the image (destroy_tex), so none // should be cached under it: one that is was made from a destroyed texture's record after its purge, and would // reach this image's memory through the dead one's. Dropped here, and counted in the log. if (!e->reused) { static volatile int32_t n_stale; int stale = vk_exec_image_gone(img); if (stale) log_limited(&n_stale, "new image %#llx had %d cached views of a destroyed image with its handle; dropped", (unsigned long long)(uintptr_t)img, stale); } VkTexImage *p = &e->pub; p->image = img; p->format = f.vk; p->type = type; p->usage = usage; p->aspect = aspect_of(&f); p->width = w, p->height = h, p->depth = depth, p->levels = levels, p->layers = layers, p->samples = samples; p->view_type = type == VK_IMAGE_TYPE_1D ? (d->target == NVN_TEXTURE_TARGET_1D_ARRAY ? VK_IMAGE_VIEW_TYPE_1D_ARRAY : VK_IMAGE_VIEW_TYPE_1D) : type == VK_IMAGE_TYPE_3D ? VK_IMAGE_VIEW_TYPE_3D : d->target == NVN_TEXTURE_TARGET_CUBEMAP && (flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) ? VK_IMAGE_VIEW_TYPE_CUBE : d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY && (flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) ? VK_IMAGE_VIEW_TYPE_CUBE_ARRAY : layers > 1 || d->target == NVN_TEXTURE_TARGET_2D_ARRAY || d->target == NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D; p->alpha_one = f.alpha_one; memcpy(p->chan, f.chan, sizeof p->chan); p->swz[0] = (uint8_t)d->swizzle_r, p->swz[1] = (uint8_t)d->swizzle_g; p->swz[2] = (uint8_t)d->swizzle_b, p->swz[3] = (uint8_t)d->swizzle_a; if (!(p->swz[0] | p->swz[1] | p->swz[2] | p->swz[3])) // descriptors built without nvnTextureBuilderSetDefaults p->swz[0] = NVN_TEXTURE_SWIZZLE_R, p->swz[1] = NVN_TEXTURE_SWIZZLE_G, p->swz[2] = NVN_TEXTURE_SWIZZLE_B, p->swz[3] = NVN_TEXTURE_SWIZZLE_A; p->mirrored = (uint8_t)tex_mirrored(t); p->bound = e->bound_to != NULL; p->block_views = (uint8_t)block_views; p->fb_usage = (uint8_t)fb_usage; p->scale = (uint8_t)scale; if (!defer_init) init_new(e); return e; } static int owned_lower(uint64_t a) { int lo = 0, hi = nowned; while (lo < hi) { int m = (lo + hi) / 2; if (owned[m]->pub.addr < a) lo = m + 1; else hi = m; } return lo; } static void own_add(Tex *e) { if (e->listed) return; plat_mutex_lock(&owned_lock); if (nowned == cowned) { cowned = cowned ? 2 * cowned : 256; if (!(owned = realloc(owned, sizeof *owned * (size_t)cowned))) rt_fatal("vk: out of memory for the owned-texture list"); } int i = owned_lower(e->pub.addr); memmove(owned + i + 1, owned + i, sizeof *owned * (size_t)(nowned - i)); owned[i] = e; __atomic_store_n(&nowned, nowned + 1, __ATOMIC_RELEASE); if (e->t.L.size > max_owned) max_owned = e->t.L.size; e->listed = 1; plat_mutex_unlock(&owned_lock); } static void own_remove(Tex *e) { if (!e->listed) return; plat_mutex_lock(&owned_lock); for (int i = owned_lower(e->pub.addr); i < nowned && owned[i]->pub.addr == e->pub.addr; i++) if (owned[i] == e) { memmove(owned + i, owned + i + 1, sizeof *owned * (size_t)(nowned - i - 1)); __atomic_store_n(&nowned, nowned - 1, __ATOMIC_RELEASE); break; } e->listed = 0; plat_mutex_unlock(&owned_lock); } static uint64_t rel_bytes; // own memory of the released entries (Tex.bytes) static void ru_link(Tex *e) { Tex *head = nvn_hm_get(&reuse, &e->ik); e->ru_prev = NULL, e->ru_next = head; if (head) head->ru_prev = e; nvn_hm_put(&reuse, &e->ik, e); e->ru_on = 1; } static void ru_unlink(Tex *e) { if (!e->ru_on) return; if (e->ru_prev) e->ru_prev->ru_next = e->ru_next; else if (e->ru_next) nvn_hm_put(&reuse, &e->ik, e->ru_next); else nvn_hm_del(&reuse, &e->ik); if (e->ru_next) e->ru_next->ru_prev = e->ru_prev; e->ru_prev = e->ru_next = NULL; e->ru_on = 0; } static void rel_append(Tex *e) { e->rel_next = NULL; e->rel_prev = rel_tail; if (rel_tail) rel_tail->rel_next = e; else rel_head = e; rel_tail = e; rel_bytes += e->bytes; if (e->reusable && tex_reuse_on()) ru_link(e); } static void rel_unlink(Tex *e) { ru_unlink(e); rel_bytes -= e->bytes; if (e->rel_prev) e->rel_prev->rel_next = e->rel_next; else rel_head = e->rel_next; if (e->rel_next) e->rel_next->rel_prev = e->rel_prev; else rel_tail = e->rel_prev; e->rel_prev = e->rel_next = NULL; } static void retire_remove(Tex *e); static void destroy_tex(Tex *e) { if (e->retiring) retire_remove(e); own_remove(e); gwin_remove(e); n_site[0]++; tex_gen_bump(e->key.addr); nvn_hm_del(&by_key, &e->key); Tex *head = nvn_hm_get(&by_addr, &e->key.addr); if (head == e) { if (e->next_at) nvn_hm_put(&by_addr, &e->key.addr, e->next_at); else nvn_hm_del(&by_addr, &e->key.addr); } else { for (Tex *p = head; p; p = p->next_at) if (p->next_at == e) { p->next_at = e->next_at; break; } } if (e->released) nreleased--, rel_unlink(e); if (e->pub.image) vk_exec_image_gone(e->pub.image); // none when another texture took it (create_tex, reuse) vk_defer_image(e->pub.image, e->alloc); free(e->snap); // The record goes once the GPU is past the submission being recorded, as its image: the replay keeps &e->pub as a // render target for the rest of a command buffer (vk_exec.c resolve_target) and DLSS for the frame, and this runs on // whichever thread released the texture. Freed here, a pass that began again read the freed record, cached a view of // the dead image (vk_exec_target_view), and a later image given the same handle drew through it into freed memory. e->pub.gone = 1; vk_defer_free(e); } // Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries the // oldest one goes at each release (it comes back on demand, a mirrored one re-uploaded from its storage). NX2navi // finalizes and initializes transient textures every frame, so a released entry is mostly taken back within frames. // Dropping all of them at once when the count passed 4096 destroyed thousands of images in one frame: 333-422 ms // frames every 40-50 s on a GTX 1050 Ti, then the textures still in use were created and uploaded again. #define RELEASED_MAX 2048 static void sweep_released(void) { while (nreleased > RELEASED_MAX && rel_head) destroy_tex(rel_head); } // NX2NAVI_TEX_VRAM_PRESSURE (percent of the device-local budget, default VRAM_PRESSURE_DEFAULT; 0 = off): above it, each // submit drops up to PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released // images (each ASTC one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason // above. Only entries that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their // storage. A render target lives only in its image; dropping it brought the texture back empty (black flashes in a // Depths boss fight, ~170 "left uninitialised" aliases a minute). // // The default lies under the seven eighths of the budget past which a pool gets no VRAM mirror (vk_memory.c // mirror_create), and the evictions aim 2% under it, so textures give way before mirrors are refused: on an 8 GiB card // that leaves the mirrors about 0.7 GiB, on a 24 GiB one 2 GiB. With the default at 90 the order was the other way // round: at 88% of a 22 GiB budget, full of textures of areas left long before, 0.4 MiB mirrors were refused and the // draws that read their pools over PCIe brought the game to 11-20 fps, while no texture had been asked to go. #define VRAM_PRESSURE_DEFAULT 80 #define PRESSURE_STEP 32 static int vram_pressure = -1; static uint64_t n_pressure_drops; static int comes_back_intact(const Tex *e) { return e->pub.mirrored && !e->pub.gpu_owned && !e->gpu_dirty; } // When a lookup last reached an entry: entry_of (every texture get and slot resolution) and the per-submit check of a // slot lookup (vk_texture_by_id_locked). The texture pool refresh looks every registered slot up once in each // submission that draws through the pool (vk_exec_draw.c pool_sync_entries), so an entry no lookup reached for // IDLE_ROUNDS such submissions and STALE_FRAMES presents (below) has no live descriptor naming it: the game keeps the // texture object, nothing samples it. The clock is look_round, which only a submission whose slot checks reached half // of the registered pool slots moves on (vk_texture_new_submit): counted in submissions, a pause screen or a fade that // draws nothing through the pool made the whole world's textures idle within seconds. #define IDLE_ROUNDS 240 static uint32_t look_round = 1; // submissions whose lookups walked the texture pool static uint32_t round_checks; // slot checks of the submission being recorded (vk_texture_by_id_locked) static uint32_t pool_slots; // registered slots the pool covers (id < VK_POOL_TEX) static int pool_ctx; // the texture pool refresh is looking slots up (vk_texture_pool_ctx) static void touch(Tex *e) { e->use_round = look_round; e->use_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (!pool_ctx) e->samp_frame = e->use_frame; // a draw binds it, or the game or the replay reads it otherwise } void vk_texture_pool_ctx(int on) { pool_ctx = on; } static int idle(const Tex *e, uint64_t now); // NX2NAVI_TEX_RELEASED_MB (default a thirty-second of the installed memory, 512 MiB to 2 GiB; 0 = no limit) bounds the // own memory of the released entries: RELEASED_MAX alone kept 5-7 GiB of released render targets and decoded textures // after an hour of play at a 2160p game resolution, device memory the driver also charges to the system's commit (the // process's private bytes reached 31 GiB). A sixteenth (1 to 4 GiB) still let them hold 1.3 GiB on a 32 GB PC whose // commit then ran out at an area load (device lost); only entries released STALE_FRAMES presents ago go, so the // transient ones the game takes back within frames stay. NX2NAVI_TEX_COMMIT_FREE_MB (default the larger of 4 GiB and a tenth of the commit // limit; 0 = off) lowers that bound to COMMIT_LOW_KEEP while less of the system's commit is free: an allocation that // fails there ends the game. On Linux nothing fails at the limit: the OOM killer ends the game when its cgroup or the // machine runs out of memory, and plat_commit_info reports the tighter of the two. Both drop only stale entries // (drop_stale), a few per check like the VRAM pressure. // // NX2NAVI_TEX_PROC_COMMIT_MB (default the installed memory, at least PROC_COMMIT_MIN_MB; 0 = off) counts the commit as // low too while this process's own commit is above it. The system's figure depends on the page file and on what else // runs: with a 64 GB limit it turned low at 59 GB in use, when this process alone held 42 GB on a PC with 32 GB of // memory, about half of it device memory, with the textures of five areas left behind. A process is not meant to need // more commit than the PC has memory: beyond that it lives on the page file, whatever its size. The bound holds until // the process is a thirty-second under it, so that it does not flip with every texture freed. The floor keeps it out // of the way on PCs with less memory, where a bound at the installed memory could hold all through play: the system's // figure decides there, as before. On Linux the process's figure is its resident anonymous memory, which cannot pass // the installed memory, so only a bound set below that holds there; a cgroup's limit and the machine's RAM act through // the system's figure. #define COMMIT_LOW_KEEP ((uint64_t)256 << 20) #define PROC_COMMIT_MIN_MB 24576 static uint64_t released_cap; // bytes; 0 = not read yet, UINT64_MAX = no limit static long long commit_free_mb = -2; // -2: not read yet, -1: the default static long long proc_commit_mb; // this process's bound in MB; 0 = none static int commit_low, commit_logs; // commit_low: either of the two below at the last check static int sys_low, proc_high; // the system's commit is low; this process's is above its bound static uint64_t n_cap_drops, n_commit_drops, n_commit_past; static uint64_t released_cap_bytes(void) { if (!released_cap) { const char *s = cfg_get("NX2NAVI_TEX_RELEASED_MB"); long long mb; if (s && *s) mb = atoll(s); else { float used, total, proc; plat_mem_info(&used, &total, &proc); mb = total > 0 ? (long long)(total / 32) : 1024; mb = mb < 512 ? 512 : mb > 2048 ? 2048 : mb; } released_cap = mb > 0 ? (uint64_t)mb << 20 : UINT64_MAX; if (mb > 0) rt_log("vk: released textures keep at most %lld MiB of their own memory", mb); else rt_log("vk: released textures: no size limit"); } return released_cap; } static int commit_is_low(void) { if (commit_free_mb == -2) { const char *s = cfg_get("NX2NAVI_TEX_COMMIT_FREE_MB"); commit_free_mb = s && *s ? atoll(s) : -1; if (commit_free_mb < -1) commit_free_mb = -1; s = cfg_get("NX2NAVI_TEX_PROC_COMMIT_MB"); proc_commit_mb = s && *s ? atoll(s) : -1; if (proc_commit_mb < 0) { float ram_used, ram_total, ram_proc; plat_mem_info(&ram_used, &ram_total, &ram_proc); proc_commit_mb = ram_total > PROC_COMMIT_MIN_MB ? (long long)ram_total : PROC_COMMIT_MIN_MB; } if (!plat_commit_enforced()) commit_free_mb = proc_commit_mb = 0; // no limit known char note[2048]; // the platform layer's own lines go to stderr only, so the log file gets this through here if (plat_commit_note(note, sizeof note)) rt_log("platform: %s", note); if (proc_commit_mb) rt_log("vk: textures give way as under low commit while this process's commit is above %lld MB " "(NX2NAVI_TEX_PROC_COMMIT_MB; 0 = off)", proc_commit_mb); } if (!commit_free_mb && !proc_commit_mb) return 0; double proc, used, limit; plat_commit_info(&proc, &used, &limit); int low = 0; if (commit_free_mb && used >= 0 && limit > 0) { double want = commit_free_mb > 0 ? (double)commit_free_mb : limit / 10 > 4096 ? limit / 10 : 4096; low = limit - used < want; } int high = proc_commit_mb && proc > (double)(proc_high ? proc_commit_mb - proc_commit_mb / 32 : proc_commit_mb); if (high != proc_high && commit_logs < 40) { commit_logs++; rt_log("vk: this process's commit %s: %.0f MB, the bound is %lld MB (system: %.0f of %.0f MB in use); released " "textures kept %d, %.0f MiB of own memory%s", high ? "high" : "no longer high", proc, proc_commit_mb, used, limit, nreleased, rel_bytes / 1048576.0, high ? ": dropping the stale ones" : ""); } proc_high = high; if (low != sys_low && commit_logs < 40) { // a few lines, however often it flips commit_logs++; rt_log("vk: system commit %s: %.0f of %.0f MB in use (this process %.0f MB); released textures kept %d, %.0f MiB " "of own memory%s", low ? "low" : "no longer low", used, limit, proc, nreleased, rel_bytes / 1048576.0, low ? ": dropping the stale ones" : ""); } sys_low = low; return commit_low = low || high; } int vk_texture_commit_low(void) { return __atomic_load_n(&commit_low, __ATOMIC_RELAXED); } // Drops up to PRESSURE_STEP released entries that own memory and were released STALE_FRAMES presents ago or earlier, // oldest first and the intact ones before the others, while the released entries own more than keep. A transient // texture is taken back within frames, so dropping it frees nothing: the game initializes it again and it is created // again, empty if it is a render target. Under commit pressure that a system-managed page file never relieves, dropping // young ones destroyed 2160p render targets about every frame for whole sessions. Frame generation also reads a frame's // depth target at the host present (vk_dlssg.c vk_dlssg_frame tags it), after the blit deferred destruction waits for. // // A released entry is not dead to the game. A descriptor registered for a render target still resolves to its released // entry (slot_resolve: a finalized slot with no live texture at its storage gets its own), and the game can sample it // or take it back long after the release. In Kakariko Village, with the system's commit low, render targets dropped at // STALE_FRAMES came back empty and the scene was drawn black for one frame every two to three seconds (interface // intact), for as long as the commit stayed low. So no entry goes while lookups still reach it (idle), and one whose // content exists only in its image waits GPU_STALE_FRAMES, or GAP_TIMES the longest it stayed released before the game // took it back when that is longer. An overwritten entry has no content left to wait for (overwrite_others): it goes // with the intact ones, whether or not a descriptor still names it. #define STALE_FRAMES PRESENTS(120) #define GPU_STALE_FRAMES PRESENTS(1800) #define GAP_TIMES 8 static int idle(const Tex *e, uint64_t now) { return look_round - e->use_round >= IDLE_ROUNDS && now - e->use_frame >= STALE_FRAMES; } static uint64_t stale_after(const Tex *e) { if (comes_back_intact(e) || e->overwritten) return STALE_FRAMES; uint64_t gap = (uint64_t)e->back_gap * GAP_TIMES; return gap > GPU_STALE_FRAMES ? gap : GPU_STALE_FRAMES; } static int drop_stale(uint64_t keep, uint64_t *count) { // returns the entries dropped: PRESSURE_STEP, more may follow uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); int dropped = 0; const uint64_t stale = STALE_FRAMES; for (int any = 0; any < 2; any++) for (Tex *e = rel_head, *next; e && now - e->rel_frame >= stale; e = next) { // in release order if (dropped == PRESSURE_STEP || rel_bytes <= keep) return dropped; next = e->rel_next; if (!e->bytes || (!any && !comes_back_intact(e) && !e->overwritten)) continue; if (now - e->rel_frame < stale_after(e) || (!e->overwritten && !idle(e, now))) continue; destroy_tex(e); dropped++, (*count)++; } return dropped; } // Drops up to PRESSURE_STEP entries that were marked overwritten OVERWRITTEN_KEEP presents ago or earlier // (overwrite_others): the image reuse found no texture to give their images to in that time. static void drop_overwritten(void) { uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); int dropped = 0; for (Tex *e = rel_head, *next; e && now - e->rel_frame >= OVERWRITTEN_KEEP && dropped < PRESSURE_STEP; e = next) { next = e->rel_next; if (!e->overwritten || now - e->ow_frame < OVERWRITTEN_KEEP) continue; destroy_tex(e); dropped++, n_over_drops++; } } // NX2NAVI_TEX_EVICT (default on; 0 = only released entries go under VRAM pressure, as before): above the VRAM pressure // threshold, once the released entries that come back intact are gone, live entries go too, least recently looked up // first: those that come back intact and that no lookup reached for IDLE_ROUNDS pool walks and STALE_FRAMES presents // (touch: no live descriptor names them). A later lookup creates the image again and uploads it from its storage, as // for a texture seen the first time. Up to EVICT_STEP entries per check, until the device-local use is 2% of the budget // under the threshold. The use counts the free space inside VMA's blocks as free (vk_mem_vram_live): a freed texture // leaves a hole the driver still counts, and measured that way the evictions went on until nearly every idle texture // was gone. The next check waits until the GPU is past the submissions that may still name the images, as their // memory goes back to VMA only then (vk_defer_image). A VRAM mirror that did not fit asks for room the same way, below // the threshold too (vk_texture_want_room): a pool without its mirror costs far more than a texture uploaded again. // Its request stands as the use the mirror fits at (room_level) for ROOM_CHECKS checks or until the use is down there // (sweep_pressure): taken as bytes to free at the one check that read it, a request for 205 MiB got the EVICT_STEP // textures of that check, 26 MiB, or nothing when the check fell before evict_wait, and was gone. #define EVICT_STEP 64 #define ROOM_CHECKS 64 static int tex_evict = -1; static uint64_t evict_wait; // vk_sync_fence_done value the next eviction waits for static uint64_t n_evict, n_evict_bytes; // since the last perf line static int evict_logs; static volatile int64_t room_wanted; // bytes refused VRAM mirrors asked for (vk_texture_want_room) static uint64_t room_level; // tex_lock: the device-local use at which those mirrors fit static uint32_t room_checks; // tex_lock: checks the request still stands for; 0 = none stands void vk_texture_want_room(uint64_t bytes) { int64_t cur = __atomic_load_n(&room_wanted, __ATOMIC_RELAXED); // the largest request since the last check while ((int64_t)bytes > cur && !__atomic_compare_exchange_n(&room_wanted, &cur, (int64_t)bytes, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} } static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex_lock held; returns the bytes freed if (vk_sync_fence_done() < evict_wait) return 0; Tex *v[EVICT_STEP]; int nv = 0; uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP oldest candidates, oldest first Tex *e = by_key.vals[i]; if (!e || e->released || e->retiring || !e->bytes || !comes_back_intact(e) || !idle(e, now)) continue; uint32_t age = look_round - e->use_round; int at = nv; while (at > 0 && look_round - v[at - 1]->use_round < age) at--; if (at == EVICT_STEP) continue; if (nv < EVICT_STEP) nv++; memmove(v + at + 1, v + at, sizeof *v * (size_t)(nv - 1 - at)); v[at] = e; } uint64_t freed = 0; int n = 0; for (; n < nv && freed < want; n++) freed += v[n]->bytes, destroy_tex(v[n]); if (!n) return 0; n_evict += (uint64_t)n, n_evict_bytes += freed; evict_wait = vk_sync_issued() + 2; if (evict_logs < 8 || ((rt_trace & RT_TRACE_PERF) && evict_logs < 64)) { evict_logs++; rt_log("vk: VRAM %.0f of %.0f MiB in use: %d idle textures evicted (%.1f MiB), least recently used first; they " "are uploaded again when the game uses them", used / 1048576.0, budget / 1048576.0, n, freed / 1048576.0); } return freed; } // ---- disuse (NX2NAVI_TEX_DISUSE, default on; vk_pipeline.c "texture pool marks", vk_exec_draw.c "pool marks") // Ocarina keeps the textures of the areas it has loaded registered in the pool, and the pool refresh looks every // registered slot up, so idle (above) never holds for them, and their memory, which the driver charges to the system's // commit, only grew. Marked shaders tell which slots they sample: vk_pool_marks_read sets samp_frame of the texture of // each marked slot. Where evict_idle frees too little, evict_disused takes live textures that come back intact and that // nothing sampled for DISUSE_FRAMES presents, least recently sampled first, in two steps: // - retired (retire): the entry stays whole, but the pool refresh no longer resolves a slot to it (slot_resolve), so // every slot that names it gets the kind's stand-in. Any use in the meantime takes it back as it is (retire_cancel): // a lookup other than the pool refresh, or a mark on one of its slots (the stand-in was sampled). // - destroyed (retire_sweep) once the texture pool has taken every set of its ring since, each brought up to date // when taken (vk_exec_draw.c pool_take_set, pool_catch_up), after a sync that left no slot behind wrote its stand-in // (vk_pool_takes counts only such takes), and the GPU has finished the submissions recorded until // then: no set can be bound with its view any more, and no submission that bound one is still running. Its key is // held then: a held key makes no entry in the pool refresh (held_out), so the slot keeps its stand-in; a mark on the // slot drops the hold and the next refresh makes the texture again, as does any other lookup. // An image freed at the eviction itself, as before, faulted the GPU once (a read of freed memory in a pass, 0.6.1.14) // without a path to it found in the code; the delay leaves no window in the pool sets, and the cancels count uses the // marks missed. NX2NAVI_TEX_DISUSE=2 retires and never destroys (stand-ins only, nothing freed): it tells the marks and // stand-ins from the freeing as the cause of a fault. Only while every module that indexes the pool was marked and two // reads arrived: an unmarked shader could sample what the marks call unused. // Two minutes, not the thirty seconds it was: a texture out of view that long and then seen again shows its stand-in // for the frames until its mark is read, which is a flash of missing ground or a dark object each time the camera // turns back, and a batch of such textures gave back 0.4 to 31 MiB in a recorded session. The textures of an area // left behind, which are what the memory grows by, still go two minutes after it. #define DISUSE_SECONDS 120 #define DISUSE_FRAMES PRESENTS(DISUSE_SECONDS * 60) #define HOLDS_MAX 65536 static nvn_hmap holds = {.klen = sizeof(TexKey)}; // value: 1 static uint32_t marks_reads; // Whether the scene is being drawn, from the marks: marks_peak is the most slots one read marked lately (in 1/256, // halving in about a minute of reads), marks_live the reads in a row that marked at least a quarter of it. A pause // screen, a menu or a fade draws next to nothing through the pool for longer than DISUSE_SECONDS: every texture of the // scene behind it then counted as unsampled, went under memory pressure, and the scene came back as stand-ins. // evict_disused waits for two such reads in a row: the first one after a pause is what notes the scene's textures as // sampled again. static uint64_t marks_peak; static uint32_t marks_live; static int resolve_held; // slot_resolve left the slot out for a held key or a retired entry static uint64_t n_disuse, n_disuse_bytes, n_disuse_back; // since the last perf line static uint64_t n_retire_cancel, n_retire_freed; // since the last perf line static int disuse_logs; static Tex **retiring; // retired entries not destroyed yet static int nretiring, cretiring; static uint64_t retiring_bytes; // Takes a retired entry back as it is: something uses it. Its slots resolve to it again (the bump). static void retire_cancel(Tex *e) { e->retiring = 0; // it leaves retiring[] at the next retire_sweep retiring_bytes -= e->bytes; if (holds.n) nvn_hm_del(&holds, &e->key); n_retire_cancel++, n_disuse_back++; tex_gen_bump(e->key.addr); } static void retire_remove(Tex *e) { // destroy_tex: the entry goes another way (released and dropped, shutdown) for (int i = 0; i < nretiring; i++) if (retiring[i] == e) { retiring[i] = retiring[--nretiring]; break; } if (e->retiring) e->retiring = 0, retiring_bytes -= e->bytes; } // Each submission: destroys the retired entries the pool sets and the GPU are done with (above). TEX_DISUSE=2 keeps them. static void retire_sweep(void) { if (!nretiring) return; static int keep = -1; if (keep < 0) keep = rt_knob("TEX_DISUSE", "1") == 2; uint64_t takes = vk_pool_takes(), done = vk_sync_fence_done(); int k = 0; for (int i = 0; i < nretiring; i++) { Tex *e = retiring[i]; if (!e->retiring) continue; // taken back if (!e->retire_due && takes - e->retire_takes > VK_POOL_SETS) e->retire_due = vk_sync_issued(); if (!keep && e->retire_due && done >= e->retire_due) { e->retiring = 0, retiring_bytes -= e->bytes; nvn_hm_put(&holds, &e->key, (void *)(uintptr_t)1); n_retire_freed++; destroy_tex(e); continue; } retiring[k++] = e; } nretiring = k; } // A lookup of key k that found no entry: 1 when the pool refresh asks for a held key (it gets the stand-in). Any other // lookup ends the hold: the game uses the texture. static int held_out(const TexKey *k) { if (!holds.n || !nvn_hm_get(&holds, k)) return 0; if (pool_ctx) return 1; nvn_hm_del(&holds, k); n_disuse_back++; return 0; } // tex_lock held; returns the bytes retired (freed a few submissions later), counting those still retiring as freed static uint64_t evict_disused(uint64_t want) { if (!want || !vk_pool_marks_enabled() || __atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) || marks_reads < 2 || marks_live < 2 || vk_sync_fence_done() < evict_wait) return 0; if (retiring_bytes >= want) return retiring_bytes; // what is on its way out covers it want -= retiring_bytes; Tex *v[EVICT_STEP]; int nv = 0; uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), disuse = DISUSE_FRAMES; for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP least recently sampled candidates, oldest first Tex *e = by_key.vals[i]; if (!e || e->released || e->retiring || !e->bytes || !comes_back_intact(e) || now - e->samp_frame < disuse) continue; int at = nv; while (at > 0 && v[at - 1]->samp_frame > e->samp_frame) at--; if (at == EVICT_STEP) continue; if (nv < EVICT_STEP) nv++; memmove(v + at + 1, v + at, sizeof *v * (size_t)(nv - 1 - at)); v[at] = e; } if (!nv) return 0; if (holds.n + (uint32_t)nv > HOLDS_MAX) { // keys of textures the game has let go of pile up: keep those a slot still holds out (cheld). Dropping them all // would make the pool refresh create every evicted texture again while memory is short. nvn_hm_clear(&holds, NULL); for (int id = 0; id < ntslots; id++) { if (!tslots[id].used || !tslots[id].cheld) continue; TexKey k; make_key(&tslots[id].t, &k); nvn_hm_put(&holds, &k, (void *)(uintptr_t)1); } rt_log("vk: holds of textures evicted by disuse past %d: kept the %u that registered slots still name", HOLDS_MAX, holds.n); } uint64_t freed = 0; int n = 0; uint64_t takes = vk_pool_takes(); for (; n < nv && freed < want; n++) { // retire: the pool refresh gives their slots the stand-in from now on Tex *e = v[n]; if (nretiring == cretiring) { int c = cretiring ? cretiring * 2 : 256; Tex **r = realloc(retiring, sizeof *r * (size_t)c); if (!r) break; retiring = r, cretiring = c; } retiring[nretiring++] = e; e->retiring = 1, e->retire_takes = takes, e->retire_due = 0; retiring_bytes += e->bytes; freed += e->bytes; tex_gen_bump(e->key.addr); } n_disuse += (uint64_t)n, n_disuse_bytes += freed; evict_wait = vk_sync_issued() + 2; if (disuse_logs < 8 || ((rt_trace & RT_TRACE_PERF) && disuse_logs < 64)) { disuse_logs++; rt_log("vk: %d textures no shader sampled for %d s or more evicted (%.1f MiB), least recently sampled first; their " "images go once the texture pool sets no longer name them, and one sampled again is made again within a " "few frames", n, DISUSE_SECONDS, freed / 1048576.0); } return retiring_bytes; } void vk_pool_marks_read(const uint32_t *marks, uint32_t n) { lock(); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (n > (uint32_t)ntslots) n = (uint32_t)ntslots; uint32_t marked = 0; for (uint32_t id = 0; id < n; id++) { if (!marks[id] || !tslots[id].used) continue; marked++; TexSlot *s = &tslots[id]; if (s->ce && s->cgen == agen[sinfo[id] & 0xffff]) { // what the slot resolves to now if (s->ce->retiring) retire_cancel(s->ce); // not expected: retiring bumps the slot's generation s->ce->samp_frame = now; continue; } TexKey k; make_key(&s->t, &k); Tex *e = nvn_hm_get(&by_key, &k); if (e) { if (e->retiring) retire_cancel(e); // retired, and its stand-in was sampled: back as it is e->samp_frame = now; } else if (holds.n && nvn_hm_del(&holds, &k)) { // the stand-in was sampled: the next pool refresh makes it n_disuse_back++; s->cheld = 0; s->cwant = 1; // on screen as a stand-in: made at the next refresh, whatever the frame's creation time tex_gen_bump(k.addr); } } marks_reads++; marks_peak -= marks_peak >> 14; if ((uint64_t)marked << 8 > marks_peak) marks_peak = (uint64_t)marked << 8; if (marked >= 16 && ((uint64_t)marked << 10) >= marks_peak) marks_live += marks_live < 1000; else marks_live = 0; unlock(); } // NX2NAVI_TEX_COMMIT_EVICT (default on; 0 = off): while the commit is low (commit_is_low), idle live textures go as // under VRAM pressure (evict_idle), COMMIT_EVICT_STEP at a time, once the released entries gave what they could: they // are down to COMMIT_LOW_KEEP, or drop_stale found less than a full step to drop. The driver charges device memory to // the commit too, and the eviction above starts only at NX2NAVI_TEX_VRAM_PRESSURE of the device's budget: on a 16-32 GB // card the textures of every area visited stayed until the commit ran out (27-37 GB processes, then a failed // allocation or a device lost, #177 #203 #215), as nothing else gave them back while they stayed below that share of // the video memory. Waiting for the released entries to be down to COMMIT_LOW_KEEP, as before, never ended where the // game releases and takes back 200-300 textures a frame: 2,048 released entries held 1.4-2.0 GiB, none of them stale, // and with the commit low for minutes no live texture went (a 42 GB process). The entries the game takes back stay // either way: drop_stale leaves them, and the evictions below take live textures only. #define COMMIT_EVICT_STEP ((uint64_t)256 << 20) static int commit_evict_on = -1; static void commit_evict(void) { // tex_lock held, the commit is low and the released entries gave what they could if (commit_evict_on < 0) commit_evict_on = rt_knob("TEX_COMMIT_EVICT", "1") == 1; if (!commit_evict_on || !tex_evict) return; if (rel_bytes > COMMIT_LOW_KEEP) { static int logs; if (!n_commit_past++ && logs < 4) { logs++; rt_log("vk: commit low: the %d released textures keep %.0f MiB the game still takes back or looks up; " "live textures nothing sampled go without waiting for them", nreleased, rel_bytes / 1048576.0); } } uint64_t used = 0, budget = 0; vk_mem_vram_live(&used, &budget); uint64_t freed = evict_idle(COMMIT_EVICT_STEP, used, budget); if (freed < COMMIT_EVICT_STEP) evict_disused(COMMIT_EVICT_STEP - freed); } static void sweep_pressure(void) { // tex_lock held if (vram_pressure < 0) { const char *s = cfg_get("NX2NAVI_TEX_VRAM_PRESSURE"); vram_pressure = s && *s ? atoi(s) : VRAM_PRESSURE_DEFAULT; if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = VRAM_PRESSURE_DEFAULT; tex_evict = rt_knob("TEX_EVICT", "1"); rt_log("vk: textures under VRAM pressure: %s%d%%%s", vram_pressure ? "dropped above " : "off", vram_pressure, !vram_pressure ? "" : tex_evict ? ", released ones first, then idle live ones (NX2NAVI_TEX_EVICT=0: released only)" : ", released ones only (NX2NAVI_TEX_EVICT=0)"); if (vram_pressure > 87) rt_log("vk: NX2NAVI_TEX_VRAM_PRESSURE=%d is above the seven eighths of the budget past which a new VRAM " "mirror is refused: textures make room only once a mirror was refused", vram_pressure); } retire_sweep(); if (submit_epoch & 7) return; // the budget and commit queries every 8 submits uint64_t used, budget; int exact, dropped = 0; uint64_t wanted = tex_evict ? (uint64_t)plat_xchg64(&room_wanted, 0) : 0; int known = vram_pressure && (rel_head || tex_evict || wanted) && (tex_evict ? vk_mem_vram_live(&used, &budget) : vk_mem_vram(&used, &budget, &exact)); uint64_t over = 0; if (known) { uint64_t limit = budget / 100 * (uint64_t)vram_pressure; if (used > limit) over = used - limit + budget / 50; if (wanted) { // the use at which the refused mirror fits; the lowest of the requests that stand uint64_t level = used > wanted ? used - wanted : 0; if (!room_checks || level < room_level) room_level = level; room_checks = ROOM_CHECKS; } if (room_checks && used <= room_level) room_checks = 0; // made: the pool gets its mirror when it asks again if (room_checks) { room_checks--; if (used - room_level > over) over = used - room_level; } } if (over) { uint64_t freed = 0; for (Tex *e = rel_head, *next; e && dropped < PRESSURE_STEP; e = next) { next = e->rel_next; if (!comes_back_intact(e)) continue; freed += e->bytes; destroy_tex(e); dropped++, n_pressure_drops++; } if (tex_evict && freed < over) freed += evict_idle(over - freed, used, budget); if (tex_evict && freed < over) evict_disused(over - freed); } int low = commit_is_low(), more = 0; // more: the released entries have stale ones left for the next check if (rel_head) { drop_overwritten(); uint64_t cap = released_cap_bytes(); if (rel_bytes > cap) drop_stale(cap, &n_cap_drops); if (rel_bytes > COMMIT_LOW_KEEP && low) more = drop_stale(COMMIT_LOW_KEEP, &n_commit_drops) == PRESSURE_STEP; } if (low && !more) commit_evict(); } // ---------------------------------------------------------------- per-frame texture work (vk.h VK_TS_*) // Sums for the frame being made, by the render and replay threads (atomics: both may add at once). vk_texstat_frame, // called at each present (perf.c ft_frame), takes them; a frame whose texture work (creation, upload and upload flush) // reaches NX2NAVI_TEXUP_LOG_MS gets a "texup:" line, so an area change's hitch shows where its time went (default 0: // no lines; 16 is a useful value). static volatile int64_t ts_ns[VK_TS_N], ts_n[VK_TS_N], ts_bytes[VK_TS_N]; static int ts_log_ms = -1; static int ts_thread(void) { uint32_t tid = plat_thread_id(); return tid == prof_render_tid || tid == prof_replay_tid; } void vk_texstat_time(int kind, uint64_t t0) { if (!t0 || !ts_thread()) return; plat_add64(&ts_ns[kind], (int64_t)(kern_now_ns() - t0)); plat_inc64(&ts_n[kind]); } void vk_texstat_add(int kind, uint64_t n, uint64_t bytes) { if (!ts_thread()) return; plat_add64(&ts_n[kind], (int64_t)n); plat_add64(&ts_bytes[kind], (int64_t)bytes); } void vk_texstat_frame(uint64_t frame, uint64_t d) { if (ts_log_ms < 0) { ts_log_ms = cfg_int("NX2NAVI_TEXUP_LOG_MS"); if (ts_log_ms < 0) ts_log_ms = 0; } int64_t ns[VK_TS_N], n[VK_TS_N], b[VK_TS_N]; for (int k = 0; k < VK_TS_N; k++) ns[k] = plat_xchg64(&ts_ns[k], 0), n[k] = plat_xchg64(&ts_n[k], 0), b[k] = plat_xchg64(&ts_bytes[k], 0); double work = (ns[VK_TS_CREATE] + ns[VK_TS_UPLOAD] + ns[VK_TS_FLUSH]) / 1e6; static int32_t n_lines; if (!ts_log_ms || work < ts_log_ms || ++n_lines > 400) return; rt_log("texup: frame %llu: %.1f ms frame, texture work %.1f ms; created %lld in %.1f ms; uploads %lld (%.1f MiB) in " "%.1f ms: staging %.1f ms (%lld new buffers, %.1f MiB), storage copy %.1f ms, ASTC record %.1f ms (%lld " "scratch buffers made in %.1f ms); upload flush %lld in %.1f ms; new device memory of 32 MiB or more %lld " "(%.0f MiB)", (unsigned long long)frame, d / 1e6, work, (long long)n[VK_TS_CREATE], ns[VK_TS_CREATE] / 1e6, (long long)n[VK_TS_UPLOAD], b[VK_TS_UPLOAD] / 1048576.0, ns[VK_TS_UPLOAD] / 1e6, ns[VK_TS_STAGE] / 1e6, (long long)n[VK_TS_STG_NEW], b[VK_TS_STG_NEW] / 1048576.0, ns[VK_TS_COPY] / 1e6, ns[VK_TS_ASTC] / 1e6, (long long)n[VK_TS_SCRATCH], ns[VK_TS_SCRATCH] / 1e6, (long long)n[VK_TS_FLUSH], ns[VK_TS_FLUSH] / 1e6, (long long)n[VK_TS_DEVMEM], b[VK_TS_DEVMEM] / 1048576.0); } // ---------------------------------------------------------------- uploads static uint32_t texel_alpha_one(int format, uint32_t *elem) { switch (format) { case 0x19: *elem = 1; return 0xFF; case 0x1A: *elem = 1; return 0x7F; case 0x1B: case 0x1C: *elem = 1; return 1; case 0x1D: *elem = 2; return 0x3C00; case 0x1E: *elem = 2; return 0xFFFF; case 0x1F: *elem = 2; return 0x7FFF; case 0x20: case 0x21: *elem = 2; return 1; case 0x22: *elem = 4; return 0x3F800000; default: *elem = 4; return 1; // 0x23, 0x24 } } // 3-component texels to 4 (n texels). static void expand_rgb(int format, const uint8_t *src, uint8_t *dst, size_t n) { uint32_t e, one = texel_alpha_one(format, &e); for (size_t i = 0; i < n; i++, src += 3 * e, dst += 4 * e) { memcpy(dst, src, 3 * e); memcpy(dst + 3 * e, &one, e); } } static void shrink_rgb(int format, const uint8_t *src, uint8_t *dst, size_t n) { uint32_t e; texel_alpha_one(format, &e); for (size_t i = 0; i < n; i++, src += 4 * e, dst += 3 * e) memcpy(dst, src, 3 * e); } // NX2NAVI_ASTC=check: the compute decoder's texels of n regions (gpu: the regions one after the other) against // astc_decode_ldr of the same raw blocks (raw: the staging bytes the regions point into). static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar, int n, const uint8_t *gpu) { static volatile int32_t n_bad; static uint64_t n_tex, n_texels, n_diff_tex; uint32_t bw = e->f.astc_bw, bh = e->f.astc_bh; uint64_t diff = 0, first = UINT64_MAX; uint32_t want = 0, got = 0; int first_k = 0; for (int k = 0; k < n; k++) { const VkAstcRegion *r = &ar[k]; size_t sb = (size_t)((r->w + bw - 1) / bw) * ((r->h + bh - 1) / bh) * 16, st = (size_t)r->w * r->h * 4; uint8_t *cpu = malloc(st); for (uint32_t z = 0; z < r->slices; z++, gpu += st) { astc_decode_ldr(raw + r->src + z * sb, r->w, r->h, bw, bh, e->f.astc_srgb, cpu); if (!memcmp(cpu, gpu, st)) continue; for (size_t i = 0; i < st / 4; i++) if (memcmp(cpu + 4 * i, gpu + 4 * i, 4)) { if (first == UINT64_MAX) { first = i, first_k = k; memcpy(&want, cpu + 4 * i, 4), memcpy(&got, gpu + 4 * i, 4); } diff++; } } free(cpu); n_texels += (uint64_t)r->w * r->h * r->slices; } n_tex++; if (diff) { n_diff_tex++; log_limited(&n_bad, "ASTC check: texture %#llx fmt %#x %ux%u: %llu texels differ, first region %d texel %llu " "cpu %08x gpu %08x", (unsigned long long)e->pub.addr, e->t.d.format, bw, bh, (unsigned long long)diff, first_k, (unsigned long long)first, want, got); } if (n_tex == 1 || !(n_tex & 63) || diff) rt_log("vk: ASTC check: %llu textures, %llu texels compared, %llu textures differ", (unsigned long long)n_tex, (unsigned long long)n_texels, (unsigned long long)n_diff_tex); } // Staging bytes of one part of a texture upload decoded or copied on the CPU (see upload). With NX2NAVI_ASTC=cpu a // large ASTC array decodes to RGBA8 in one go: a staging buffer of 676682336 bytes failed (-2) on an Intel UHD with // 8 GB of shared memory and the game closed before the title screen. #define UPLOAD_PART (32ull << 20) // fits the staging pool (vk_staging takes up to half of its 64 MiB block) // A texture of at most CHASH_MAX bytes keeps a hash of its storage as it was uploaded. A write to its pages then // re-uploads it only when its own bytes changed: Ocarina's frames rewrite hundreds of small textures' pages (shared // with data the CPU writes, or written with the same bytes), about 420 re-uploads a frame of 1.5 KiB each, and one // costs a staging buffer, a copy and a barrier, several microseconds, where hashing 1.5 KiB takes a fraction of one. // A CPU write during the hash sets the pages' write bits again, so the next check sees it. enum { CHASH_MAX = 64 << 10 }; static uint64_t content_hash(const void *p, uint64_t n) { const uint8_t *b = p; uint64_t h = 0x9e3779b97f4a7c15ull ^ n, i = 0; for (; i + 8 <= n; i += 8) { uint64_t w; memcpy(&w, b + i, 8); h = (h ^ w) * 0xbf58476d1ce4e5b9ull; h ^= h >> 29; } for (; i < n; i++) h = (h ^ b[i]) * 0x94d049bb133111ebull; return h ^ (h >> 31); } // ASTC for the GPU decoder without the CPU deswizzle: the storage goes into staging as the game laid it out, in one // copy, and the decoder reads each block from its block-linear place (VkAstcRegion.bl, astc_decode.comp bl_block). // The CPU deswizzle moved every 16 bytes through the GOB arithmetic and was most of an ASTC upload's CPU time when an // area loads. Fills ar (n = levels * layers, level-major as upload's regions) and returns 1 when every level and layer // qualifies: block-linear, starting 16-byte aligned, its layout inside the storage. NX2NAVI_ASTC_GPU_DESWIZZLE=0 keeps // the CPU deswizzle. static int astc_bl_regions(const Tex *e, uint32_t levels, uint32_t layers, int bw, int bh, VkAstcRegion *ar) { static int on = -1; if (on < 0) { on = cfg_bool("NX2NAVI_ASTC_GPU_DESWIZZLE"); if (!on) rt_log("vk: ASTC blocks deswizzled on the CPU (NX2NAVI_ASTC_GPU_DESWIZZLE=0)"); } const nvn_texdesc *d = &e->t.d; if (!on || nvn_tex_is_linear(d)) return 0; uint32_t k = 0; for (uint32_t lv = 0; lv < levels; lv++) for (uint32_t ly = 0; ly < layers; ly++) { uint32_t row_bytes, rows, slices, bpb; uint64_t base; unsigned th, td; if (nvn_deswizzle_geom(d, lv, ly, &row_bytes, &rows, &slices, &base, &th, &td, &bpb) || (base & 15) || th > 5 || td > 5) return 0; ar[k] = (VkAstcRegion){.src = base, .w = mip(d->width, lv), .h = mip(d->height, lv), .slices = slices, .level = lv, .layer = ly, .bl = 1, .th = (uint8_t)th, .td = (uint8_t)td}; if (base + vk_astc_region_bytes(&ar[k], (uint32_t)bw, (uint32_t)bh) > e->t.L.size) return 0; k++; } return 1; } // NX2NAVI_ASTC_ARENA=1 (default 0 until a play test measured it): the raw ASTC blocks for the GPU decoder stage in a // range of the staging arena (vk_staging_at) like the other formats, not in a VkBuffer of their own. vk_astc_record // reads its source only through a buffer copy at each region's src, so the range's offset is added to the regions. A // buffer per texture cost a vmaCreateBuffer and, after the batch, a vmaDestroyBuffer each; an area load uploads // thousands. The check mode keeps its own buffer (astc_check reads the blocks at the regions' src offsets). 0: a buffer // per texture. static int astc_arena(void) { static int on = -1; if (on < 0) { on = cfg_bool("NX2NAVI_ASTC_ARENA"); rt_log("vk: ASTC blocks for the GPU decoder stage in %s (NX2NAVI_ASTC_ARENA=%d)", on ? "the staging arena" : "a buffer per texture", on); } return on; } // Copies the whole storage into the image (every level and layer the image has). static void upload(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; const tex_t *t = &e->t; const nvn_texdesc *d = &t->d; int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); if (!bpb || !t->mem) return; // A stale slot (the bindless pools look every slot up): no upload; a later write to mapped storage uploads it // (make_fresh_snap). if (!mem_readable(e->pub.addr, e->t.L.size)) { static volatile int32_t n_unmapped; log_limited(&n_unmapped, "texture %#llx (%llu bytes): storage not mapped; not uploaded", (unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size); e->uploaded = 1; return; } e->chash_ok = e->t.L.size <= CHASH_MAX; // the hash of what this upload copies (make_fresh_snap compares it) if (e->chash_ok) e->chash = content_hash((const void *)(uintptr_t)e->pub.addr, e->t.L.size); static __thread char ctx[160]; // what this thread copies, for a crash report snprintf(ctx, sizeof ctx, "texture upload %#llx (%llu bytes, fmt %#x %ux%u)", (unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size, d->format, d->width, d->height); uint64_t p0 = perf_begin(), u0 = kern_now_ns(); int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D; uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers; int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb; int agpu = astc && vk_astc_gpu(), acheck = agpu && vk_astc_mode() == VK_ASTC_CHECK; // raw blocks to the GPU uint32_t n = levels * layers, k = 0; VkBufferImageCopy *rg = calloc(n, sizeof *rg); VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL; VkDeviceSize total = 0, decoded = 0; for (uint32_t lv = 0; lv < levels; lv++) for (uint32_t ly = 0; ly < layers; ly++) { uint32_t w = mip(d->width, lv), h = one_d ? 1 : mip(d->height, lv), sl = is3d ? mip(d->depth, lv) : 1; uint64_t texels = (uint64_t)w * h * sl, raw = (uint64_t)(w + bw - 1) / bw * bpb * ((h + bh - 1) / bh) * sl; uint64_t out = agpu ? raw : astc ? texels * 4 : rgb ? texels * (uint64_t)bpb / 3 * 4 : raw; rg[k++] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {e->pub.aspect, lv, ly, 1}, .imageExtent = {w, h, sl}}; total += (out + 15) & ~15ull; decoded += texels * 4; } VkBuffer sb, rbuf = VK_NULL_HANDLE; VmaAllocation ralloc = NULL; uint8_t *rmap = NULL, *raw_copy = NULL; int checked = 0, parts = 1; // The GPU decoder reads the blocks in place (astc_bl_regions); the check mode compares against tight blocks. int gdsw = agpu && !acheck && !one_d && astc_bl_regions(e, levels, layers, bw, bh, ar); // A CPU decode or copy stages at most UPLOAD_PART bytes per staging buffer (a region larger than that alone) and // records one copy per buffer; every part stays in the same upload batch, submitted as before. The GPU decoder // takes the raw blocks of every region from one buffer. for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) { VkDeviceSize base = rg[k0].bufferOffset, size; for (k1 = k0 + 1; k1 < n && (agpu || (k1 + 1 < n ? rg[k1 + 1].bufferOffset : total) - base <= UPLOAD_PART);) k1++; size = (k1 < n ? rg[k1].bufferOffset : total) - base; if (gdsw) size = t->L.size; // the storage as it is (one part: the GPU decoder takes every region at once) if (k0) parts++; VkCommandBuffer cb = vk_upload_begin(); // The part stages in a range of the staging arena (vk_staging_at, its offset added to the regions below), or in // a buffer of its own at offset 0 when larger than an arena range; the GPU decoder takes one with // NX2NAVI_ASTC_ARENA=0 or in the check mode (astc_arena). VkDeviceSize soff = 0; uint64_t s0 = kern_now_ns(); uint8_t *stg = agpu && (acheck || !astc_arena()) ? vk_staging(size, &sb, NULL) : vk_staging_at(size, &sb, &soff); vk_texstat_time(VK_TS_STAGE, s0); uint64_t c0 = kern_now_ns(); // the storage copies and decodes of this part (VK_TS_COPY) if (acheck) rmap = vk_staging(decoded, &rbuf, &ralloc); // The part's copies out of the storage, with it held mapped (mem_read_hold): the game releases a texture's // memory on its own thread, and a decommit in the middle of a copy faulted (tex_bl_copy). Only the copies run // held, so a waiting unmap and the guest's memory calls behind it never wait for the staging block or a decode. // Storage released since the check above uploads as zeros, as the game no longer has that texture. uint8_t **tmps = calloc(k1 - k0, sizeof *tmps); int held = mem_read_hold(e->pub.addr, e->t.L.size); if (held) plat_crash_context(ctx); else { static volatile int32_t n_gone; log_limited(&n_gone, "texture %#llx (%llu bytes): storage released during the upload; uploaded as zeros", (unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size); } if (gdsw) { if (held) memcpy(stg, t->mem, size); else memset(stg, 0, size); } for (k = k0; k < k1 && !gdsw; k++) { uint32_t lv = rg[k].imageSubresource.mipLevel, ly = rg[k].imageSubresource.baseArrayLayer; uint32_t w = rg[k].imageExtent.width, h = rg[k].imageExtent.height, sl = rg[k].imageExtent.depth; uint32_t rb = (w + bw - 1) / bw * (uint32_t)bpb, rows = (h + bh - 1) / bh; size_t raw = (size_t)rb * rows * sl; rg[k].bufferOffset -= base; uint8_t *dst = stg + rg[k].bufferOffset, *tmp = (astc && !agpu) || rgb ? malloc(raw) : NULL; tmps[k - k0] = tmp; if (!held) { memset(tmp ? tmp : dst, 0, raw); } else if (nvn_deswizzle(d, lv, ly, t->mem, tmp ? tmp : dst, rb)) { log_limited(&n_bad, "texture %#llx fmt %#x target %d: level %u layer %u has no CPU layout; uploaded as zeros", (unsigned long long)e->pub.addr, d->format, d->target, lv, ly); memset(tmp ? tmp : dst, 0, raw); } } if (held) { plat_crash_context(NULL); mem_read_release(); // the part's reads of the storage are done } for (k = k0; k < k1 && !gdsw; k++) { // the decodes, from the copies (gdsw: ar already names the blocks) uint32_t lv = rg[k].imageSubresource.mipLevel, ly = rg[k].imageSubresource.baseArrayLayer; uint32_t w = rg[k].imageExtent.width, h = rg[k].imageExtent.height, sl = rg[k].imageExtent.depth; uint32_t rb = (w + bw - 1) / bw * (uint32_t)bpb, rows = (h + bh - 1) / bh; uint8_t *dst = stg + rg[k].bufferOffset, *tmp = tmps[k - k0]; if (agpu) { ar[k] = (VkAstcRegion){.src = rg[k].bufferOffset, .w = w, .h = h, .slices = sl, .level = lv, .layer = ly}; } else if (astc) { uint64_t a0 = perf_begin(); for (uint32_t z = 0; z < sl; z++) astc_decode_ldr_mt(tmp + (size_t)z * rb * rows, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, dst + (size_t)z * w * h * 4); perf_end(PERF_ASTC, a0); rg[k].bufferRowLength = w, rg[k].bufferImageHeight = h; } else if (rgb) { expand_rgb(d->format, tmp, dst, (size_t)w * h * sl); rg[k].bufferRowLength = w, rg[k].bufferImageHeight = h; } else { rg[k].bufferRowLength = rb / (uint32_t)bpb * (uint32_t)bw, rg[k].bufferImageHeight = rows * (uint32_t)bh; } free(tmp); } free(tmps); vk_texstat_time(VK_TS_COPY, c0); if (acheck) { raw_copy = malloc(size); memcpy(raw_copy, stg, size); } barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); if (agpu) { for (k = 0; k < n; k++) ar[k].src += soff; // a range of the arena (astc_arena); 0 in a buffer of its own uint64_t a0 = perf_begin(); checked = vk_astc_record(cb, sb, size, ar, (int)n, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, e->pub.image, e->pub.aspect, rbuf); perf_end(PERF_ASTC, a0); vk_texstat_time(VK_TS_ASTC, a0); if (!checked) log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was", (unsigned long long)e->pub.addr, d->format, d->width, d->height); } else { for (k = k0; k < k1; k++) rg[k].bufferOffset += soff; vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, k1 - k0, rg + k0); } barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_upload_end(); } if (parts > 1) log_limited(&n_split, "texture %#llx fmt %#x %ux%u, %u levels, %u layers: %.1f MiB staged in %d parts", (unsigned long long)e->pub.addr, d->format, d->width, d->height, levels, layers, total / 1048576.0, parts); if (acheck) { // waits for the GPU vk_upload_flush(1); if (checked) { vmaInvalidateAllocation(vk_ctx()->vma, ralloc, 0, VK_WHOLE_SIZE); astc_check(e, raw_copy, ar, (int)n, rmap); } vmaDestroyBuffer(vk_ctx()->vma, rbuf, ralloc); } free(raw_copy); free(ar); free(rg); uint64_t dt = kern_now_ns() - u0; if (e->uploaded) { n_reuploads++, perf_count(PERF_CNT_TEX_REUP); ru_n++, ru_bytes += t->L.size, ru_ns += dt; if (e->wt_n) { ru_wt_n++, ru_wt_bytes += t->L.size, ru_wt_ns += dt; if (rt_trace & RT_TRACE_PERF) rt_log("perf: frame %llu: re-upload after %u WriteTexels: texture %#llx fmt %#x %ux%u, %.1f KiB, %.3f ms", (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), e->wt_n, (unsigned long long)e->pub.addr, d->format, d->width, d->height, t->L.size / 1024.0, dt / 1e6); } } else { n_uploads++, perf_count(PERF_CNT_TEX_UP); } e->wt_n = 0; e->uploaded = 1; e->n_up++; perf_end(PERF_TEX, p0); vk_texstat_time(VK_TS_UPLOAD, u0); vk_texstat_add(VK_TS_UPLOAD, 0, t->L.size); } // Whether e's storage fills no 4 KiB page of its own, so the write tracking can only see writes to pages it shares. static int fills_no_page(const Tex *e) { return ((e->pub.addr + e->t.L.size) & ~0xfffull) <= ((e->pub.addr + 0xfff) & ~0xfffull); } // A texture that fills no page keeps a copy of its storage bytes from when a GPU write to it was recorded // (vk_texture_gpu_written), up to SNAP_MAX bytes; wb_exact (NX2NAVI_WB_EXACT, default on) enables it. enum { SNAP_MAX = 4096 }; static int wb_exact = -1; static volatile int32_t n_kept; // vk_mem_changed over the pages wholly inside e's storage (all of its pages when it fills none): a neighbour that shares // the first or last page does not count as a write to e. A texture that fills no page shares all of its pages, so // there a page write counts only when e's own bytes differ from the copy taken at the GPU write. NX2navi's Purah Pad // camera renders the ID, screen rect and depth of what is under the reticle into 4x4 targets of 512 bytes, two // slots used in turn, and the CPU reads them a frame later. One slot's rect target shares its page with data the CPU // writes every frame: without the copy its renders were never written back, the game read a zero rect every other // frame and the target's name label blinked. NX2NAVI_WB_EXACT=0 restores the page-only decision. static int storage_rewritten(const Tex *e, uint32_t since, uint32_t *now) { uint64_t lo = (e->pub.addr + 0xfff) & ~0xfffull, hi = (e->pub.addr + e->t.L.size) & ~0xfffull; if (hi <= lo) lo = e->pub.addr, hi = e->pub.addr + e->t.L.size; int ch = vk_mem_changed(lo, hi - lo, since, now); if (ch != 1 || !since || !e->snap || memcmp(e->snap, (const void *)(uintptr_t)e->pub.addr, e->t.L.size)) return ch == 1; log_limited(&n_kept, "write-back of %#llx kept: fmt %#x %ux%u, %llu bytes; a neighbour wrote its page", (unsigned long long)e->pub.addr, e->t.d.format, e->t.d.width, e->t.d.height, (unsigned long long)e->t.L.size); return 0; } // The pages of an uploaded texture show a write (or are not tracked): whether its own bytes are still those it was // uploaded with (upload, content_hash). WriteTexels since the upload always counts as a change. static int same_content(const Tex *e) { static int on = -1; // NX2NAVI_TEX_SAME=0: every write to the pages uploads again (diagnosis) if (on < 0) { const char *v = cfg_get("NX2NAVI_TEX_SAME"); on = !(v && !strcmp(v, "0")); } if (!on || !e->chash_ok || e->wt_n || e->t.L.size > CHASH_MAX) return 0; if (content_hash((const void *)(uintptr_t)e->pub.addr, e->t.L.size) != e->chash) return 0; ru_same_n++, ru_same_bytes += e->t.L.size; return 1; } // A mirrored texture is uploaded on first use and again when its storage pages carry a newer write generation. static void make_fresh(Tex *e) { if (!e->pub.mirrored || e->pub.gpu_owned) return; uint64_t size = e->t.L.size; if (!e->uploaded) { vk_mem_changed(e->pub.addr, size, 0, &e->wgen); // resets the write bits before the storage is read upload(e); return; } uint32_t now; int ch = vk_mem_changed(e->pub.addr, size, e->wgen, &now); e->wgen = now; e->last_ch = ch; if (ch && !same_content(e)) upload(e); // 1 written, -1 not tracked: both upload again, unless the bytes match } // make_fresh for draw bindings: the page check answers from this submission's write-watch snapshot (vk_memory.c // vk_mem_changed_snap) instead of a system call per texture. A write the snapshot of the texture's region missed (made // after it, during this replay) keeps its write-watch bit and is caught by the next submission's snapshot. static void make_fresh_snap(Tex *e) { if (!e->pub.mirrored || e->pub.gpu_owned) return; uint32_t now; n_checks++; int ch = vk_mem_changed_snap(e->pub.addr, e->t.L.size, e->uploaded ? e->wgen : 0, submit_epoch, &now); e->wgen = now; if (!e->uploaded) { upload(e); return; } e->last_ch = ch; if (ch && !same_content(e)) upload(e); } // ---------------------------------------------------------------- aliases (NX2NAVI_TEX_ALIAS) // Bytes per block and block size of the image's VkFormat (ASTC decodes to RGBA8, RGB expands to RGBA); 0 when the // VkFormat substitutes another one of unknown size. static int img_block(const Tex *e, int *bw, int *bh) { if (e->f.astc_bc) { *bw = *bh = 4; return 16; } if (e->f.astc_bw) { *bw = *bh = 1; return 4; } int b = nvn_fmt_bpb(e->t.d.format, bw, bh); if (e->f.expand_rgb) { *bw = *bh = 1; return b / 3 * 4; } return e->f.vk == e->f.ideal ? b : 0; } // The new entry e can take src's content texel for texel: colour, one sample, the same image type and storage layout, // and image formats of one block size (0 with a reason in *why when not). Block-compressed images need the same shape; // uncompressed ones take the region both cover (alias_copy). Ocarina moves render targets whose size follows an effect // on screen, so a descriptor made over a live target's memory is often a few texels larger or smaller than it. The game // reads the bytes there as its content (the same memory on the console); started empty instead, the scene went black // for a frame about once a second while the camera turned, from 0.6.1 on (issue #253). static int alias_compatible(const Tex *e, const Tex *src, const char **why) { const VkTexImage *p = &e->pub, *q = &src->pub; if (e->f.depth || e->f.stencil || src->f.depth || src->f.stencil || p->samples > 1 || q->samples > 1) return *why = "depth, stencil or multisample", 0; if (p->type != q->type) return *why = "another image type", 0; if (nvn_tex_is_linear(&e->t.d) != nvn_tex_is_linear(&src->t.d)) return *why = "linear and block-linear", 0; int ew, eh, sw, sh, eb = img_block(e, &ew, &eh), sb = img_block(src, &sw, &sh); if (!eb || eb != sb || ew != sw || eh != sh) return *why = "another block size", 0; if (ew == 1 && eh == 1) return 1; if (p->width != q->width || p->height != q->height || p->depth != q->depth || e->t.L.size != src->t.L.size) return *why = "compressed, another shape", 0; return 1; } // src holds content the storage bytes do not: the GPU wrote it and, for a mirrored texture, it was not written back // and the CPU has not rewritten the storage since (the new entry's upload then reads the newer bytes). static int alias_has_gpu_content(const Tex *src) { if (!src->pub.gpu_owned) return 0; if (!src->pub.mirrored) return 1; uint32_t now; return src->gpu_dirty && !storage_rewritten(src, src->gpu_gen, &now); } // Maxwell reads the same bytes through both descriptors: src's content goes into the new image e at the replay's // current position, before any later command of the submission uses e. Through a buffer, since bound images share // their memory and image copies may not overlap; e leaves UNDEFINED between the two halves, so its first transition // cannot clobber src's bytes before they are read. 0 when no replay is recording (e is then not initialised yet). static int alias_copy(Tex *e, const Tex *src) { int reopen; VkCommandBuffer cb = vk_exec_inline_begin(&reopen); if (!cb) return 0; int bw, bh, bpb = img_block(e, &bw, &bh); uint32_t levels = e->pub.levels < src->pub.levels ? e->pub.levels : src->pub.levels; uint32_t layers = e->pub.layers < src->pub.layers ? e->pub.layers : src->pub.layers, n = levels * layers, k = 0; // the region both images cover (alias_compatible: the same shape for block-compressed ones); the rest of a larger // new image starts as init_image would leave it int partial = src->pub.width < e->pub.width || src->pub.height < e->pub.height || src->pub.depth < e->pub.depth || levels < e->pub.levels || layers < e->pub.layers; VkBufferImageCopy *rg = calloc(n, sizeof *rg); if (!rg) rt_fatal("vk: out of memory for an alias copy"); VkDeviceSize total = 0; for (uint32_t lv = 0; lv < levels; lv++) for (uint32_t ly = 0; ly < layers; ly++, k++) { uint32_t w = umin(mip(e->pub.width, lv), mip(src->pub.width, lv)); uint32_t h = umin(mip(e->pub.height, lv), mip(src->pub.height, lv)); uint32_t z = umin(mip(e->pub.depth, lv), mip(src->pub.depth, lv)); rg[k] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, lv, ly, 1}, .imageExtent = {w, h, z}}; total += ((VkDeviceSize)(w + bw - 1) / bw * ((h + bh - 1) / bh) * z * bpb + 15) & ~15ull; } VkBuffer buf = vk_exec_inline_buffer(total); if (!buf) { free(rg); vk_exec_inline_end(reopen); return 0; } barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); vkCmdCopyImageToBuffer(cb, src->pub.image, VK_IMAGE_LAYOUT_GENERAL, buf, n, rg); VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT}; VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = e->pub.image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; vkCmdPipelineBarrier2(cb, &di); if (partial && vk_fix_zero() && !e->bound_to && !e->pub.mirrored && !format_compressed(e->f.vk)) { VkClearColorValue zero = {0}; VkImageSubresourceRange all = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; vkCmdClearColorImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &all); barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); } vkCmdCopyBufferToImage(cb, buf, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, n, rg); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_exec_inline_end(reopen); free(rg); return 1; } // Descriptors whose image creation failed (VRAM exhausted, an unsupported format): not tried again for 2 s, so a // draw that names one every frame does not repeat a failing allocation on every use. Texture lock held. #define FAILED_HOLD 32 static struct { TexKey k; uint64_t until_ns; } failed_hold[FAILED_HOLD]; static unsigned failed_next; static int create_held(const TexKey *k) { uint64_t now = plat_now_ns(); for (int i = 0; i < FAILED_HOLD; i++) if (failed_hold[i].until_ns > now && !memcmp(&failed_hold[i].k, k, sizeof *k)) return 1; return 0; } static void create_failed(const TexKey *k) { unsigned i = failed_next++ % FAILED_HOLD; failed_hold[i].k = *k, failed_hold[i].until_ns = plat_now_ns() + 2000000000ull; } // The start of e's content at its address, for a new entry and for a superseded one taken back (taken_back): another // descriptor's released entry there is superseded, and a live one holding GPU-written content is the alias source, // preferring the one that differs only in swizzle (same_but_swizzle). A superseded entry is kept for its own // descriptor, which takes it back here: Ocarina alternates descriptors at some addresses every frame (a 2560x1440 // target and a 480x270 one in Kokiri Forest), and destroying the released one made both images again each frame. It // starts over as a new entry would (alias copy or init_image) when a texture at its address holds GPU content written // since its release (newer), and else keeps its own: the memory still holds what it held then. Ocarina gives its scene // picture (r11g11b10f) a second descriptor at the same address that nothing draws into (rgb9e5, read raw by the post // pass), and the picture, taken back about once a second and restarted empty, went black for a frame (issue #253). // Released mirrored entries are still destroyed: their content is the storage's, uploaded when an entry is made. // A released entry that holds GPU content gives it to e when no live texture there holds any (rsrc): its memory on // the console still holds what it drew, and a new descriptor made for that picture after the old one was let go read // it there (a black frame with FSR 3.1 and its frame generation, two presents long). static void start_at(Tex *e, const TexKey *k, int taken_back) { int alias = 0; Tex *src = NULL, *rsrc = NULL; const char *why = "no live texture with GPU content"; // for the log of an alias left empty const Tex *other = NULL; int newer = 0; // taken back: a texture at this address holds GPU content written since e's release int live_content = 0; // a live texture at this address holds GPU content for (Tex *p = nvn_hm_get(&by_addr, &k->addr), *nx; p; p = nx) { nx = p->next_at; if (p == e) continue; if (taken_back && p->use_frame >= e->rel_frame && alias_has_gpu_content(p)) newer = 1; if (p->released) { if (p->pub.mirrored) { destroy_tex(p); continue; } p->superseded = 1; const char *unused; if (tex_alias && alias_has_gpu_content(p) && alias_compatible(e, p, &unused) && (!rsrc || p->use_frame > rsrc->use_frame)) rsrc = p; continue; } alias = 1; other = p; if (alias_has_gpu_content(p)) live_content = 1; if (!tex_alias || (src && !(same_but_swizzle(&p->key, k) && !same_but_swizzle(&src->key, k)))) continue; if (!alias_compatible(e, p, &why)) continue; if (!alias_has_gpu_content(p)) { why = "the live texture holds no GPU content"; continue; } src = p; } int from_released = !src && rsrc && !live_content; if (from_released) src = rsrc; if (alias) perf_count(PERF_CNT_TEX_ALIAS); if (taken_back && !newer && !e->pub.mirrored) { // its image still holds its content n_alias_kept++; return; } if (!tex_alias) { // create_tex initialised a new entry if (taken_back) init_new(e); return; } if (src && alias_copy(e, src)) { n_alias_copied++; if (from_released) n_alias_released++; if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen); e->uploaded = 1; e->pub.gpu_owned = 1; } e->chash_ok = 0; } else { init_new(e); if (alias) { if (e->pub.mirrored) n_alias_upload++; else { n_alias_uninit++; static int logged; // what an empty start over live memory was, to tell a black frame's cause; depth buffers start empty // over another target's memory at every area load, harmlessly, and are left out int depth = e->f.depth || e->f.stencil || other->f.depth || other->f.stencil; if (logged < 24 && other && !depth) { logged++; rt_log("vk: texture %#llx (%ux%u fmt %#x%s) starts empty over a live one (%ux%u fmt %#x): %s; frame %llu", (unsigned long long)k->addr, e->pub.width, e->pub.height, e->t.d.format, taken_back ? ", taken back" : "", other->pub.width, other->pub.height, other->t.d.format, src ? "no replay recording to copy in" : why, (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED)); } } } } } static Tex *entry_of(const tex_t *t) { TexKey k; make_key(t, &k); Tex *e = nvn_hm_get(&by_key, &k); if (e && e->retiring) { // retired by disuse: the pool refresh leaves it out, any other lookup takes it back if (pool_ctx) return NULL; retire_cancel(e); } if (!e) { if (held_out(&k) || create_held(&k)) return NULL; uint64_t c0 = kern_now_ns(); // the entry's creation (VK_TS_CREATE): image, memory, first transition if (tex_alias < 0) { tex_alias = cfg_bool("NX2NAVI_TEX_ALIAS"); rt_log("vk: texture aliases: %s", tex_alias ? "a new image takes the live image's content" : "a new image starts empty"); } if (!(e = create_tex(t, &k, tex_alias))) return create_failed(&k), NULL; perf_count(PERF_CNT_TEX_NEW); start_at(e, &k, 0); e->next_at = nvn_hm_get(&by_addr, &k.addr); nvn_hm_put(&by_addr, &k.addr, e); nvn_hm_put(&by_key, &k, e); n_site[1]++; tex_gen_bump(k.addr); vk_texstat_time(VK_TS_CREATE, c0); e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // new: not disused before it is drawn } if (e->released) { e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one uint64_t gap = e->samp_frame - e->rel_frame; if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); e->overwritten = 0; if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work e->superseded = 0, e->reused = 1; n_restarts++; start_at(e, &k, 1); } } e->t = *t; touch(e); return e; } // vk_dump: upload bookkeeping of an image vk_texture_get / vk_texture_by_id returned. pages_written: whether its // storage pages were written since the last check (1, 0, -1 not tracked); the query leaves that check's state alone. void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written) { lock(); const Tex *e = (const Tex *)img; // pub is the first member uint32_t now; *uploads = e->n_up; *last_change = e->last_ch; *pages_written = e->pub.mirrored && e->uploaded ? vk_mem_changed(e->pub.addr, e->t.L.size, e->wgen, &now) : 0; unlock(); } const VkTexImage *vk_texture_get(const tex_t *t) { if (!t || !t->mem) return NULL; lock(); Tex *e = entry_of(t); if (e) make_fresh(e); unlock(); return e ? &e->pub : NULL; } // Render targets the CPU reads without NVN's invalidate (vk_texture_write_back_small); small_pending: one may be dirty. // wb_limit: targets up to this size (bytes) always go back when the CPU waits for a fence, NX2NAVI_WB_FENCE_LIMIT, read // once. A larger one goes back unless the GPU wrote it in each of the last WB_RUN frames (wb_wanted). wb_trace: // NX2NAVI_TRACE=gfx, read together with it. enum { WB_SMALL_DEFAULT = 64 << 10, WB_RUN = 16 }; static int small_pending; static uint64_t wb_limit = WB_SMALL_DEFAULT; static int wb_trace; static int wb_cfg_read; static void wb_cfg_init(void) { if (wb_cfg_read) return; int v = cfg_int("NX2NAVI_WB_FENCE_LIMIT"); wb_limit = v < 0 ? 0 : (uint64_t)v; wb_trace = (rt_trace & RT_TRACE_GFX) != 0; wb_cfg_read = 1; if (wb_limit != WB_SMALL_DEFAULT) rt_log("vk: fence write-back limit %llu bytes", (unsigned long long)wb_limit); } // Whether the fence write-back takes a dirty target. A target the GPU draws every frame is scratch the CPU does not // read: in Hyrule Field three 256x256 RGBA8 targets and a 256x256 RG8 one, which cost 2.7 ms a frame to write back. // NX2navi's 260x260 RG8 grass maps, which its fire spread reads after a fence wait (#132), are drawn once per change, or // up to 9 frames in a row while a tile loads. static int wb_wanted(const Tex *e) { return e->t.L.size <= wb_limit || e->run < WB_RUN; } static void mark_released(Tex *e) { // tex_lock held; e is not released e->released = 1; e->rel_epoch = submit_epoch; e->rel_serial = reg_serial; e->rel_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); nreleased++; rel_append(e); n_site[3]++; tex_gen_flip(e, 0); } // e, in the GPU-only address window, is written by the GPU in frame fr: the entries its storage runs over are // overwritten (section "Moved render targets", above gwin). static void overwrite_others(const Tex *e, uint64_t fr) { uint64_t lo = e->key.addr, hi = lo + e->t.L.size; int released = 0; for (int i = gwin_lower(lo > gwin_span ? lo - gwin_span : 0); i < ngwin && gwin[i]->key.addr < hi; i++) { Tex *q = gwin[i]; uint64_t qlo = q->key.addr, qhi = qlo + q->t.L.size; if (q->overwritten || qlo == lo || qhi <= lo || (lo >= qlo && hi <= qhi)) continue; if (fr - q->gw_frame < OVERWRITE_FRAMES) continue; q->overwritten = 1, q->ow_frame = fr; n_overwritten++; if (!q->released) mark_released(q), released = 1; } if (released) sweep_released(); } void vk_texture_gpu_written(const tex_t *t) { if (!t || !t->mem) return; vk_exec_frame_written(t); lock(); Tex *e = entry_of(t); if (e) { make_fresh(e); // a partial render keeps what the storage held before uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); e->gw_frame = now; if (e->scan_frame != now + 1 && nvn_gpu_only(e->key.addr)) { // once per frame and entry e->scan_frame = now + 1; overwrite_others(e, now); } e->pub.gpu_owned = 1; e->chash_ok = 0; // the image now holds the GPU's content: matching storage bytes no longer mean "unchanged" uint64_t f = vk_exec_recording_fence(); if (f > e->wr_fence) e->wr_fence = f; if (e->pub.mirrored) { e->gpu_dirty = 1; storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render if (wb_exact < 0) wb_exact = cfg_bool("NX2NAVI_WB_EXACT"); if (wb_exact && !e->snap && e->t.L.size <= SNAP_MAX && fills_no_page(e) && !(e->snap = malloc(SNAP_MAX))) rt_fatal("vk: out of memory for a texture's storage copy"); if (e->snap) memcpy(e->snap, (const void *)(uintptr_t)e->pub.addr, e->t.L.size); e->gpu_seq = ++gpu_writes; e->gpu_fence = vk_exec_recording_fence(); uint64_t fr = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (fr != e->run_frame || !e->run) { e->run = fr == e->run_frame + 1 && e->run ? (uint8_t)(e->run < WB_RUN ? e->run + 1 : WB_RUN) : 1; e->run_frame = fr; } wb_cfg_init(); if (wb_wanted(e)) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE); own_add(e); } } unlock(); } void vk_texture_release(const tex_t *t) { if (!t || !t->mem) return; TexKey k; make_key(t, &k); lock(); Tex *e = nvn_hm_get(&by_key, &k); if (e && !e->released) { mark_released(e); sweep_released(); } unlock(); } void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc) { lock(); Tex **list = malloc(sizeof *list * (size_t)(by_key.n + 1)); int n = 0; for (uint32_t i = 0; i < by_key.cap; i++) { Tex *e = by_key.vals[i]; if (e && ((e->key.addr >= lo && e->key.addr - lo < size) || (alloc && e->bound_to == alloc))) list[n++] = e; } for (int i = 0; i < n; i++) destroy_tex(list[i]); free(list); unlock(); } // ---------------------------------------------------------------- descriptor slots void vk_texture_register(int id, const tex_t *t, const view_t *v) { if (id < 0 || id >= (1 << 20)) rt_fatal("vk: texture slot %d out of range", id); lock(); if (id >= ntslots) { int n = ntslots ? ntslots : 1024; while (n <= id) n *= 2; TexSlot *s = realloc(tslots, sizeof *s * (size_t)n); if (!s) rt_fatal("vk: out of memory for texture slots"); memset(s + ntslots, 0, sizeof *s * (size_t)(n - ntslots)); tslots = s, ntslots = n; } if (!sinfo) { uint64_t *p = calloc(1 << 20, sizeof *p); slot_next = malloc(sizeof *slot_next << 20), slot_prev = malloc(sizeof *slot_prev << 20); slot_bucket = calloc(1 << 20, sizeof *slot_bucket); if (!p || !slot_next || !slot_prev || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens"); for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1; for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1; __atomic_store_n(&sinfo, p, __ATOMIC_RELEASE); } TexSlot *s = &tslots[id]; // The same texture and view registered again (Ocarina rewrites much of its descriptor pool every frame): the slot // keeps its serial and resolved entry, so the texture pool refresh (vk_exec_draw.c pool_sync_entries) does not // resolve it again. Only while that entry is live: after a release, a registration is what makes a slot current // again (slot_resolve's stale rule), so it goes the full way. A slot held out of the pool (cheld) keeps that too: // the same descriptor names the same held texture. Both only while the resolution is current (cgen): a destroyed // entry bumped the generation and its record may be freed, so ce is not read past that. int current = s->used && s->cgen == agen[sinfo[id] & 0xffff]; if (current && ((s->ce && !s->ce->released) || s->cheld) && !memcmp(&s->t, t, sizeof *t) && s->has_view == (v != NULL) && (!v || !memcmp(&s->v, v, sizeof *v))) { n_same_regs++; unlock(); return; } s->t = *t; if (!s->used && id < VK_POOL_TEX) pool_slots++; s->used = 1; s->ce = NULL; s->cheld = 0; slots_gen++; if (!++reg_serial) reg_serial = 1; __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); slot_rebucket(id, agen_idx((uintptr_t)t->mem)); dirty_push((uint32_t)id); s->has_view = v != NULL; if (v) s->v = *v; else memset(&s->v, 0, sizeof s->v); s->khash = 0; if (!tex_mirrored(t)) { // FNV-1a over the key (make_key zeroes its padding) and the view TexKey k; make_key(t, &k); uint64_t h = 0xcbf29ce484222325ull; const uint8_t *b = (const uint8_t *)&k; for (size_t i = 0; i < sizeof k; i++) h = (h ^ b[i]) * 0x100000001b3ull; b = (const uint8_t *)&s->v; for (size_t i = 0; i < sizeof s->v; i++) h = (h ^ b[i]) * 0x100000001b3ull; s->khash = (h ^ s->has_view) | 1; } unlock(); } // The descriptor hash of a GPU-written slot: two slots with equal hashes, the same address generation and no entry // created, destroyed, released or taken back between their lookups (vk_texture_entry_gen) resolve alike. 0 for an // unregistered or CPU-written slot (its resolution depends on when it was registered: slot_resolve's stale rule). uint64_t vk_texture_slot_khash_locked(int id) { return id >= 0 && id < ntslots && tslots[id].used ? tslots[id].khash : 0; } uint32_t vk_texture_entry_gen(void) { return tex_gen; } // The entry slot s resolves to, and the view and has_view a caller gets with it (the slot's, with the descriptor's // swizzle when the entry is another descriptor at the same storage). NULL when the slot names no texture. // Entry creation budget of slot lookups (vk_texture_create_budget_locked): -1 none. The bindless pools look up every // registered slot, and most of a pool is stale descriptors (storage freed or reused); with a budget a lookup creates at // most that many entries, only for mapped storage, and notes the slots it left for later (create_deferred). static int create_budget = -1, create_deferred; // A slot whose stand-in a shader sampled is on screen as that stand-in: its texture is made by the next lookup even // with the budget at 0 (the frame's creation time spent, vk_exec_draw.c pool_create_ns), WANT_PER_FRAME of them a // present. Left to the budget, it stayed dark for as long as the refresh had no time over, about a second in a town. #define WANT_PER_FRAME 32 static int want_take(void) { static uint64_t frame = ~0ull; static int left; uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (f != frame) frame = f, left = WANT_PER_FRAME; return left > 0 ? left-- : 0; } void vk_texture_create_budget_locked(int n) { create_budget = n, create_deferred = 0; } int vk_texture_create_deferred_locked(void) { int d = create_deferred; create_deferred = 0; return d; } static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { *view = s->v; *has_view = s->has_view; TexKey k; make_key(&s->t, &k); Tex *e = nvn_hm_get(&by_key, &k); int stale = 0; if (!e || e->released) { // a finalized slot follows the live texture at its storage // Prefer the live entry that differs from the slot descriptor only in its swizzle: transient render targets // share storage, and the newest one at the address is usually another size. The map screen samples its // 2250x1080 map target through an (R,G,B,1) descriptor and followed a 960x60 target at the same address. Tex *own = e, *any = NULL; e = NULL; // A slot whose content the CPU writes (tex_mirrored) follows another image only for content its own bytes do // not hold, the GPU's: Ocarina rasterizes each UI caption on the CPU into a small r8 glyph atlas and reuses the // memory of a freed one for the next, of another width. Following the older 256x39 atlas at the address, the // replay uploaded the new 128-wide bytes with a 256-texel row, and the Basic Controls screen's ZL caption // ("Target") sampled a mix of its glyphs and stale ones. Without such an image the slot gets its own below // (entry_of), uploaded from its storage with its own geometry. int cpu_slot = tex_mirrored(&s->t); for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) { if (p->released) continue; // Only an entry of the slot's target: Ocarina's volumetric fog registers a 3D two-channel volume (32x24x64) // over the storage of the bloom pass's 480x270 2D target; followed, the fog computes and the fog pass got // that 2D image, no 3D view can be made of it (vk_exec_draw.c get_view) and they read and wrote stand-ins. // With the target kept they get the volume itself. if (p->key.d.target != k.d.target) continue; if (same_but_swizzle(&p->key, &k)) { e = p; break; } if (!any && (!cpu_slot || alias_has_gpu_content(p))) any = p; } // own is missing or released here; for a CPU slot entry_of below finds or revives it by key. A CPU slot whose // own entry was released after the slot was registered is stale: the game finalized that texture and is // freeing its storage, and reviving it uploaded from that storage while another thread unmapped it (an // access violation in tex_bl_copy from the pool refresh, Kokiri Forest). It resolves to nothing; a slot // registered again after the release is live and revives the entry. if (!e && any) e = any; else if (!e && !cpu_slot) e = own && own->superseded ? NULL : own; // a superseded one: entry_of takes it back else if (!e && own && (uint32_t)(sinfo[s - tslots] >> 16) <= own->rel_serial) stale = 1; } if (!e && s->t.mem && !stale) { // evicted by disuse: the pool refresh keeps the stand-in, without spending the creation budget on it if (pool_ctx && holds.n && nvn_hm_get(&holds, &k)) resolve_held = 1; else if (create_budget < 0) e = entry_of(&s->t); else if (!create_budget && !(s->cwant && want_take())) create_deferred = 1; else if (mem_readable((uint64_t)(uintptr_t)s->t.mem, s->t.L.size)) { e = entry_of(&s->t); if (create_budget > 0) create_budget--; } if (e) s->cwant = 0; } if (e && e->retiring) { // retired by disuse: the pool refresh gets the stand-in, any other lookup takes it back if (pool_ctx) e = NULL, resolve_held = 1; else retire_cancel(e); } // A followed texture is another descriptor at the same storage: its image holds the content, but sampling takes the // slot descriptor's swizzle. The UI renders text into RGBA8 and samples it through a (A,A,A,A) descriptor; with the // render target's RGBA swizzle the text blur saw R = 1 everywhere and drew solid boxes behind tips and labels. if (e && !(s->has_view && s->v.set_swizzle)) { const nvn_texdesc *d = &s->t.d; uint8_t sw[4] = {(uint8_t)d->swizzle_r, (uint8_t)d->swizzle_g, (uint8_t)d->swizzle_b, (uint8_t)d->swizzle_a}; if (!(sw[0] | sw[1] | sw[2] | sw[3])) // descriptors built without nvnTextureBuilderSetDefaults sw[0] = NVN_TEXTURE_SWIZZLE_R, sw[1] = NVN_TEXTURE_SWIZZLE_G, sw[2] = NVN_TEXTURE_SWIZZLE_B, sw[3] = NVN_TEXTURE_SWIZZLE_A; if (memcmp(sw, e->pub.swz, 4)) { if (!s->has_view) memset(view, 0, sizeof *view); memcpy(view->swz, sw, 4); view->set_swizzle = 1; *has_view = 1; } } return e; } void vk_texture_lock(void) { lock(); } void vk_texture_unlock(void) { unlock(); } uint32_t vk_texture_slots_gen(void) { return slots_gen; } uint64_t vk_texture_same_regs(void) { return __atomic_load_n(&n_same_regs, __ATOMIC_RELAXED); } // Diagnosis for the texture pool log: address-generation bumps since the last call, the slots of the fullest buckets // and the bumps those buckets took. Texture lock held by none: takes it. void vk_texture_bucket_stats(char *out, size_t n) { lock(); static uint32_t occ[AGEN_N]; memset(occ, 0, sizeof occ); for (int i = 0; i < ntslots && sinfo; i++) if (tslots[i].used) occ[sinfo[i] & 0xffff]++; int top[4] = {-1, -1, -1, -1}; for (int b = 0; b < AGEN_N; b++) for (int k = 0; k < 4; k++) if (top[k] < 0 || occ[b] > occ[top[k]]) { for (int j = 3; j > k; j--) top[j] = top[j - 1]; top[k] = b; break; } uint64_t hot = 0; // bumps weighted by the slots they invalidate for (int b = 0; b < AGEN_N; b++) hot += (uint64_t)bump_count[b] * occ[b]; snprintf(out, n, "%llu new generations (destroy %llu, create %llu, take-back %llu, release %llu; %llu releases and " "take-backs went back to a generation their state had), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u", (unsigned long long)n_bumps, (unsigned long long)n_site[0], (unsigned long long)n_site[1], (unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone, (unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]], bump_count[top[0]], bump_count[top[1]], bump_count[top[2]], bump_count[top[3]]); n_bumps = n_undone = 0; memset(n_site, 0, sizeof n_site); memset(bump_count, 0, sizeof bump_count); unlock(); } uint64_t vk_texture_slot_token(int id) { const uint64_t *si = __atomic_load_n(&sinfo, __ATOMIC_ACQUIRE); if (!si || id < 0 || id >= (1 << 20)) return 0; uint64_t v = __atomic_load_n(&si[id], __ATOMIC_ACQUIRE); return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0; } uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); } int vk_texture_slot_count_locked(void) { return ntslots; } int vk_sampler_slot_count_locked(void) { return nsslots; } const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { lock(); const VkTexImage *r = vk_texture_by_id_locked(id, view, has_view); unlock(); return r; } const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { if (id < 0 || id >= ntslots || !tslots[id].used) return NULL; TexSlot *s = &tslots[id]; uint32_t g = agen[sinfo[id] & 0xffff]; // resolved again: an entry at its address was created, dropped or released, or the slot registered (s->ce // cleared); an empty result is not kept, as image creation may succeed later, except a held one for the pool // refresh (cheld): the pool walks every slot each submission, and the hold ends with a generation bump if ((!s->ce || s->cgen != g) && !(s->cheld && s->cgen == g && pool_ctx)) { view_t v; int hv; resolve_held = 0; in_lookup = 1; s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it in_lookup = 0; s->cview = v, s->chas_view = (uint8_t)hv; s->cgen = g; s->cepoch = 0; s->cheld = (uint8_t)(!s->ce && resolve_held); } Tex *e = s->ce; // a draw binding: the texture is in use, whether or not the pool walk checked it first in this submission if (e && !pool_ctx) e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a // replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not // mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A // held slot counts as checked for look_round: the textures evicted by disuse are registered slots all the same. if ((e || s->cheld) && s->cepoch != submit_epoch) { round_checks++; if (e && (!e->uploaded || e->checked != submit_epoch)) { make_fresh_snap(e); e->checked = submit_epoch; touch(e); } s->cepoch = submit_epoch; } if (view) *view = has_view ? s->cview : s->v; if (has_view) *has_view = view ? s->chas_view : s->has_view; return e ? &e->pub : NULL; } // vk_dump: the swizzle selectors of the descriptor registered in slot id; 0 when the slot is empty. int vk_texture_slot_swz(int id, uint8_t swz[4]) { lock(); int ok = id >= 0 && id < ntslots && tslots[id].used; if (ok) { const nvn_texdesc *d = &tslots[id].t.d; swz[0] = (uint8_t)d->swizzle_r, swz[1] = (uint8_t)d->swizzle_g, swz[2] = (uint8_t)d->swizzle_b; swz[3] = (uint8_t)d->swizzle_a; } unlock(); return ok; } uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); } void vk_texture_new_submit(void) { lock(); if (round_checks && (uint64_t)round_checks * 2 >= pool_slots) look_round++; // the last one walked the pool round_checks = 0; if (!++submit_epoch) submit_epoch = 1; rate_tick(); sweep_pressure(); static uint64_t log_at; uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) / 60; if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) { log_at = f; rt_log("perf: frame %llu: texture aliases: content copied %llu (%llu from a released texture), filled from storage " "%llu, left uninitialised %llu; taken back with their own content %llu", (unsigned long long)f * 60, (unsigned long long)n_alias_copied, (unsigned long long)n_alias_released, (unsigned long long)n_alias_upload, (unsigned long long)n_alias_uninit, (unsigned long long)n_alias_kept); n_alias_copied = n_alias_upload = n_alias_uninit = n_alias_kept = n_alias_released = 0; rt_log("perf: frame %llu: texture re-uploads %llu (%.1f MiB, %.2f ms), after WriteTexels %llu (%.1f MiB, %.2f ms); " "skipped with unchanged bytes %llu (%.1f MiB)", (unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6, (unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6, (unsigned long long)ru_same_n, ru_same_bytes / 1048576.0); ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = ru_same_n = ru_same_bytes = 0; uint64_t ck, calls, untr; vk_mem_snap_stats(&ck, &calls, &untr); rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu", (unsigned long long)f * 60, (unsigned long long)ck, (unsigned long long)n_checks, (unsigned long long)calls, (unsigned long long)untr); n_checks = 0; rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, " "under VRAM pressure %llu, under commit pressure %llu (%llu checks went on to live textures with the " "released ones still above their bound); idle live textures evicted %llu (%.1f MiB); " "unsampled ones evicted %llu (%.1f MiB), retiring %d (%.1f MiB), freed %llu, taken back while retiring " "%llu, back %llu, held %u (marks read %u); images reused %llu, made %llu, pool bindings not tried %llu, " "superseded taken back %llu; render targets overwritten at another address %llu, dropped unused %llu", (unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops, (unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_commit_past, (unsigned long long)n_evict, n_evict_bytes / 1048576.0, (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0, nretiring, retiring_bytes / 1048576.0, (unsigned long long)n_retire_freed, (unsigned long long)n_retire_cancel, (unsigned long long)n_disuse_back, holds.n, marks_reads, (unsigned long long)n_reused, (unsigned long long)n_reuse_new, (unsigned long long)n_bind_skips, (unsigned long long)n_restarts, (unsigned long long)n_overwritten, (unsigned long long)n_over_drops); n_cap_drops = n_pressure_drops = n_commit_drops = n_commit_past = n_evict = n_evict_bytes = 0; n_disuse = n_disuse_bytes = n_disuse_back = n_retire_freed = n_retire_cancel = 0; n_reused = n_reuse_new = n_bind_skips = n_restarts = n_overwritten = n_over_drops = 0; } unlock(); } // ---------------------------------------------------------------- texel reads and writes typedef struct { unsigned level, layer0; int nz, is3d, bw, bh, bpb; size_t wb, hb, row, img, x0, y0; } Region; static int region_of(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img, Region *g) { g->bpb = nvn_fmt_bpb(t->d.format, &g->bw, &g->bh); if (!g->bpb || r->width <= 0 || r->height <= 0 || r->depth < 0 || r->xoffset < 0 || r->yoffset < 0 || r->zoffset < 0 || r->xoffset % g->bw || r->yoffset % g->bh) return 0; g->level = v && v->set_levels ? v->base_level : 0; g->layer0 = v && v->set_layers ? v->min_layer : 0; g->nz = r->depth ? r->depth : 1; g->is3d = t->d.target == NVN_TEXTURE_TARGET_3D; g->wb = (size_t)((r->width + g->bw - 1) / g->bw) * (size_t)g->bpb; g->hb = (size_t)(r->height + g->bh - 1) / (size_t)g->bh; g->row = row ? row : g->wb; g->img = img ? img : g->row * g->hb; g->x0 = (size_t)(r->xoffset / g->bw) * (size_t)g->bpb; g->y0 = (size_t)(r->yoffset / g->bh); return 1; } // Pool bytes <-> caller rows of the region alone (nvn_texels_box), not through a // linear copy of the whole level: for a small region of a large texture that cost tens of milliseconds while the guest // thread held its virtual core and the caller the texture lock (monsters2 replay, frames 765-790). static int storage_texels(const tex_t *t, const Region *g, const NVNcopyRegion *r, uint8_t *mem, int write) { for (int z = 0; z < g->nz; z++) { unsigned layer = g->is3d ? 0 : g->layer0 + (unsigned)r->zoffset + (unsigned)z; uint32_t slice = g->is3d ? (uint32_t)(r->zoffset + z) : 0; if (nvn_texels_box(&t->d, g->level, layer, (uint32_t)g->x0, (uint32_t)g->y0, slice, (uint32_t)g->wb, (uint32_t)g->hb, t->mem, mem + (size_t)z * g->img, g->row, !write)) return 0; } return 1; } static int read_aspect(const Tex *e, VkImageAspectFlags *a) { if (!e->f.depth && !e->f.stencil) { *a = VK_IMAGE_ASPECT_COLOR_BIT; return 1; } switch (e->t.d.format) { // packed depth-stencil storage has no single-aspect copy case 0x31: *a = VK_IMAGE_ASPECT_STENCIL_BIT; return 1; case 0x32: case 0x34: *a = VK_IMAGE_ASPECT_DEPTH_BIT; return 1; default: return 0; } } // An image region read through a staging buffer. texels_read_record records the copy into the open upload batch; // texels_read_take copies the rows out once that batch is done (vk_upload_flush(1)) and frees the buffer. Several // records before one flush cost one GPU round trip instead of one each. typedef struct { VkBuffer sb; VmaAllocation sa; const uint8_t *stg; size_t slice, ib; // ib: bytes per texel of an expanded RGB image, 0 when the image holds the storage layout uint32_t w; } TexRead; static int texels_read_record(Tex *e, const Region *g, const NVNcopyRegion *r, TexRead *t) { static volatile int32_t n_astc, n_ds; VkImageAspectFlags aspect; if (e->f.astc_bw) { log_limited(&n_astc, "ReadTexels of ASTC texture %#llx: the image holds decoded RGBA8; zeros returned", (unsigned long long)e->pub.addr); return 0; } if (!read_aspect(e, &aspect)) { log_limited(&n_ds, "texels of packed depth-stencil format %#x are not copied", e->t.d.format); return 0; } if (g->level >= e->pub.levels) return 0; uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height; t->w = w; t->ib = e->f.expand_rgb ? (size_t)g->bpb / 3 * 4 : 0; t->slice = t->ib ? (size_t)w * h * t->ib : g->wb * g->hb; t->stg = vk_staging(t->slice * (size_t)g->nz, &t->sb, &t->sa); VkBufferImageCopy *rg = calloc((size_t)g->nz, sizeof *rg); if (!rg) rt_fatal("vk: out of memory for a texture read"); for (int z = 0; z < g->nz; z++) rg[z] = (VkBufferImageCopy){.bufferOffset = t->slice * (size_t)z, .imageSubresource = {aspect, g->level, g->is3d ? 0 : g->layer0 + (uint32_t)r->zoffset + (uint32_t)z, 1}, .imageOffset = {r->xoffset, r->yoffset, g->is3d ? r->zoffset + z : 0}, .imageExtent = {w, h, 1}}; VkCommandBuffer cb = vk_upload_begin(); barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); vkCmdCopyImageToBuffer(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, t->sb, (uint32_t)g->nz, rg); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT, VK_ACCESS_2_HOST_READ_BIT); vk_upload_end(); free(rg); return 1; } static void texels_read_take(const Tex *e, const Region *g, const TexRead *t, uint8_t *mem) { VkCtx *c = vk_ctx(); vmaInvalidateAllocation(c->vma, t->sa, 0, VK_WHOLE_SIZE); for (int z = 0; z < g->nz; z++) { const uint8_t *s = t->stg + t->slice * (size_t)z; uint8_t *m = mem + (size_t)z * g->img; for (size_t y = 0; y < g->hb; y++) { if (t->ib) shrink_rgb(e->t.d.format, s + y * t->w * t->ib, m + y * g->row, t->w); else memcpy(m + y * g->row, s + y * g->wb, g->wb); } } vmaDestroyBuffer(c->vma, t->sb, t->sa); } // Image <-> caller rows through a staging buffer; the GPU copy is waited for. static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t *mem, int write) { static volatile int32_t n_astc, n_ds; VkCtx *c = vk_ctx(); VkImageAspectFlags aspect; if (!write) { TexRead t; if (!texels_read_record(e, g, r, &t)) return 0; vk_upload_flush(1); texels_read_take(e, g, &t, mem); return 1; } if (!read_aspect(e, &aspect)) { log_limited(&n_ds, "texels of packed depth-stencil format %#x are not copied", e->t.d.format); return 0; } if (g->level >= e->pub.levels) return 0; uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height; int rgb = e->f.expand_rgb, astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu(); size_t texels = (size_t)w * h, ib = agpu ? 0 : astc ? 4 : rgb ? (size_t)g->bpb / 3 * 4 : 0; size_t slice = ib ? texels * ib : g->wb * g->hb; // agpu: the raw blocks, decoded by vk_astc_record VkBuffer sb; VmaAllocation sa; uint8_t *stg = vk_staging(slice * (size_t)g->nz, &sb, &sa); VkBufferImageCopy *rg = calloc((size_t)g->nz, sizeof *rg); for (int z = 0; z < g->nz; z++) { uint8_t *s = stg + slice * (size_t)z; const uint8_t *m = mem + (size_t)z * g->img; if (agpu) { for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb); } else if (astc) { uint8_t *tight = malloc(g->wb * g->hb); for (size_t y = 0; y < g->hb; y++) memcpy(tight + y * g->wb, m + y * g->row, g->wb); uint64_t a0 = perf_begin(); astc_decode_ldr_mt(tight, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, s); perf_end(PERF_ASTC, a0); free(tight); } else if (rgb) { for (size_t y = 0; y < g->hb; y++) expand_rgb(e->t.d.format, m + y * g->row, s + y * w * ib, w); } else { for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb); } rg[z] = (VkBufferImageCopy){.bufferOffset = slice * (size_t)z, .imageSubresource = {aspect, g->level, g->is3d ? 0 : g->layer0 + (uint32_t)r->zoffset + (uint32_t)z, 1}, .imageOffset = {r->xoffset, r->yoffset, g->is3d ? r->zoffset + z : 0}, .imageExtent = {w, h, 1}}; } VkCommandBuffer cb = vk_upload_begin(); vmaFlushAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); int done = 0; // A BC3 image takes whole 4x4 blocks, or a part that ends on the mip's edge; a region of a 5x5 or 6x6 footprint // may end inside a block (vk_astc_record checks the start corner). uint32_t mw = e->t.d.width >> g->level, mh = e->t.d.height >> g->level; int bc_cut = e->f.astc_bc && (((w & 3) && (uint32_t)r->xoffset + w != (mw ? mw : 1)) || ((h & 3) && (uint32_t)r->yoffset + h != (mh ? mh : 1))); if (agpu && !bc_cut) { // bc_cut: nothing recorded, logged below as not decoded VkAstcRegion *ar = calloc((size_t)g->nz, sizeof *ar); for (int z = 0; z < g->nz; z++) ar[z] = (VkAstcRegion){.src = slice * (size_t)z, .w = w, .h = h, .slices = 1, .level = rg[z].imageSubresource.mipLevel, .layer = rg[z].imageSubresource.baseArrayLayer, .offset = rg[z].imageOffset}; uint64_t a0 = perf_begin(); done = vk_astc_record(cb, sb, slice * (size_t)g->nz, ar, g->nz, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, e->pub.image, aspect, VK_NULL_HANDLE); perf_end(PERF_ASTC, a0); free(ar); } if (!agpu) vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)g->nz, rg); else if (!done) log_limited(&n_astc, "WriteTexels of ASTC texture %#llx not decoded on the GPU: left as it was", (unsigned long long)e->pub.addr); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_upload_end(); vk_upload_flush(1); free(rg); vmaDestroyBuffer(c->vma, sb, sa); return 1; } static void zero_rows(const Region *g, uint8_t *dst) { for (int z = 0; z < g->nz; z++) for (size_t y = 0; y < g->hb; y++) memset(dst + (size_t)z * g->img + y * g->row, 0, g->wb); } int vk_texture_read_image(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) { Region g; if (!t || !region_of(t, v, r, row, img, &g)) return 0; lock(); Tex *e = entry_of(t); int ok = 0; if (e) { make_fresh(e); ok = image_texels(e, &g, r, dst, 0); } unlock(); if (!ok) zero_rows(&g, dst); return ok; } // ReadTexels: the pool bytes are the content of a mirrored texture the GPU has not rendered into; every other texture // is read back from its image. void vk_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) { static volatile int32_t n_bad; Region g; if (!region_of(t, v, r, row, img, &g)) { log_limited(&n_bad, "ReadTexels: invalid region %d,%d,%d %dx%dx%d (format %#x)", r->xoffset, r->yoffset, r->zoffset, r->width, r->height, r->depth, t->d.format); return; } lock(); Tex *e = entry_of(t); int ok = 0; if (e && e->pub.mirrored && !e->pub.gpu_owned) ok = storage_texels(t, &g, r, dst, 0); else if (e) { make_fresh(e); ok = image_texels(e, &g, r, dst, 0); } unlock(); if (!ok) zero_rows(&g, dst); } // WriteTexels: a mirrored texture gets the texels in its pool bytes (and in its image once the GPU owns it); every // other texture in its image. void vk_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row, uint64_t img) { static volatile int32_t n_bad; Region g; if (!region_of(t, v, r, row, img, &g) || (v && v->set_format && v->format != t->d.format)) { log_limited(&n_bad, "WriteTexels: invalid region %d,%d,%d %dx%dx%d or view format (format %#x)", r->xoffset, r->yoffset, r->zoffset, r->width, r->height, r->depth, t->d.format); return; } lock(); Tex *e = entry_of(t); if (e && e->pub.mirrored) { if (!storage_texels(t, &g, r, (uint8_t *)src, 1)) log_limited(&n_bad, "WriteTexels: region outside level %u of texture %#llx", g.level, (unsigned long long)e->pub.addr); vk_guest_written((uintptr_t)t->mem, t->L.size); if (e->pub.gpu_owned) image_texels(e, &g, r, (uint8_t *)src, 1); else if (e->uploaded) e->wt_n++; } else if (e) { image_texels(e, &g, r, (uint8_t *)src, 1); } unlock(); } // ---------------------------------------------------------------- CPU reads of GPU-written storage // A mirrored texture the GPU rendered into holds its texels in its image only; its storage keeps what the CPU last put // there. The game may still read that storage with the CPU: NX2navi moves its generated grass maps between pools with // memcpy once the GPU is done, and the copy then carried zeros. Such a read writes the image back first. // // Every level and layer of every texture in v is copied to staging in one upload batch with one wait: one GPU round // trip per level and layer took 0.1 ms each, and a 260x260 grass map has 9 levels. The storage is then written in the // order of v, so where two textures share storage the later one wins. typedef struct { Tex *e; Region g; NVNcopyRegion r; TexRead t; } WbPart; static void write_back_n(Tex **v, int n) { int np = 0, cap = 0; WbPart *p = NULL; for (int i = 0; i < n; i++) { Tex *e = v[i]; const nvn_texdesc *d = &e->t.d; int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D; uint32_t levels = linear ? 1 : e->pub.levels, layers = linear || is3d ? 1 : e->pub.layers; for (uint32_t lv = 0; lv < levels; lv++) for (uint32_t ly = 0; ly < layers; ly++) { if (np == cap) { cap = cap ? cap * 2 : 16; if (!(p = realloc(p, sizeof *p * (size_t)cap))) rt_fatal("vk: out of memory for a texture write-back"); } WbPart *q = &p[np]; view_t vw = {.set_levels = 1, .set_layers = 1, .base_level = (uint8_t)lv, .min_layer = (uint16_t)ly}; q->e = e; q->r = (NVNcopyRegion){.width = (int)mip(d->width, lv), .height = d->target == 0 || d->target == 3 ? 1 : (int)mip(d->height, lv), .depth = is3d ? (int)mip(d->depth, lv) : 1}; if (region_of(&e->t, &vw, &q->r, 0, 0, &q->g) && texels_read_record(e, &q->g, &q->r, &q->t)) np++; } } if (np) vk_upload_flush(1); for (int i = 0; i < np; i++) { WbPart *q = &p[i]; uint8_t *buf = malloc(q->g.img * (size_t)q->g.nz); if (!buf) rt_fatal("vk: out of memory for a texture write-back"); texels_read_take(q->e, &q->g, &q->t, buf); storage_texels(&q->e->t, &q->g, &q->r, buf, 1); free(buf); } free(p); for (int i = 0; i < n; i++) v[i]->gpu_dirty = 0; } // Mirrored textures overlapping [addr, addr+size) that the GPU wrote since their last write-back. static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) { int n = 0; for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) { Tex *e = owned[i]; if (__atomic_load_n(&e->gpu_dirty, __ATOMIC_RELAXED) && e->pub.addr + e->t.L.size > addr) { if (!hit) return 1; hit[n++] = e; } } return n; } const VkTexImage *vk_texture_at_locked(const VkTexImage *img, VkFormat format) { for (Tex *p = nvn_hm_get(&by_addr, &((Tex *)img)->key.addr); p; p = p->next_at) if (!p->released && p->pub.format == format && p->pub.width == img->width && p->pub.height == img->height) return &p->pub; return NULL; } // A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c // tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels. void vk_texture_storage_bound_locked(const VkTexImage *img) { Tex *e = (Tex *)img; // pub is the first member uint64_t f = vk_exec_recording_fence(); if (f > e->wr_fence) e->wr_fence = f; } uint64_t vk_texture_last_write(const tex_t *t) { lock(); Tex *e = entry_of(t); uint64_t f = e ? e->wr_fence : 0; unlock(); return f; } size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) { Region g; if (!region_of(t, v, r, row, img, &g)) return 0; return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb; } int vk_texture_cpu_stale(uint64_t addr, uint64_t size) { if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0; plat_mutex_lock_shared(&owned_lock); int dirty = owned_dirty(addr, size, NULL); plat_mutex_unlock_shared(&owned_lock); return dirty; } // The newest submission recorded to write a GPU-written mirrored texture overlapping [addr, addr+size); 0: none. uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size) { if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0; lock(); uint64_t f = 0; for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) { const Tex *e = owned[i]; if (!e->gpu_dirty || e->pub.addr + e->t.L.size <= addr) continue; if (e->gpu_fence > f) f = e->gpu_fence; if (e->wr_fence > f) f = e->wr_fence; } unlock(); return f; } // The image is the newest content only while nothing wrote the storage since the GPU render: NX2navi finalizes grass maps, // reuses their storage for new data and memcpys that data on, and writing the old image back first handed the copy // a stale map or stale resource bytes (wrong textures in the distance). Every hit is checked before any is written // back, since a write-back is itself a CPU write to storage another hit may share; the hits then go back oldest render // first, so where two descriptors share storage the newer render wins. void vk_texture_write_back(uint64_t addr, uint64_t size) { lock(); Tex **hit = malloc(sizeof *hit * (size_t)(nowned + 1)); if (!hit) rt_fatal("vk: out of memory for a texture write-back"); int n = owned_dirty(addr, size, hit), k = 0; // owned[] cannot change while tex_lock is held for (int i = 0; i < n; i++) { uint32_t now; if (storage_rewritten(hit[i], hit[i]->gpu_gen, &now) == 1) hit[i]->gpu_dirty = 0; else hit[k++] = hit[i]; } for (int i = 1; i < k; i++) for (int j = i; j > 0 && hit[j - 1]->gpu_seq > hit[j]->gpu_seq; j--) { Tex *x = hit[j]; hit[j] = hit[j - 1], hit[j - 1] = x; } write_back_n(hit, k); for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB); for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP); free(hit); unlock(); } // NX2navi's grass system renders a 128x128 RGBA8 map of cut and trampled grass into a CPU-cached pool every frame and its // CPU code reads the pool bytes without nvnMemoryPoolInvalidateMappedRange (cut reactions: blades, insects, the cut // itself). On the Switch the bytes are there once the frame's fence has passed. So when the // CPU waits for a fence, mirrored targets rendered by work up to that fence go back to their storage (wb_wanted: // targets over wb_limit only when not drawn every frame), unless the CPU rewrote it since. Targets drawn every frame // still go back on an invalidate, memcpy or ReadTexels. int vk_texture_small_dirty(void) { return __atomic_load_n(&small_pending, __ATOMIC_ACQUIRE); } // NX2NAVI_TRACE=gfx: a dirty target the fence write-back leaves alone because it is over the limit and drawn every frame // (what a CPU read after the wait would miss), or sends back although it is over the default limit. One line per // target address and outcome; the table is small and a full one stops logging. static void wb_note(const Tex *e, int skipped) { static uint64_t seen[2][64]; static int n_seen[2]; for (int i = 0; i < n_seen[skipped]; i++) if (seen[skipped][i] == e->pub.addr) return; if (n_seen[skipped] == 64) return; seen[skipped][n_seen[skipped]++] = e->pub.addr; const pool_t *p = e->t.pool; unsigned fl = p ? p->flags : 0; rt_log("vk: fence write-back %s %#llx %ux%u fmt %#x size %llu pool %s (limit %llu)", skipped ? "skipped-drawn-every-frame" : "over-default", (unsigned long long)e->pub.addr, e->t.d.width, e->t.d.height, e->t.d.format, (unsigned long long)e->t.L.size, !p ? "none" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_CACHED) ? "cpu-cached" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED) ? "cpu-uncached" : "other", (unsigned long long)wb_limit); } void vk_texture_write_back_small(uint64_t fence) { lock(); wb_cfg_init(); __atomic_store_n(&small_pending, 0, __ATOMIC_RELAXED); plat_mutex_lock_shared(&owned_lock); int n = 0, later = 0; for (int i = 0; i < nowned; i++) n += owned[i]->gpu_dirty && wb_wanted(owned[i]); Tex **hit = n ? malloc(sizeof *hit * (size_t)n) : NULL; if (n && !hit) rt_fatal("vk: out of memory for a texture write-back"); n = 0; for (int i = 0; i < nowned; i++) { Tex *e = owned[i]; if (!e->gpu_dirty) continue; if (!wb_wanted(e)) { if (wb_trace && e->gpu_fence <= fence) { wb_note(e, 1); perf_count(PERF_CNT_WB_SKIP); } continue; } if (e->gpu_fence > fence) later = 1; else { if (wb_trace && e->t.L.size > WB_SMALL_DEFAULT) wb_note(e, 0); hit[n++] = e; } } plat_mutex_unlock_shared(&owned_lock); if (later) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE); int k = 0; for (int i = 0; i < n; i++) { uint32_t now; if (storage_rewritten(hit[i], hit[i]->gpu_gen, &now) == 1) hit[i]->gpu_dirty = 0; else hit[k++] = hit[i]; } for (int i = 1; i < k; i++) for (int j = i; j > 0 && hit[j - 1]->gpu_seq > hit[j]->gpu_seq; j--) { Tex *x = hit[j]; hit[j] = hit[j - 1], hit[j - 1] = x; } write_back_n(hit, k); for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB); for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP); free(hit); unlock(); } // ---------------------------------------------------------------- samplers (LAS textures.c) static VkCompareOp compare_op(uint32_t f) { switch (f) { case NVN_COMPARE_FUNC_NEVER: return VK_COMPARE_OP_NEVER; case NVN_COMPARE_FUNC_LESS: return VK_COMPARE_OP_LESS; case NVN_COMPARE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL; case NVN_COMPARE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL; case NVN_COMPARE_FUNC_GREATER: return VK_COMPARE_OP_GREATER; case NVN_COMPARE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL; case NVN_COMPARE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL; default: return VK_COMPARE_OP_ALWAYS; } } static struct { int init, mirror_clamp_edge, minmax, border_no_format; float aniso; } sf; // aniso: NX2NAVI_ANISO, 0 = the game's static float aniso_setting(void) { const char *an = cfg_value("NX2NAVI_ANISO"); return strcmp(an, "game") ? (float)atoi(an) : 0; } static void aniso_log(void) { if (sf.aniso > 1) rt_log("vk: anisotropic filtering: %gx", sf.aniso); else rt_log("vk: anisotropic filtering: the game's"); } // Caller holds the lock. static void sampler_features(VkCtx *c) { if (sf.init) return; VkPhysicalDeviceCustomBorderColorFeaturesEXT cbf = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT}; VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, .pNext = c->has_custom_border_color ? &cbf : NULL}; VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12}; vkGetPhysicalDeviceFeatures2(c->phys, &f2); sf.mirror_clamp_edge = f12.samplerMirrorClampToEdge; sf.minmax = f12.samplerFilterMinmax; sf.border_no_format = c->has_custom_border_color && cbf.customBorderColors && cbf.customBorderColorWithoutFormat; sf.aniso = aniso_setting(); aniso_log(); sf.init = 1; } // NX2NAVI_ANISO raises the level for filtered, mipmapped colour samplers that repeat on U and V: the surface textures, // which tile, so ground and walls seen at a shallow angle stay sharp into the distance. Depth-compare (shadow) // samplers, nearest or unmipmapped ones (data maps, LUTs) and clamped ones keep the game's level: clamped samplers // read screen-sized buffers, whose coordinates can stretch steeply at an object's edge, where anisotropic taps reach // across it and leave a speckled fringe (the inventory's character preview). The game's own wrap modes decide: a // mirror-clamp one maps to MIRRORED_REPEAT where the device lacks mirror-clamp-to-edge (make_sampler). Caller holds the // lock, after sampler_features. static int aniso_raised(const samp_par *d) { int tiled = (d->ws == NVN_WRAP_MODE_REPEAT || d->ws == NVN_WRAP_MODE_MIRRORED_REPEAT) && (d->wt == NVN_WRAP_MODE_REPEAT || d->wt == NVN_WRAP_MODE_MIRRORED_REPEAT); return sf.aniso > d->aniso && tiled && !d->cmp_mode && d->magf == NVN_MAG_FILTER_LINEAR && (d->minf == NVN_MIN_FILTER_LINEAR_MIPMAP_NEAREST || d->minf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR); } static VkSampler make_sampler(VkCtx *c, const samp_par *d) { sampler_features(c); VkSamplerCreateInfo si = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; si.magFilter = d->magf == NVN_MAG_FILTER_LINEAR ? VK_FILTER_LINEAR : VK_FILTER_NEAREST; uint32_t mf = d->minf; si.minFilter = mf == 1 || mf == 3 || mf == 5 ? VK_FILTER_LINEAR : VK_FILTER_NEAREST; si.mipmapMode = mf == 4 || mf == 5 ? VK_SAMPLER_MIPMAP_MODE_LINEAR : VK_SAMPLER_MIPMAP_MODE_NEAREST; VkSamplerAddressMode modes[3]; const uint8_t wrap[3] = {d->ws, d->wt, d->wr}; int border = 0; for (int i = 0; i < 3; i++) { switch (wrap[i]) { case NVN_WRAP_MODE_CLAMP: case NVN_WRAP_MODE_CLAMP_TO_EDGE: modes[i] = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; break; case NVN_WRAP_MODE_MIRROR_CLAMP: case NVN_WRAP_MODE_MIRROR_CLAMP_TO_EDGE: modes[i] = sf.mirror_clamp_edge ? VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE : VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT; break; case NVN_WRAP_MODE_MIRROR_CLAMP_TO_BORDER: case NVN_WRAP_MODE_CLAMP_TO_BORDER: modes[i] = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; border = 1; break; case NVN_WRAP_MODE_MIRRORED_REPEAT: modes[i] = VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT; break; default: modes[i] = VK_SAMPLER_ADDRESS_MODE_REPEAT; } } si.addressModeU = modes[0], si.addressModeV = modes[1], si.addressModeW = modes[2]; si.minLod = d->lod_min; si.maxLod = d->lod_max > 1000 ? VK_LOD_CLAMP_NONE : d->lod_max; if (mf <= NVN_MIN_FILTER_LINEAR) si.maxLod = si.minLod + 0.25f; // not mipmapped: stay on the base level if (si.maxLod < si.minLod) si.maxLod = si.minLod; float lim = c->props.limits.maxSamplerLodBias; si.mipLodBias = fmaxf(-lim, fminf(lim, d->lod_bias)); // d->aniso holds NX2NAVI_ANISO's level where it applies (sampler_for, aniso_raised) if (d->aniso > 1) { si.anisotropyEnable = VK_TRUE; si.maxAnisotropy = fminf(d->aniso, c->props.limits.maxSamplerAnisotropy); } if (d->cmp_mode) { si.compareEnable = VK_TRUE; si.compareOp = compare_op(d->cmp_func); // A depth-compare sampler set to nearest returns 0 or 1 per tap, and the game's shaders smooth shadow edges by // a few taps spaced for its own shadow map texels. With NX2NAVI_SHADOW_SCALE those taps cover a fraction of the // area and the edges turn hard and stepped; linear filtering makes every tap a 2x2 PCF instead. if (shadow_scale() > 1) si.magFilter = si.minFilter = VK_FILTER_LINEAR; } VkSamplerCustomBorderColorCreateInfoEXT cb = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT}; VkSamplerReductionModeCreateInfo rm = {.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO}; const void **tail = &si.pNext; if (border) { const float *b = d->border; if (b[0] == 0 && b[1] == 0 && b[2] == 0 && b[3] == 0) si.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; else if (b[0] == 0 && b[1] == 0 && b[2] == 0 && b[3] == 1) si.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK; else if (b[0] == 1 && b[1] == 1 && b[2] == 1 && b[3] == 1) si.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE; else if (sf.border_no_format) { memcpy(cb.customBorderColor.float32, b, 16); si.borderColor = VK_BORDER_COLOR_FLOAT_CUSTOM_EXT; *tail = &cb; tail = &cb.pNext; } else si.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; } if (d->reduction && sf.minmax && !si.compareEnable) { rm.reductionMode = d->reduction == NVN_SAMPLER_REDUCTION_MIN ? VK_SAMPLER_REDUCTION_MODE_MIN : VK_SAMPLER_REDUCTION_MODE_MAX; *tail = &rm; } VkSampler s; VkResult r = vkCreateSampler(c->device, &si, NULL, &s); if (r != VK_SUCCESS) rt_fatal("vk: vkCreateSampler failed (%d)", (int)r); return s; } void vk_sampler_register(int id, const samp_par *s) { if (id < 0 || id >= (1 << 12)) rt_fatal("vk: sampler slot %d out of range", id); lock(); if (id >= nsslots) { int n = nsslots ? nsslots : 256; while (n <= id) n *= 2; SampSlot *p = realloc(sslots, sizeof *p * (size_t)n); if (!p) rt_fatal("vk: out of memory for sampler slots"); memset(p + nsslots, 0, sizeof *p * (size_t)(n - nsslots)); sslots = p, nsslots = n; } sslots[id].p = *s; sslots[id].s = sslots[id].sp = sslots[id].sc = VK_NULL_HANDLE; sslots[id].used = 1; samp_gen++; unlock(); } // DLSS upscaling (vk_sampler_dlss_bias): the scene's material textures, the filtered mipmapped ones that repeat, are // sampled for the output's resolution; clamped ones (post effects, LUTs, the UI) keep the game's bias. static float dlss_bias; static int dlss_biased(const samp_par *k) { int rep_s = k->ws == NVN_WRAP_MODE_REPEAT || k->ws == NVN_WRAP_MODE_MIRRORED_REPEAT; int rep_t = k->wt == NVN_WRAP_MODE_REPEAT || k->wt == NVN_WRAP_MODE_MIRRORED_REPEAT; return !k->cmp_mode && k->minf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR && (rep_s || rep_t); } void vk_sampler_dlss_bias(float b) { lock(); if (b != dlss_bias) { rt_log("vk: texture LOD bias %.2f on the scene's tiled textures (the upscaler's render scale; was %.2f)", b, dlss_bias); dlss_bias = b; for (int i = 0; i < nsslots; i++) sslots[i].s = sslots[i].sp = sslots[i].sc = VK_NULL_HANDLE; // looked up again (the key moves) samp_gen++; } unlock(); } // NX2NAVI_ANISO, read again at each present after a settings change (vk_backend.c present_image, beside the DLSS bias of // vk_dlaa_frame): the raised level is part of the sampler key (sampler_for), so on a change the slots look their // samplers up again and the descriptor caches follow samp_gen, as for vk_sampler_dlss_bias. The old level's samplers // stay until shutdown like every sampler: one set per level at most. Before the first sampler there is nothing to do: // sampler_features reads the setting then. void vk_sampler_aniso_live(void) { static unsigned seen = ~0u; unsigned v = cfg_version(); if (v == seen) return; seen = v; lock(); float a = aniso_setting(); if (sf.init && a != sf.aniso) { sf.aniso = a; aniso_log(); for (int i = 0; i < nsslots; i++) sslots[i].s = sslots[i].sp = sslots[i].sc = VK_NULL_HANDLE; // looked up again (the key moves) samp_gen++; } unlock(); } // Caller holds the lock. static VkSampler sampler_for(VkCtx *c, const samp_par *s) { sampler_features(c); samp_par k; memset(&k, 0, sizeof k); k = *s; if (dlss_bias != 0 && dlss_biased(&k)) k.lod_bias += dlss_bias; // part of the key: samplers live until shutdown if (aniso_raised(&k)) k.aniso = sf.aniso; // likewise VkSampler v = (VkSampler)nvn_hm_get(&samplers, &k); if (!v) { v = make_sampler(c, &k); nvn_hm_put(&samplers, &k, (void *)v); } return v; } VkSampler vk_sampler_for(const samp_par *s) { VkCtx *c = vk_ctx(); if (!c || !s) return VK_NULL_HANDLE; lock(); VkSampler v = sampler_for(c, s); unlock(); return v; } // Samplers live until shutdown, so a slot keeps the one its parameters map to until it is registered again. VkSampler vk_sampler_by_id_locked(int id) { VkCtx *c = vk_ctx(); if (!c) return VK_NULL_HANDLE; VkSampler v = VK_NULL_HANDLE; if (id >= 0 && id < nsslots && sslots[id].used) { SampSlot *s = &sslots[id]; if (!s->s) s->s = sampler_for(c, &s->p); v = s->s; } return v; } VkSampler vk_sampler_point_by_id_locked(int id) { VkCtx *c = vk_ctx(); if (!c || id < 0 || id >= nsslots || !sslots[id].used) return VK_NULL_HANDLE; SampSlot *s = &sslots[id]; if (!s->sp) { samp_par k = s->p; k.magf = NVN_MAG_FILTER_NEAREST; k.minf = k.minf == NVN_MIN_FILTER_LINEAR ? NVN_MIN_FILTER_NEAREST : k.minf > NVN_MIN_FILTER_LINEAR ? NVN_MIN_FILTER_NEAREST_MIPMAP_NEAREST : k.minf; k.aniso = 1; s->sp = sampler_for(c, &k); } return s->sp; } // A depth-compare read through a sampler without a compare is undefined in Vulkan. NVIDIA's driver compares anyway; AMD's // returns the raw depth, so the game's shadow tests (which sample this way) come out wrong there. Such a read gets the // slot's sampler with the compare on: the game's function when it set a valid one, LEQUAL otherwise, as NVIDIA's driver // compares. NX2NAVI_SHADOW_COMPARE=0 binds the game's sampler as it is. VkSampler vk_sampler_compare_by_id_locked(int id) { VkCtx *c = vk_ctx(); if (!c || id < 0 || id >= nsslots || !sslots[id].used) return VK_NULL_HANDLE; SampSlot *s = &sslots[id]; static int on = -1; if (on < 0) on = cfg_int("NX2NAVI_SHADOW_COMPARE") != 0; if (s->p.cmp_mode || !on) return vk_sampler_by_id_locked(id); if (!s->sc) { samp_par k = s->p; k.cmp_mode = 1; if (k.cmp_func < NVN_COMPARE_FUNC_NEVER || k.cmp_func > NVN_COMPARE_FUNC_ALWAYS) k.cmp_func = NVN_COMPARE_FUNC_LEQUAL; static int told; if (told++ < 8) rt_log("vk: a depth-compare read through sampler slot %d without a compare (function %u): compared with %u", id, (unsigned)s->p.cmp_func, (unsigned)k.cmp_func); s->sc = sampler_for(c, &k); } return s->sc; } VkSampler vk_sampler_by_id(int id) { lock(); VkSampler v = vk_sampler_by_id_locked(id); unlock(); return v; } int vk_sampler_params(int id, samp_par *out) { lock(); int ok = id >= 0 && id < nsslots && sslots[id].used; if (ok) *out = sslots[id].p; unlock(); return ok; } // ---------------------------------------------------------------- statistics and shutdown void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) { lock(); if (images) *images = by_key.n; if (uploads) *uploads = n_uploads; if (reuploads) *reuploads = n_reuploads; unlock(); } void vk_texture_census(VkTexCensus *s) { VkCtx *c = vk_ctx(); memset(s, 0, sizeof *s); lock(); for (uint32_t i = 0; i < by_key.cap; i++) { const Tex *e = by_key.vals[i]; if (!e) continue; uint64_t bytes = 0; if (e->alloc && c) { VmaAllocationInfo ai; vmaGetAllocationInfo(c->vma, e->alloc, &ai); bytes = ai.size; } if (e->released) s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes; else s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes; } unlock(); } // With NX2NAVI_TRACE=perf every 600 frames, after vk_memory.c's vram line: what the live entries' own memory holds. By // kind: decoded ASTC (as BC3 or RGBA8) and other textures whose content the CPU provides, all of which a re-upload // brings back (comes_back_intact), and the GPU-written ones (render targets, textures the GPU drew or copied into), // whose content lives only in the image. "idle" is the part no lookup reached for IDLE_ROUNDS pool walks and // STALE_FRAMES presents (touch). Then the entries that share their storage address with another live entry (each // descriptor of the same storage gets an image of its own), the formats holding the most, and the largest entries. void vk_texture_census_log(uint64_t frame) { enum { K_BC3, K_RGBA8, K_CPU, K_GPU, K_N }; static const char *const kn[K_N] = {"ASTC as BC3", "ASTC as RGBA8", "other CPU-written", "GPU-written"}; uint64_t n[K_N] = {0}, b[K_N] = {0}, in[K_N] = {0}, ib[K_N] = {0}, bound = 0, dup_n = 0, dup_b = 0; struct { int fmt; uint64_t n, b; } fm[48]; int nfm = 0; struct { uint64_t b; uint32_t w, h, d, levels; int fmt, gpu; } big[4] = {{0}}; // copied: read after unlock // the addresses whose extra entries hold the most, with their largest few entries (copied: read after unlock) struct { uint64_t addr, cnt, extra; struct { uint64_t b; uint32_t w, h, d; int fmt, gpu; } e[4]; } dup[3] = {{0}}; const double M = 1048576.0; lock(); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); for (uint32_t i = 0; i < by_key.cap; i++) { const Tex *e = by_key.vals[i]; if (!e || e->released) continue; if (!e->bytes) { bound += e->bound_to != NULL; continue; } int k = !comes_back_intact(e) ? K_GPU : e->f.astc_bw ? (e->f.astc_bc ? K_BC3 : K_RGBA8) : K_CPU; n[k]++, b[k] += e->bytes; if (idle(e, now)) in[k]++, ib[k] += e->bytes; int j = 0; while (j < nfm && fm[j].fmt != (int)e->f.vk) j++; if (j == nfm && nfm < 48) fm[nfm++] = (typeof(fm[0])){(int)e->f.vk, 0, 0}; if (j < nfm) fm[j].n++, fm[j].b += e->bytes; for (int q = 0; q < 4; q++) if (e->bytes > big[q].b) { memmove(big + q + 1, big + q, sizeof *big * (size_t)(3 - q)); big[q] = (typeof(big[0])){e->bytes, e->pub.width, e->pub.height, e->pub.layers * e->pub.depth, e->pub.levels, (int)e->f.vk, k == K_GPU}; break; } } for (uint32_t i = 0; i < by_addr.cap; i++) { // live entries with own memory at one address, beyond the largest uint64_t cnt = 0, sum = 0, most = 0; for (const Tex *p = by_addr.vals[i]; p; p = p->next_at) if (!p->released && p->bytes) cnt++, sum += p->bytes, most = p->bytes > most ? p->bytes : most; if (cnt > 1) dup_n += cnt - 1, dup_b += sum - most; for (int q = 0; cnt > 1 && q < 3; q++) { if (sum - most <= dup[q].extra) continue; memmove(dup + q + 1, dup + q, sizeof *dup * (size_t)(2 - q)); memset(&dup[q], 0, sizeof dup[q]); dup[q].cnt = cnt, dup[q].extra = sum - most; for (const Tex *p = by_addr.vals[i]; p; p = p->next_at) { if (p->released || !p->bytes) continue; dup[q].addr = p->pub.addr; for (int x = 0; x < 4; x++) if (p->bytes > dup[q].e[x].b) { memmove(dup[q].e + x + 1, dup[q].e + x, sizeof dup[q].e[0] * (size_t)(3 - x)); dup[q].e[x].b = p->bytes, dup[q].e[x].w = p->pub.width, dup[q].e[x].h = p->pub.height; dup[q].e[x].d = p->pub.layers * p->pub.depth, dup[q].e[x].fmt = (int)p->f.vk; dup[q].e[x].gpu = !comes_back_intact(p); break; } } break; } } unlock(); char line[1024]; int o = snprintf(line, sizeof line, "mem: textures frame %llu: live with own memory by kind, idle part in brackets:", (unsigned long long)frame); for (int k = 0; k < K_N; k++) o += snprintf(line + o, sizeof line - (size_t)o, " %s %llu %.0f MiB (%llu %.0f MiB);", kn[k], (unsigned long long)n[k], b[k] / M, (unsigned long long)in[k], ib[k] / M); snprintf(line + o, sizeof line - (size_t)o, " bound into pool memory %llu; sharing an address with another %llu %.0f MiB", (unsigned long long)bound, (unsigned long long)dup_n, dup_b / M); rt_log("%s", line); for (int x = 0; x < nfm; x++) // by bytes, largest first for (int y = x + 1; y < nfm; y++) if (fm[y].b > fm[x].b) { typeof(fm[0]) t = fm[x]; fm[x] = fm[y], fm[y] = t; } o = snprintf(line, sizeof line, "mem: textures frame %llu: VkFormats holding the most:", (unsigned long long)frame); for (int x = 0; x < nfm && x < 8; x++) o += snprintf(line + o, sizeof line - (size_t)o, " %d: %llu %.0f MiB;", fm[x].fmt, (unsigned long long)fm[x].n, fm[x].b / M); o += snprintf(line + o, sizeof line - (size_t)o, " largest:"); for (int q = 0; q < 4 && big[q].b; q++) o += snprintf(line + o, sizeof line - (size_t)o, " %ux%ux%u fmt %d %u levels %.0f MiB%s;", big[q].w, big[q].h, big[q].d, big[q].fmt, big[q].levels, big[q].b / M, big[q].gpu ? " (GPU-written)" : ""); rt_log("%s", line); if (!dup[0].cnt) return; // Which entries share an address: each descriptor of the same storage (another size, format or level count) gets // an image of its own, and its memory beside the others'. o = snprintf(line, sizeof line, "mem: textures frame %llu: addresses with the most extra entries:", (unsigned long long)frame); for (int q = 0; q < 3 && dup[q].cnt && o < (int)sizeof line - 200; q++) { o += snprintf(line + o, sizeof line - (size_t)o, " %#llx: %llu entries, %.0f MiB beyond the largest (", (unsigned long long)dup[q].addr, (unsigned long long)dup[q].cnt, dup[q].extra / M); for (int x = 0; x < 4 && dup[q].e[x].b && o < (int)sizeof line - 80; x++) o += snprintf(line + o, sizeof line - (size_t)o, "%s%ux%ux%u fmt %d %.1f MiB%s", x ? ", " : "", dup[q].e[x].w, dup[q].e[x].h, dup[q].e[x].d, dup[q].e[x].fmt, dup[q].e[x].b / M, dup[q].e[x].gpu ? " GPU-written" : ""); o += snprintf(line + o, sizeof line - (size_t)o, ");"); } rt_log("%s", line); } void vk_texture_shutdown(void) { VkCtx *c = vk_ctx(); lock(); Tex **list = malloc(sizeof *list * (size_t)(by_key.n + 1)); int n = 0; for (uint32_t i = 0; i < by_key.cap; i++) if (by_key.vals[i]) list[n++] = by_key.vals[i]; for (int i = 0; i < n; i++) destroy_tex(list[i]); free(list); for (uint32_t i = 0; i < samplers.cap; i++) if (samplers.vals[i] && c) vkDestroySampler(c->device, (VkSampler)samplers.vals[i], NULL); free(by_key.keys), free(by_key.vals), free(by_addr.keys), free(by_addr.vals), free(samplers.keys), free(samplers.vals); by_key = (nvn_hmap){.klen = sizeof(TexKey)}; by_addr = (nvn_hmap){.klen = 8}; for (uint32_t i = 0; i < img_reqs.cap; i++) free(img_reqs.vals[i]); free(img_reqs.keys), free(img_reqs.vals), free(reuse.keys), free(reuse.vals); // reuse: emptied by destroy_tex img_reqs = (nvn_hmap){.klen = sizeof(ImgKey)}; reuse = (nvn_hmap){.klen = sizeof(ImgKey)}; samplers = (nvn_hmap){.klen = sizeof(samp_par)}; free(tslots), free(sslots); tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0, pool_slots = 0; rel_head = rel_tail = NULL, rel_bytes = 0; if (sinfo) memset(sinfo, 0, sizeof *sinfo << 20); // no slot is registered: tokens 0 unlock(); }