// Vulkan textures of the graphics layer: a VkImage cache keyed by storage // GPU address plus descriptor, uploads through staging buffers with block-linear deswizzle and ASTC decode (vk_astc.c), // re-uploads when page write generations (vk_memory.c) show the storage written, ReadTexels / WriteTexels, the texture // and sampler descriptor slots, and VkSamplers. // // A texture that mirrors its pool memory (tex_mirrored) stops re-uploading once the GPU rendered into it (the // "GPU-owned" rule); a finalized slot follows live GPU content or samples its own image as kept, but never revives // freed CPU-backed storage. // Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md). // // Every image lives in VK_IMAGE_LAYOUT_GENERAL. Uploads are recorded into the upload batch of vk_memory.c; the replay // submits it (vk_upload_flush) before a submission that samples the textures. #include #include #include #include #include #include "cfg.h" #include "flight.h" #include "gfx.h" #include "host.h" #include "nvn_api.h" #include "nvn_hmap.h" #include "nvn_tex.h" #include "par_jobs.h" #include "tex.h" #include "vk.h" #include "vk_dlaa.h" #include "vk_exec.h" #include "vk_tex_compact.h" #include "vk_tex_detail.h" #include "vk_tex_release.h" #include "prof.h" #include "platform/platform.h" typedef struct { uint64_t addr; nvn_texdesc d; uint8_t has_layout, layout[8], pad[7]; } TexKey; // What makes two images interchangeable: the parameters create_tex makes them with (img_key_of). Image reuse and the // memory requirement cache match on it. typedef struct { uint32_t flags, type, format, w, h, d, levels, layers, samples, usage; } ImgKey; typedef struct Tex { VkTexImage pub; // first: vk_texture_get hands out &pub TexKey key; tex_t t; // the newest texture object seen with this key TexVkFormat f; VmaAllocation alloc; // own memory; NULL when bound into pool memory uint64_t bytes; // the size of alloc (0 without own memory), for the released entries' total uint64_t blk_mem, blk_size; // where alloc lies (own_memory): the device memory handle and size of its block uint8_t blk_type, blk_own; // that memory's type; blk_own: memory of its own, not a part of one of VMA's blocks VmaAllocation bound_to; // the pool memory it is bound into uint64_t wgen; // page generation of the last upload check uint32_t checked; // submit_epoch of the last check from a draw binding uint32_t use_round; // look_round when a lookup last reached it (touch) uint64_t use_frame; // gfx_counters.frames then uint64_t samp_frame; // gfx_counters.frames when it was made, a marked shader last sampled it, or a lookup // other than the texture pool refresh reached it ("disuse") uint8_t retiring; // evicted by disuse, in retiring[], not destroyed yet: the pool refresh leaves it out uint64_t retire_takes; // vk_pool_takes when it was retired uint64_t retire_frame; // gfx_counters.frames when it was retired uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it int uploaded, released; uint64_t up_retry; // gfx_counters.frames from which an upload that had no memory is tried again (upload) uint64_t uid; // names the entry in its bucket's released set (tex_gen_flip); 0 until its first release uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale) uint32_t back_gap; // most presents between a release and its taking back so far (stale_after) uint32_t n_up; // uploads of this entry (vk_texture_debug) int last_ch; // last vk_mem_changed result of make_fresh: 1 written, 0 clean, -1 not tracked struct Tex *next_at; // other entries at the same storage address uint8_t gpu_dirty; // mirrored, and the GPU wrote the image after the last write-back to the storage uint64_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes) uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one) uint64_t run_frame; // mirrored: the frame of the newest GPU write uint8_t run; // frames in a row up to run_frame with a GPU write, counted up to WB_RUN uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear, // storage binding); ReadTexels waits for it instead of draining the replay queue uint8_t listed; // in owned[] uint8_t *snap; // a texture that fills no page: its storage bytes when the GPU write was recorded uint32_t wt_n; // WriteTexels into the storage since the last upload uint64_t chash; // content_hash of the storage at its last upload (chash_wanted) uint8_t chash_ok; // chash holds it uint8_t chash_miss; // over CHASH_MAX: comparisons in a row that found other bytes (CHASH_MISSES) struct Tex *rel_prev, *rel_next; // while released: the list of released entries, oldest first ImgKey ik; // the image's creation parameters uint8_t reusable; // its image may go to a new texture once it is released (own memory, GPU-only content) uint8_t reused; // its image may still be in use from before (a released entry's, or its own restarted by // start_at): the first barrier waits for all earlier work uint8_t ru_on; // in the reuse list of ik uint8_t superseded; // released, and another descriptor's entry was made at its address since (start_at) uint32_t rel_epoch; // submit_epoch when it was released struct Tex *ru_prev, *ru_next; // the reuse list of ik, newest release first uint64_t gw_frame; // gfx_counters.frames at its creation or newest GPU write (overwrite_others) uint64_t scan_frame; // one more than the frame vk_texture_gpu_written last looked for overwritten entries uint64_t ow_frame; // gfx_counters.frames when overwritten was set uint8_t overwritten; // released, and a texture at another address was drawn over its storage (overwrite_others) uint64_t kept_frame; // drop_overwritten: the present (plus one) in which it was last left because it is used uint8_t in_gwin; // in gwin[] int pidx; // 1 + its index in pent[] (vk_texture_pool_note_locked); 0: not listed int ph; // 1 + its index in pref[] (pool_ref); 0: none // Where the entry and its content came from ("origins", below drop_note), for the lines of NX2NAVI_TRACE=perf uint8_t had; // HAD_*: what put content into the image of an entry the CPU does not write uint8_t og_start; // START_*: how its content started (start_at) uint8_t og_drop; // DROP_*: the drop of this descriptor's last entry, when the ring still knew it uint8_t og_reimg; // made with the image of a released entry (create_tex, reuse) uint8_t bc1_try; // ASTC as BC3 that its first upload makes BC1 when every block is opaque (upload) uint8_t said; // marks_check wrote its line for this entry uint8_t sfill; // split fill: the CPU does not write it, and its image is the storage's bytes: 1 as a // CPU-written one's until the GPU writes it, 2 once (from_storage) uint32_t og_backs; // times it was taken back after a release (entry_of) uint64_t og_frame; // gfx_counters.frames when it was made uint64_t og_drop_frame; // gfx_counters.frames of the drop og_drop names uint64_t og_back_frame; // gfx_counters.frames when it was last taken back VkTexImage *flat3d; // sampled as a 3D texture: its one-slice 3D copy (vk_texture_flat3d_locked), else NULL VmaAllocation flat3d_alloc; uint32_t flat3d_up, flat3d_layer; // n_up and the layer of the copy's last refresh uint64_t flat3d_wr; // wr_fence then uint64_t used_frame; // presented frame (gfx_counters.frames) when a draw binding or lookup last named it uint8_t in_rt; // in rts[] (bound into pool memory, the GPU wrote it) uint8_t compacting; // a compaction pass lists it (CP.item): destroy_tex takes it off the list uint8_t clobbered; // another image of another layout wrote over its pool memory since its own last write } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the // entry (ce), valid while the generation of its address (agen) is still cgen, the view and has_view handed out with it, // and the submit_epoch of the last freshness check through the slot (cepoch). Hot fields first: a lookup that hits // touches one or two cache lines. typedef struct { struct Tex *ce; // NULL: nothing cached (registered since, or never resolved) uint32_t cgen; // agen of the slot's address when ce was resolved uint32_t cepoch; view_t cview; uint8_t chas_view, used, has_view, finalized; // finalized survives eviction of the released Tex entry uint8_t cheld; // ce is NULL as the texture was evicted by disuse, valid while cgen is (held_out) uint8_t cwant; // a shader sampled its stand-in (vk_pool_marks_read): slot_resolve makes it at once uint8_t fresh; // registered and not resolved to a texture since (vk_texture_register): shown_take uint64_t khash; // hash of the descriptor key and view, 0 for CPU-written textures (vk_texture_slot_khash_locked) view_t v; tex_t t; } TexSlot; typedef struct { samp_par p; VkSampler s, sp, sc, sd[2]; uint8_t used; } SampSlot; // s: vk_sampler_for(&p) once looked up; sp: // its point-sampled form (vk_sampler_point_by_id_locked), // sc its comparing form (vk_sampler_compare_by_id_locked), // sd[n - 1] its form for a texture made without n // levels (vk_sampler_detail_by_id_locked) static PlatRMutex tex_lock; static PlatOnce tex_once = PLAT_ONCE_INIT; static nvn_hmap by_key = {.klen = sizeof(TexKey)}; static nvn_hmap by_addr = {.klen = 8}; static TexSlot *tslots; static int ntslots; static SampSlot *sslots; static int nsslots; static nvn_hmap samplers = {.klen = sizeof(samp_par)}; static int nreleased; static Tex *rel_head, *rel_tail; // released entries in release order (sweep_released drops from the head) static uint64_t n_uploads, n_reuploads; // Re-uploads since the last perf line (tex_lock): all, and those of textures a WriteTexels wrote since their last upload. static uint64_t ru_n, ru_bytes, ru_ns, ru_wt_n, ru_wt_bytes, ru_wt_ns; static uint64_t ru_same_n, ru_same_bytes; // re-uploads skipped: the pages were written, the texture's bytes were not // The same for textures over CHASH_MAX (NX2NAVI_TEX_SAME_BIG), since the last pool log line: comparisons that found the // bytes unchanged and their bytes, comparisons that found others, and uploads that took a hash with their bytes. static uint64_t big_same_n, big_same_bytes, big_diff_n, big_hash_n, big_hash_bytes; // NX2NAVI_TEX_ALIAS (default on; =0: a new image over live storage starts empty): a new descriptor over the storage of a // live image that the GPU wrote gets that image's content (alias_copy). Counts since the last log line. static int tex_alias = -1; static uint64_t n_alias_copied, n_alias_upload, n_alias_uninit, n_alias_kept, n_alias_released; // New depth and stencil images (not reused) init_new started at the replay's position, those of them for which an open // pass was ended, and those that went in the upload batch (no replay recording), since the last perf line. static uint64_t n_dinit_inline, n_dinit_inpass, n_dinit_upload; static uint32_t submit_epoch = 1; // bumped per queue submit: draw bindings check a texture's storage once per submit // The thresholds further down that count presents (STALE_FRAMES, GPU_STALE_FRAMES, OVERWRITTEN_KEEP, DISUSE_FRAMES) // were set at 60 presents a second. Without a limit the game presents 85 to 240 a second (a menu), and "120 presents" // was then half a second and "30 s without a sample" 7 to 20 s: under memory pressure textures went three times as // readily as meant, and the ones still on screen showed their dark stand-ins until they were made again. // tex_rate is the presents per second as a multiple of 60 in 1/256, the highest of the last RATE_N seconds so that one // slow second does not shorten the thresholds; PRESENTS(n) is n presents at 60 in today's presents. // NX2NAVI_TEX_RATE_LOW (default 1; 0: as before): the rate follows the presents below 60 a second too, down to 15 // (vk_tex_release.h trl_rate_sample). It stopped at 60, so at 21 presents a second a texture stayed out of the VRAM // pressure's reach for 5.7 s after its release and not 2, and the released ones held 2.1 GiB of a 12 GiB card that // was over its budget. #define RATE_N 8 #define RATE_SAY 16 #define RATE_SAY_UNDER 240u // the log's line: under 56 presents a second (a limit of 60 measures 59 now and then) static uint32_t tex_rate = TRL_RATE_60; #define PRESENTS(n) trl_presents((uint64_t)(n), tex_rate) // The wall clock of the presents (vk_tex_release.h TrlClock): rate_tick notes the present count about once a second, // so that the released textures' hold (drop_aged) and the take-back ages (vk_texture_reuse_log) count seconds, not // presents at whatever rate the game runs. tex_lock. // The seconds are seconds of play, not of the wall: a gap between two submissions counts at most PLAY_GAP_MAX_NS. A // window minimized for ten minutes (freeze_idle), a game stopped at a breakpoint or a long load then does not age the // released textures past their hold while nothing could take them back, and the first frames after it find the // textures the frames just before it released. #define PLAY_GAP_MAX_NS 1000000000ull static TrlClock pclock; static uint64_t pclock_t0; // kern_now_ns at the first tick static uint64_t play_ns; // seconds of play so far, in ns (rate_tick) static uint64_t play_last; // kern_now_ns at the last submission rate_tick saw static uint32_t now_sec(void) { if (!pclock_t0) return 0; uint64_t gap = kern_now_ns() - play_last; return (uint32_t)((play_ns + (gap < PLAY_GAP_MAX_NS ? gap : PLAY_GAP_MAX_NS)) / 1000000000ull); } static uint32_t secs_since(uint64_t frame) { return trl_clock_age(&pclock, frame, now_sec()); } static void rate_tick(void) { // each submission, tex_lock held static uint64_t t0, f0; static uint32_t ring[RATE_N], at; uint64_t t = kern_now_ns(), f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (play_last) play_ns += t - play_last < PLAY_GAP_MAX_NS ? t - play_last : PLAY_GAP_MAX_NS; play_last = t; if (!t0) t0 = t, f0 = f, pclock_t0 = t, trl_clock_tick(&pclock, f, 0); if (t - t0 < 1000000000ull) return; trl_clock_tick(&pclock, f, (uint32_t)(play_ns / 1000000000ull)); static int low = -1, said; if (low < 0) low = rt_knob("TEX_RATE_LOW", "1") != 0; ring[at++ % RATE_N] = trl_rate_sample(f - f0, t - t0, low); t0 = t, f0 = f; uint32_t m = trl_rate(ring, RATE_N); // one line at each crossing of RATE_SAY_UNDER, RATE_SAY times a run: the log then shows which waits the drops went by static int slow; if ((m < RATE_SAY_UNDER) != slow && said < RATE_SAY) { said++, slow = m < RATE_SAY_UNDER; if (slow) rt_log("vk: the game presents %u frames a second, fewer than 60, for %d seconds: the textures' waits count " "their seconds in these (2 s are %llu presents, not 120; NX2NAVI_TEX_RATE_LOW=0: 120)", m * 60 >> 8, RATE_N, (unsigned long long)trl_presents(120, m)); else rt_log("vk: the game presents 60 frames a second or more again: the textures' waits count 60 or more"); } tex_rate = m; } // Mirrored textures the GPU wrote into, sorted by address (vk_texture_cpu_stale, vk_texture_write_back). Changed under // tex_lock and owned_lock; read under either. static PlatMutex owned_lock = PLAT_MUTEX_INIT; static Tex **owned; static int nowned, cowned; static uint64_t max_owned; // largest storage size in owned[] static uint64_t gpu_writes; // GPU writes into mirrored textures so far: Tex.gpu_seq // Bumped whenever the entry a slot resolves to can change: an entry created or destroyed, released or taken back into // use, or a registered slot finalized (the follow reads by_key, by_addr and the entry and slot lifetime flags). static uint32_t tex_gen = 1; // Bumped with tex_gen and whenever a slot is registered: what vk_texture_by_id returns for any slot may have changed. static uint32_t slots_gen = 1; static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line // tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so // a slot's resolution stays valid while its address's generation does. NX2navi finalizes and initializes transient // textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws, // which emptied every cache keyed on the global generations. 65536 buckets, the most the 16-bit index of sinfo, // slot_bucket and the dirty ring's bucket entries holds: with 4096, about 8 slots shared each bucket, and every // transient texture made or released at one address made the texture pool refresh (vk_exec_draw.c pool_sync_entries) // refill its neighbours too, most of its 3-3.7 ms per frame in Kokiri Forest. #define AGEN_N 65536 static uint32_t agen[AGEN_N]; // Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered) // and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are // 20 bits), unlike tslots, which moves when it grows. static uint64_t *sinfo; static uint32_t reg_serial; static uint64_t n_same_regs; // vk_texture_register calls that changed nothing (vk_texture_same_regs) static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 48); } // Slots that may resolve differently since a position, for the texture pool descriptors (vk_exec_draw.c // pool_sync_entries), which otherwise looked at every registered slot after each change: Ocarina registers slots all // through a submission, and the refresh walked the whole table about twenty times per submission (3.5 ms). A ring of // changes, each a slot id (registered) or an address bucket (an entry at that agen index changed: every slot registered // over it, kept in per-bucket lists). Lock held for all of it. #define DIRTY_N (1u << 16) #define DIRTY_BUCKET 0x80000000u static uint32_t dirty[DIRTY_N]; static uint64_t ndirty; static int32_t bucket_head[AGEN_N]; static int32_t *slot_next, *slot_prev; // per slot id: its bucket list links (slot_prev -2: in no list) static uint16_t *slot_bucket; static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; } // tex_gen_bump calls, by site: destroy, create, take-back, release; and the releases and take-backs that went back to a // generation their bucket's state had before (tex_gen_flip), for vk_texture_bucket_stats static uint64_t n_bumps, n_site[4], n_undone; static uint32_t bump_count[AGEN_N]; // The highest generation each bucket ever had. A new state takes the next one after it, not the one after the current // value, so a value a bucket went back from (tex_gen_flip) is never given to another state. static uint32_t agen_top[AGEN_N]; // What a slot at an address resolves to depends on the entries there and on which of them are released // (slot_resolve). Per bucket: epoch counts every change other than a release or a take-back (an entry made, destroyed, // retired, held or taken back with more than its flag changing), sig is the released entries' uids XORed, which a // release and a take-back each flip. (bucket, epoch, sig) names the state; gs remembers the generation each state // got, so a bucket that comes back to a state gets that state's generation back. static uint32_t bepoch[AGEN_N]; static uint64_t bsig[AGEN_N]; #define GS_N 8192 static struct { uint64_t sig; uint32_t b, epoch, gen, valid; } gs[GS_N]; // A slot lookup is resolving (vk_texture_by_id_locked around slot_resolve). The lookup's callers stamp what they keep // with the generation they read before it: the slot's own cache, the pool's tokens, the descriptor, type and feedback // memos (vk_exec_draw.c). A lookup that itself changes the state (it takes a released entry back for a slot registered // since the release) leaves such a stamp naming the state it left, with a result that state would not give: the next // release of that entry is later than the slot's registration, the slot is stale then (slot_resolve), and the kept // result would hand out the released entry, whose storage the game may be unmapping. So the state such a lookup // leaves loses its remembered generation (tex_gen_flip): it gets a new one when the bucket is next in it, and the // stamp never matches again. static int in_lookup; static uint32_t gs_at(uint32_t b) { return (uint32_t)(((bsig[b] ^ ((uint64_t)b << 32 | bepoch[b])) * 0x9E3779B97F4A7C15ull) >> 51); } static void tex_gen_bump(uint64_t addr) { n_bumps++; bump_count[agen_idx(addr)]++; if (!++tex_gen) tex_gen = 1; slots_gen++; uint32_t b = agen_idx(addr); bepoch[b]++; __atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE); gs[gs_at(b)] = (typeof(gs[0])){bsig[b], b, bepoch[b], agen[b], 1}; dirty_push(DIRTY_BUCKET | b); } // NX2NAVI_TEX_GEN_UNDO (default 1; 0: every release and take-back is a new generation). Ocarina finalizes and // initializes transient textures all through a frame: in a large town about 40,000 releases and as many take-backs // per 3,600 submissions, each a bump of its address bucket, and the buckets with the most slots (1,300 registered over // one address) were the ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it. // A release or a take-back changes one entry's released flag and nothing else, so it moves the bucket between states // it has usually been in before: the textures the game keeps released and takes back for a moment, and those it // releases and takes straight back. The bucket then gets the generation that state had, and whatever was resolved in // that state counts as current again: the pool's tokens, the descriptor memo and the slots' own cached entries. A // state seen for the first time gets a new generation (agen_top), and any other change starts a new epoch, so a // generation never stands for two states. The global generations still move, so their readers look, and find the // tokens equal. plain: the take-back of an entry that was superseded or overwritten while released changes more than // the flag, and is a change like any other. // One input of slot_resolve is not in the state: the stale rule reads the slot's finalized flag, which a release sets // on the slots registered with its descriptor and a registration clears. A slot is stale at every later release once // it is at one, so what it resolves to in a released state stays right; the one that is not stale takes the entry // back, which is the lookup-made change in_lookup deals with. static void tex_gen_flip(Tex *e, int plain) { static int on = -1; static uint64_t uids; if (on < 0) on = rt_knob("TEX_GEN_UNDO", "1") == 1; if (!e->uid) { uint64_t z = ++uids * 0x9E3779B97F4A7C15ull; z = (z ^ z >> 30) * 0xBF58476D1CE4E5B9ull, z = (z ^ z >> 27) * 0x94D049BB133111EBull; e->uid = (z ^ z >> 31) | 1; } uint32_t b = agen_idx(e->key.addr); if (in_lookup) { // see in_lookup: the state this lookup leaves is not to get its generation again typeof(gs[0]) *m0 = &gs[gs_at(b)]; if (m0->valid && m0->b == b && m0->epoch == bepoch[b] && m0->sig == bsig[b]) m0->valid = 0; } bsig[b] ^= e->uid; if (!on || plain) { tex_gen_bump(e->key.addr); return; } if (!++tex_gen) tex_gen = 1; slots_gen++; typeof(gs[0]) *m = &gs[gs_at(b)]; if (m->valid && m->b == b && m->epoch == bepoch[b] && m->sig == bsig[b]) { n_undone++; __atomic_store_n(&agen[b], m->gen, __ATOMIC_RELEASE); } else { n_bumps++; bump_count[b]++; __atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE); *m = (typeof(*m)){bsig[b], b, bepoch[b], agen[b], 1}; } dirty_push(DIRTY_BUCKET | b); } // Slot groups: the GPU-written slots of one address bucket that share a descriptor hash (TexSlot.khash) resolve alike // under the same address and entry generations, as the texture pool's memo has it (vk_exec_draw.c pool_fill). Ocarina // keeps up to ~400 descriptors of each transient render target registered over its storage, and the fullest bucket, // 1,281 slots, moves 8 times a submission in Kokiri Forest: 12,800 of the 17,600 slots a submission's dirty walks // refilled held their views already. The dirty walk can hand each group to the pool whole (vk_texture_dirty_slots_locked's // gfn), which leaves its members alone when their resolution did not change (NX2NAVI_POOL_GROUP). A slot is in its // bucket's own list (khash 0: a CPU-written texture) or in its group's member list, through the same links. A slot that // joins a group goes to its tail and leaves the group's stamp as it was: its registration put it on the dirty ring, so // the walk that follows refills it on its own, and the head, whose state the pool tests, is a member from before. (The // game's 118 registrations a submission cleared the stamps of the groups they joined, and the walk at a submission's // start went member by member as before: 11,300 of 11,550 slots.) typedef struct { VkSlotGroup pub; int32_t gnext, gprev, tail; uint32_t bucket; } SlotGroup; static SlotGroup *grp; static int ngrp, cgrp, grp_free = -1, ngrp_live; static uint64_t n_grouped; // slots in groups static int32_t bucket_ghead[AGEN_N]; // per bucket: its first group (-1: none; set with the lists) static int32_t *slot_grp; // per slot id: its group, -1 none typedef struct { uint64_t kh, b; } GrpKey; static nvn_hmap grp_by = {.klen = sizeof(GrpKey)}; // -> group + 1 static int grp_get(uint64_t kh, uint32_t b) { GrpKey k = {kh, b}; void *v = nvn_hm_get(&grp_by, &k); if (v) return (int)((intptr_t)v - 1); int g; if (grp_free >= 0) { g = grp_free, grp_free = grp[g].gnext; } else { if (ngrp == cgrp) { int n = cgrp ? cgrp * 2 : 1024; SlotGroup *p = realloc(grp, sizeof *p * (size_t)n); if (!p) rt_fatal("vk: out of memory for texture slot groups"); grp = p, cgrp = n; } g = ngrp++; } SlotGroup *G = &grp[g]; memset(G, 0, sizeof *G); G->pub.kh = kh, G->pub.head = G->tail = -1, G->bucket = b; G->gprev = -1, G->gnext = bucket_ghead[b]; if (G->gnext >= 0) grp[G->gnext].gprev = g; bucket_ghead[b] = g; nvn_hm_put(&grp_by, &k, (void *)(intptr_t)(g + 1)); ngrp_live++; return g; } static void grp_drop(int g) { SlotGroup *G = &grp[g]; GrpKey k = {G->pub.kh, G->bucket}; nvn_hm_del(&grp_by, &k); if (G->gprev >= 0) grp[G->gprev].gnext = G->gnext; else bucket_ghead[G->bucket] = G->gnext; if (G->gnext >= 0) grp[G->gnext].gprev = G->gprev; G->pub.kh = 0, G->pub.head = G->tail = -1, G->pub.n = 0, G->pub.stamp = 0; G->gnext = grp_free, grp_free = g; ngrp_live--; } // Places slot id in bucket b: at the tail of its group when it has a descriptor hash, else at the head of the bucket's // own list (the lists exist once a slot was registered: vk_texture_register). static void slot_place(int id, uint32_t b, uint64_t kh) { if (slot_prev[id] != -2) { // out of its old list int32_t g = slot_grp[id]; int32_t *head = g >= 0 ? &grp[g].pub.head : &bucket_head[slot_bucket[id]]; if (slot_prev[id] >= 0) slot_next[slot_prev[id]] = slot_next[id]; else *head = slot_next[id]; if (slot_next[id] >= 0) slot_prev[slot_next[id]] = slot_prev[id]; else if (g >= 0) grp[g].tail = slot_prev[id]; slot_grp[id] = -1; if (g >= 0) { n_grouped--; if (!--grp[g].pub.n) grp_drop(g); } } slot_bucket[id] = (uint16_t)b; if (kh) { int g = grp_get(kh, b); // may move grp SlotGroup *G = &grp[g]; slot_grp[id] = g; G->pub.n++, n_grouped++; slot_next[id] = -1; slot_prev[id] = G->tail; if (G->tail >= 0) slot_next[G->tail] = id; else G->pub.head = id; G->tail = id; return; } slot_prev[id] = -1; slot_next[id] = bucket_head[b]; if (bucket_head[b] >= 0) slot_prev[bucket_head[b]] = id; bucket_head[b] = id; } VkSlotGroup *vk_texture_group_locked(int g) { return &grp[g].pub; } int vk_texture_group_next_locked(int id) { return slot_next[id]; } void vk_texture_groups_unstamp_locked(void) { for (int g = 0; g < ngrp; g++) grp[g].pub.stamp = 0; } uint64_t vk_texture_dirty_pos_locked(void) { return ndirty; } // A bucket listed several times since *pos is walked once a round: the walk reads its slots' tokens after every change // listed before the round began (the lock is held). Changes the walk itself makes (an entry a lookup created) are the // next round's. A bucket's slot groups go to gfn when given, else their members to fn. int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void (*gfn)(int g, void *arg), void *arg) { if (!slot_next || ndirty - *pos > DIRTY_N) { *pos = ndirty; return 0; } static uint32_t seen[AGEN_N], round; for (uint64_t q = *pos; q < ndirty;) { uint64_t end = ndirty; if (!++round) memset(seen, 0, sizeof seen), round = 1; for (; q < end; q++) { uint32_t x = dirty[q % DIRTY_N]; if (x & DIRTY_BUCKET) { uint32_t b = x & ~DIRTY_BUCKET; if (seen[b] == round) continue; seen[b] = round; for (int32_t id = bucket_head[b]; id >= 0; id = slot_next[id]) fn(id, arg); for (int32_t g = bucket_ghead[b]; g >= 0; g = grp[g].gnext) { // the walk registers nothing if (gfn) gfn(g, arg); else for (int32_t id = grp[g].pub.head; id >= 0; id = slot_next[id]) fn(id, arg); } } else { fn((int)x, arg); } } } *pos = ndirty; return 1; } static volatile int32_t tex_ready; static void tex_init(void *p) { (void)p; __atomic_store_n(&tex_ready, 1, __ATOMIC_RELEASE); return; } // NX2NAVI_LOCK_TIME: the outermost hold of the lock is timed from here, and one over 1 ms is reported with the code // that took it (perf_lock_hold); lock is not inlined so that its return address names that code. static uint64_t hold_t0; static void *hold_site; __attribute__((noinline)) static void lock(void) { if (!__atomic_load_n(&tex_ready, __ATOMIC_ACQUIRE)) plat_once(&tex_once, tex_init, NULL); plat_rmutex_lock_at(&tex_lock, __builtin_return_address(0)); if (tex_lock.depth == 1 && perf_lock_holds_on()) hold_t0 = plat_now_ns(), hold_site = __builtin_return_address(0); } static void unlock(void) { if (tex_lock.depth == 1 && hold_t0) { uint64_t d = plat_now_ns() - hold_t0; hold_t0 = 0; if (d > 1000000) perf_lock_hold(hold_site, d); } plat_rmutex_unlock(&tex_lock); } static uint32_t umax(uint32_t a, uint32_t b) { return a > b ? a : b; } static uint32_t umin(uint32_t a, uint32_t b) { return a < b ? a : b; } static uint32_t mip(uint32_t v, unsigned lv) { return umax(v >> lv, 1); } static void make_key(const tex_t *t, TexKey *k) { memset(k, 0, sizeof *k); k->addr = (uintptr_t)t->mem; k->d = t->d; k->has_layout = t->has_layout; memcpy(k->layout, t->layout, 8); } // Keys equal apart from the descriptor's swizzle selectors: the same storage, format and geometry. static int same_but_swizzle(const TexKey *a, const TexKey *b) { TexKey x = *a, y = *b; x.d.swizzle_r = x.d.swizzle_g = x.d.swizzle_b = x.d.swizzle_a = 0; y.d.swizzle_r = y.d.swizzle_g = y.d.swizzle_b = y.d.swizzle_a = 0; return !memcmp(&x, &y, sizeof x); } // Whether a slot (key k) may follow entry p, another descriptor at the same storage, when the slot's own entry is // released (slot_resolve). Ocarina's per-frame render targets share storage, so the live entry at an address is often // a target of another size that the frame laid out there: an effect's 1130x86 strip over the 480x270 half-size // lighting term at a 1280x720 scene. Followed, the compute pass that writes the term and the full-screen pass that // reads it both got the strip's image, the term read back as zero over most of the screen and a jump attack's hit // effect drew the right 44% of the scene black. A slot follows only an entry of its own size; block-compressed // formats are left out of the test, as a GPU encoder renders their blocks through a smaller uncompressed view of the // same storage. Counted in n_follow_refused. static uint64_t n_follow_refused; static int follow_size_ok(const Tex *p, const TexKey *k) { int bw1 = 1, bh1 = 1, bw2 = 1, bh2 = 1; nvn_fmt_bpb(p->key.d.format, &bw1, &bh1); nvn_fmt_bpb(k->d.format, &bw2, &bh2); if (bw1 > 1 || bh1 > 1 || bw2 > 1 || bh2 > 1) return 1; if (p->key.d.width == k->d.width && p->key.d.height == k->d.height && p->key.d.depth == k->d.depth) return 1; if (n_follow_refused++ < 8) rt_log("vk: a texture slot at %#llx (%ux%u format %#x) did not follow a %ux%u texture at the same storage: " "its own image instead", (unsigned long long)k->addr, k->d.width, k->d.height, k->d.format, p->key.d.width, p->key.d.height); return 0; } // Textures whose content the CPU provides in pool memory. Display, compressible, zcull, // depth, multisample and GPU-only storage stays GPU-owned. // mirror_refused: NULL for such a texture, else the rule that leaves it out, in words for a log line. static const char *mirror_refused(const tex_t *t) { if (!t->mem) return "no storage"; if (nvn_gpu_only((uintptr_t)t->mem)) return "storage in the GPU-only address window"; if (t->has_layout) return NULL; if (!(t->pool_flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) return "a pool the CPU has no access to"; if (t->d.flags & (NVN_TEXTURE_FLAGS_DISPLAY | NVN_TEXTURE_FLAGS_COMPRESSIBLE | NVN_TEXTURE_FLAGS_ADAPTIVE_ZCULL | NVN_TEXTURE_FLAGS_DEPTH16_PREFER_FAST_CLEAR | NVN_TEXTURE_FLAGS_ZCULL_SUPPORT_STENCIL)) return "texture flags (one of 0x1, 0x8, 0x80, 0x100, 0x400)"; if (t->d.samples > 1) return "multisample"; if (t->d.target == NVN_TEXTURE_TARGET_BUFFER) return "a buffer texture"; if (t->d.format >= 0x31 && t->d.format <= 0x36) return "a depth or stencil format"; return NULL; } static int tex_mirrored(const tex_t *t) { return !mirror_refused(t); } static void log_limited(volatile int32_t *n, const char *fmt, ...) __attribute__((format(printf, 2, 3))); static void log_limited(volatile int32_t *n, const char *fmt, ...) { if (plat_inc32(n) > 16) return; char buf[512]; va_list ap; va_start(ap, fmt); vsnprintf(buf, sizeof buf, fmt, ap); va_end(ap); rt_log("vk: %s", buf); } static void barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst, VkAccessFlags2 da) { VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa, .dstStageMask = dst, .dstAccessMask = da}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); } #define ALL_RW VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT // ---------------------------------------------------------------- images static VkFormatFeatureFlags features_of(VkFormat f) { VkFormatProperties p; vkGetPhysicalDeviceFormatProperties(vk_ctx()->phys, f, &p); return p.optimalTilingFeatures; } static VkImageAspectFlags aspect_of(const TexVkFormat *f) { VkImageAspectFlags a = 0; if (f->depth) a |= VK_IMAGE_ASPECT_DEPTH_BIT; if (f->stencil) a |= VK_IMAGE_ASPECT_STENCIL_BIT; return a ? a : VK_IMAGE_ASPECT_COLOR_BIT; } // Binds img into the pool memory behind its GPU address (physical pools, virtual pools through the page table) when // the memory type, alignment and contiguous size allow; 0 otherwise. // A depth / stencil image that needs more bytes than its NVN storage (guest_size) is not bound: AMD's RDNA2 driver // (26.10.1) puts the depth compression metadata after the depth data, and above 1080p that tail lies over the next // texture of the game's heap; each depth write then scrambled it (garbage band at the top, pink glow, blocks in the // grass, a blue checkered scene in the menu). Colour images keep their place: the RX 9070 also sizes some of them // above their storage, and nothing there showed that the GPU writes the excess. static int format_compressed(VkFormat f) { // vkCmdClearColorImage takes no block-compressed format return (f >= VK_FORMAT_BC1_RGB_UNORM_BLOCK && f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK) || (f >= VK_FORMAT_ASTC_4x4_SFLOAT_BLOCK && f <= VK_FORMAT_ASTC_12x12_SFLOAT_BLOCK); } // Image reuse (NX2NAVI_TEX_REUSE, default on): a GPU-only texture that needs an image of its own takes the image of a // released entry made with the same parameters (ImgKey) instead of a new one, which then starts over as a new image // would (init_image: undefined content, cleared). Ocarina places its per-frame render targets with a linear allocator // behind small targets whose size follows an effect on screen (77x77 to 97x97), so every few frames the targets after // them move to new addresses, new descriptors whose images were made again: vkCreateImage, the memory requirements and // the binding of a dedicated allocation cost 8% of the replay in Kokiri Forest. The released entries of each parameter // set are kept in a list, newest release first; create_tex takes the one nearest in address of those released at least // two frames earlier, never during the current submission's replay, whose later parts may still use it, and an // overwritten one (below) before any other. static nvn_hmap reuse = {.klen = sizeof(ImgKey)}; // ImgKey -> the newest released reusable entry static int tex_reuse = -1; static uint64_t n_reused, n_reuse_new; // since the last perf line: images taken from a released entry, made anew static uint64_t n_restarts; // superseded entries taken back (start_at) // Moved render targets. Ocarina keeps about a thousand texture objects for its per-frame render targets and lays the // targets out anew every frame with a linear allocator in one GPU-only pool. A target early in that layout changes its // size with an effect on screen (811x811, then 778x778 a frame later in Kakariko Village), and every target after it // then lies about 100 KB further on: eleven targets, 55 MiB of images, at new addresses in each frame the size // changes, each a new entry with an image of its own, as few of them can lie in the pool memory itself (pool_fits). // The game finalizes an old texture object only when it needs the object again, 90 to 300 frames later, and a // descriptor in the texture pool goes on naming it, so the entries were neither released nor idle: after 6,000 frames // of walking through the village 1,043 of them held 6.1 GiB of device memory, against 233 MiB in the targets still // drawn, and once released they waited GPU_STALE_FRAMES more. A driver that charges device memory to the system's // commit kept the commit low from then on, and the idle textures of the area went and came back (commit_evict). // // On the console those targets are views of one pool's memory: when another target is drawn over a target's bytes, its // content is gone. overwrite_others says so here. When the GPU writes a texture in the GPU-only address window // (nvn_gpu_only), each entry at another address whose storage overlaps it is marked overwritten and released as if // the game had finalized it. Its image then goes to the next texture made with the same parameters (create_tex), or // away OVERWRITTEN_KEEP presents later (drop_overwritten), or as early as an intact entry's while memory is short // (drop_stale). Left alone are an entry at the same address (start_at decides about those), an entry the written // texture lies inside (a level or a layer of it, which a game draws through a texture of its own) and an entry // written in this frame or the one before: two layouts that take turns within a frame keep their images. The mark // goes when the game takes the entry back (entry_of). // gwin: the entries this applies to (reusable, in the window), sorted by address; gwin_span: the largest storage size // one of them had, which bounds how far below an address an overlapping entry can start. #define OVERWRITE_FRAMES 2 #define OVERWRITTEN_KEEP PRESENTS(600) static struct Tex **gwin; static int ngwin, cgwin; static uint64_t gwin_span; static uint64_t n_overwritten, n_over_drops; // since the last perf line: entries marked, dropped by drop_overwritten static int gwin_lower(uint64_t a) { int lo = 0, hi = ngwin; while (lo < hi) { int m = (lo + hi) / 2; if (gwin[m]->key.addr < a) lo = m + 1; else hi = m; } return lo; } static void gwin_add(Tex *e) { if (ngwin == cgwin) { cgwin = cgwin ? 2 * cgwin : 256; if (!(gwin = realloc(gwin, sizeof *gwin * (size_t)cgwin))) rt_fatal("vk: out of memory for the render target list"); } int i = gwin_lower(e->key.addr); memmove(gwin + i + 1, gwin + i, sizeof *gwin * (size_t)(ngwin - i)); gwin[i] = e, ngwin++; if (e->t.L.size > gwin_span) gwin_span = e->t.L.size; e->in_gwin = 1; } static void gwin_remove(Tex *e) { if (!e->in_gwin) return; for (int i = gwin_lower(e->key.addr); i < ngwin && gwin[i]->key.addr == e->key.addr; i++) if (gwin[i] == e) { memmove(gwin + i, gwin + i + 1, sizeof *gwin * (size_t)(ngwin - i - 1)); ngwin--; break; } e->in_gwin = 0; } static int tex_reuse_on(void) { if (tex_reuse < 0) { tex_reuse = cfg_bool("NX2NAVI_TEX_REUSE"); rt_log("vk: texture image reuse %s (NX2NAVI_TEX_REUSE)", tex_reuse ? "on" : "off"); } return tex_reuse; } static void destroy_tex(Tex *e); static int used_lately(const struct Tex *e, uint64_t now, uint64_t within); static int sampled_lately(const struct Tex *e, uint64_t now, uint64_t within); static int marks_tell(void); // The presents a released entry must have gone unused before its image may go to another texture (create_tex): the // pool's marks reach an entry a few presents after the draw that sampled it. #define REUSE_UNUSED 16 // The same for an overwritten entry while the card is past the VRAM pressure's share of its budget (over_share): what // the marks need and no more. A mark is read at most MARKS_RB copies of MARKS_EVERY submissions after its draw // (vk_exec_draw.c vk_exec_pool_marks), three frames of six submissions, so after eight presents without a stamp none // is on its way. At 3840x2160 the targets moved in 16 presents hold 3.1 GiB, which a 16 GiB card does not have beside // the 10 GiB of a town. #define REUSE_UNUSED_FULL 8 static int over_share; // sweep_pressure: the driver's figure of the memory in use is past that share // The released entries of one parameter set create_tex looks at, newest release first. While the game's layout moves // in every frame, each frame releases every target of a set and none of them is unused yet for REUSE_UNUSED presents: // with 128 looked at, eight targets of one set a frame hid every older entry behind the young ones and each texture // got a new image. #define REUSE_LOOK 1024 // since the last perf line: released entries create_tex passed by as released too lately, as used, as named by a slot static uint64_t n_reuse_young, n_reuse_used, n_reuse_slot; // NX2NAVI_TEX_REUSE_NAMED (default 0; 1: as before, any released entry's image may go): a registered slot that is not // finalized names e's descriptor, so a shader can still sample e through the pool. The game finalizes a texture object // while another one it initialized at the same storage with the same parameters stays registered (vk_texture_release // leaves that slot current), and the entry they share is released all the same. Its image then went to the next // texture of its parameters two frames later (create_tex, reuse), the entry was dropped, and the slot had nothing to // show: the pool does not make an entry for storage the CPU cannot read, so the slot kept its stand-in until the game // drew into the texture again. Ocarina's 384x384 shadow maps are such textures: in a player's recording four slots of // them were sampled as stand-ins in the frame after three of their images had gone to other shadow maps, and the // front of a house lost its shadow for that frame. Such an entry keeps its image. static uint64_t n_reuse_named; // released entries left out of the reuse because a current slot names them static int slot_names(const Tex *e) { static int off = -1; if (off < 0) off = cfg_bool("NX2NAVI_TEX_REUSE_NAMED"); if (off || !sinfo) return 0; for (int id = bucket_head[agen_idx(e->key.addr)]; id >= 0; id = slot_next[id]) { const TexSlot *s = &tslots[id]; if (s->finalized || (uintptr_t)s->t.mem != e->key.addr) continue; TexKey sk; make_key(&s->t, &sk); if (memcmp(&sk, &e->key, sizeof sk)) continue; if (!n_reuse_named++ || !(n_reuse_named & (n_reuse_named - 1))) // the first, then at each power of two rt_log("vk: a released texture keeps its image, a registered slot still names it: %#llx fmt %#x %ux%u, slot " "%d (%llu so far; NX2NAVI_TEX_REUSE_NAMED=1 lets such an image go to another texture)", (unsigned long long)e->key.addr, e->t.d.format, e->t.d.width, e->t.d.height, id, (unsigned long long)n_reuse_named); return 1; } return 0; } // Why an entry was destroyed (drop_tex), in the order of drop_names. enum { DROP_NONE, DROP_MAX, DROP_CAP, DROP_COMMIT, DROP_VRAM, DROP_IDLE_VRAM, DROP_IDLE_COMMIT, DROP_DISUSE, DROP_OVERWRITTEN, DROP_REPLACED, DROP_REUSED, DROP_RANGE, DROP_EXACT, DROP_SHUTDOWN, DROP_COMPACT, DROP_AGE, DROP_HARD, DROP_N }; static void drop_tex(Tex *e, int why); static void reuse_dropped(const Tex *e, int why); // "reuse of released textures", after vk_texture_census static void reuse_made(const Tex *e); static void reuse_taken_back(const Tex *e); static int compact_on(void); // NX2NAVI_TEX_COMPACT ("compaction", near the end of the file) static void compact_step(void); static void compact_forget(Tex *e); // How an entry's content started (start_at; Tex.og_start), in the order of start_names. enum { START_NEW, START_UPLOAD_LIVE, START_COPY_LIVE, START_COPY_RELEASED, START_EMPTY_LIVE, START_EMPTY_RELEASED, START_COPY_GONE, START_FILL, START_N }; // What put content into the image of an entry the CPU does not write (Tex.had); a GPU write is pub.gpu_owned. enum { HAD_COPY = 1, HAD_TEXELS = 2, HAD_FILL = 4 }; // ---- split fill (NX2NAVI_TEX_SPLIT_FILL, tex_split_fill in nx2navi.ini: 1 the default, 0 off, 2 filled once) // Two descriptors can name one storage the CPU writes: a texture whose content the cache takes from the CPU // (tex_mirrored), and a second one with a texture flag for which tex_mirrored refuses. The second has no upload of its // own: its slot shows the first one's image for as long as that entry is live (slot_resolve). Once the first is // released, a lookup of the second's slot makes the second an entry of its own (entry_of), start_at destroys the // released first one, and the new image has nothing to start from: cleared, never uploaded, and from then on what // the slot shows, also after the first descriptor is registered again and gets a new entry beside it. On the console // both descriptors read the same bytes. tex_probe.c test_split makes the sequence and reads the image back: zeros. // // With split fill, a new entry the CPU does not write, at storage the CPU can see, starts with those bytes: // - by a copy on the GPU from the released CPU-written entry start_at is about to destroy, when that image holds the // storage as it is now (uploaded, and no write since its last check) or content the GPU wrote, and // alias_compatible allows; // - else from the storage itself, uploaded as a CPU-written texture's is (upload). // With 1 the image then follows the storage as a CPU-written one's does (from_storage: make_fresh, the pool's list) // until the GPU or WriteTexels writes it. The texture pool refresh makes such an entry also for a slot whose descriptor // the game has let go of, long before the next texture's bytes arrive in that memory, and when both descriptors are // registered again the slot finds that entry: filled once (2), it shows the texture that was there before. // Left out: depth, stencil and multisample images, buffer textures, pools the CPU has no access to, storage over // SPLIT_FILL_MAX, and an entry made with a released entry's image, whose first transition is recorded at the replay's // position, after the upload batch (init_new); so create_tex gives an entry that is to be filled a new image. Nothing // is filled where a live entry at the address holds content the GPU wrote (that is what the new one should show, and // start_at copies it when it can), nor with NX2NAVI_TEX_ALIAS off. // // What follows the storage also takes what a stale descriptor's entry does not need: an entry made for a slot the game // let go of is uploaded again whenever its pages are written, by the next textures loaded into that memory, read in // its own format. The perf line counts those ("filled again after a write"); 2 does without them. // // What it costs: an upload (a copy out of the storage, a decode for ASTC) for each such entry that finds no image to // copy from, where the image was only cleared before. A render target in a CPU pool that the game makes anew often // pays that every time and no longer takes a released entry's image; the perf line counts the fills, their bytes and // their time. Until the GPU first writes it, its storage is also checked for writes once a submission. // What it can show: the bytes in the storage, whatever they are. For a render target that is the last picture the // game or a write-back left there, where the image was zero before: a pass that reads a new target before anything // drew into it sees that older picture instead of nothing. #define SPLIT_FILL_MAX ((uint64_t)32 << 20) static int split_mode = -1; // since the last perf line: entries started by a copy from the released entry they replace, filled from storage (with // the bytes and the time), filled again after a write; left empty for a reused image, for their size static uint64_t n_split_copy, n_split_fill, n_split_bytes, n_split_ns, n_split_refill, n_split_reused, n_split_big; static uint64_t split_total[2]; // copies and fills so far (vk_texture_split_counts) static int split_fill_mode(void) { if (split_mode < 0) { int v = cfg_int("NX2NAVI_TEX_SPLIT_FILL"); split_mode = v == 0 ? 0 : v == 2 ? 2 : 1; rt_log("vk: a texture the CPU does not write, made at storage the CPU writes, %s (NX2NAVI_TEX_SPLIT_FILL=%d)", !split_mode ? "starts empty" : split_mode == 2 ? "starts with the storage's bytes, once" : "starts with the storage's bytes and follows them", split_mode); } return split_mode; } // A test's hook (tex_probe.c): the mode from here on, and the copies and fills made so far. void vk_texture_split_fill_set(int mode) { split_mode = mode; } void vk_texture_split_counts(uint64_t *copies, uint64_t *fills) { *copies = split_total[0], *fills = split_total[1]; } // Whether split fill gives a new entry for t its storage's bytes, apart from the size (SPLIT_FILL_MAX). static int split_wants(const tex_t *t, const TexVkFormat *f) { if (!split_fill_mode() || !t->mem || nvn_gpu_only((uintptr_t)t->mem) || tex_mirrored(t)) return 0; if (!(t->pool_flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) return 0; if (f->depth || f->stencil || t->d.samples > 1 || t->d.target == NVN_TEXTURE_TARGET_BUFFER) return 0; int bw, bh; return nvn_fmt_bpb(t->d.format, &bw, &bh) != 0; } // Whether e's image takes its content from the storage: a texture the CPU writes, or one split fill fills. static int from_storage(const Tex *e) { return e->pub.mirrored || e->sfill == 1 || (e->sfill == 2 && !e->uploaded); } // Whether an image with these requirements can lie in the pool memory at e's address (*sp: that memory). static int pool_fits(VkCtx *c, const Tex *e, const VkMemoryRequirements *req, uint64_t guest_size, VkMemSpan *sp) { if ((e->f.depth || e->f.stencil) && req->size > guest_size) return 0; if (!req->size) return 0; // an image the device could not lay out (image_size) if (!vk_mem_lookup(e->pub.addr, req->size, sp) || !sp->alloc || sp->alloc_offset % req->alignment) return 0; VmaAllocationInfo info; vmaGetAllocationInfo(c->vma, sp->alloc, &info); return req->memoryTypeBits >> info.memoryType & 1; } // The memory requirements of each parameter set (ImgKey) an image was made with; images made with the same parameters // have the same requirements. With them create_tex makes no image that pool_fits would turn down: Ocarina's moving // render targets mostly miss the device's alignment, and making the image only to read its requirements // (vkGetImageMemoryRequirements does most of the driver's layout work) cost 5% of the replay in Kokiri Forest. static nvn_hmap img_reqs = {.klen = sizeof(ImgKey)}; // ImgKey -> VkMemoryRequirements (malloc'd) static uint64_t n_bind_skips; // since the last perf line #define DEDICATED_MIN ((VkDeviceSize)32 << 20) // create_tex: images this large get memory of their own static int bind_pool_memory(VkCtx *c, Tex *e, VkImage img, uint64_t guest_size) { VkMemoryRequirements req; vkGetImageMemoryRequirements(c->device, img, &req); if (!nvn_hm_get(&img_reqs, &e->ik)) { VkMemoryRequirements *q = malloc(sizeof *q); if (!q) rt_fatal("vk: out of memory"); *q = req; nvn_hm_put(&img_reqs, &e->ik, q); } VkMemSpan sp; if (!pool_fits(c, e, &req, guest_size, &sp)) return 0; if (vmaBindImageMemory2(c->vma, sp.alloc, sp.alloc_offset, img, NULL) != VK_SUCCESS) return 0; e->bound_to = sp.alloc; return 1; } static void img_key_of(const VkImageCreateInfo *ci, ImgKey *k) { memset(k, 0, sizeof *k); k->flags = ci->flags, k->type = ci->imageType, k->format = ci->format, k->w = ci->extent.width; k->h = ci->extent.height, k->d = ci->extent.depth, k->levels = ci->mipLevels, k->layers = ci->arrayLayers; k->samples = ci->samples, k->usage = ci->usage; } // agl's BC4 encoder draws a BC texture's blocks through an RGBA16UI view, one texel per 8-byte block. // Vulkan does it with an uncompressed // view of a BLOCK_TEXEL_VIEW_COMPATIBLE image; EXTENDED_USAGE lets the BC image carry the COLOR_ATTACHMENT usage that // only such views support. Whether the device takes that combination for this format and shape: static int block_views_ok(const VkImageCreateInfo *ci) { VkPhysicalDeviceImageFormatInfo2 fi = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, .format = ci->format, .type = ci->imageType, .tiling = ci->tiling, .usage = ci->usage, .flags = ci->flags}; VkImageFormatProperties2 p = {.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2}; if (vkGetPhysicalDeviceImageFormatProperties2(vk_ctx()->phys, &fi, &p) != VK_SUCCESS) return 0; const VkImageFormatProperties *q = &p.imageFormatProperties; return ci->extent.width <= q->maxExtent.width && ci->extent.height <= q->maxExtent.height && ci->extent.depth <= q->maxExtent.depth && ci->mipLevels <= q->maxMipLevels && ci->arrayLayers <= q->maxArrayLayers; } // The memory the device wants for an image made from ci: 0 when it cannot lay it out. RADV on GFX9 (Vega, Renoir) // hands back a 0-byte image for a mipmapped block-compressed 3D image at least as deep as it is wide in blocks, with // COLOR_ATTACHMENT usage and no STORAGE: that usage asks for a thin 3D layout (Mesa radv_image.c), for which the GFX9 // address library has no swizzle mode left (gfx9addrlib.cpp), and the error is dropped; VMA then refuses the 0-byte // allocation. The format properties call it supported, so only the size shows it. static VkDeviceSize image_size(const VkImageCreateInfo *ci) { VkDeviceImageMemoryRequirements q = {.sType = VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS, .pCreateInfo = ci}; VkMemoryRequirements2 r = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2}; vkGetDeviceImageMemoryRequirements(vk_ctx()->device, &q, &r); return r.memoryRequirements.size; } // UNDEFINED -> GENERAL; new depth / stencil images start cleared (LAS textures.c: the game copies depth // without stencil, and the stencil of those images must not be whatever the memory held). // A reused image (create_tex) was another texture's: the transition waits for every earlier command, which may still // read or write it. static void init_image(Tex *e, VkCommandBuffer cb) { VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = e->reused ? VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT : VK_PIPELINE_STAGE_2_NONE, .srcAccessMask = e->reused ? VK_ACCESS_2_MEMORY_WRITE_BIT : 0, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = e->pub.image, .subresourceRange = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; vkCmdPipelineBarrier2(cb, &di); if (e->f.depth || e->f.stencil) { VkClearDepthStencilValue zero = {0.0f, 0}; VkImageSubresourceRange rg = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; vkCmdClearDepthStencilImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &rg); barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); } else if (vk_fix_zero() && !e->bound_to && !e->pub.mirrored && !format_compressed(e->f.vk)) { // GPU-only storage that got memory of its own starts at zero, as the pool it stands for (vk_memory.c // alloc_physical). Images aliasing pool memory are not cleared: that would erase what other textures rendered // there; mirrored images are uploaded from their storage on first use. VkClearColorValue zero = {0}; VkImageSubresourceRange rg = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; vkCmdClearColorImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &rg); barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); } } // NX2NAVI_DEPTH_INIT_INLINE (default 1; 0: in the upload batch, as before): a new depth or stencil image goes through // init_image at the replay's position too. Its clear writes the image's memory, and an image bound into pool memory // (bind_pool_memory) shares that with the live texture at its address; from the upload batch, which runs before the // whole part being recorded (vk_sync.c split_part), the clear can zero what earlier commands of the part still read as // that texture. Ocarina makes a new depth descriptor over a live one about 40 times a second at the castle town gate // (a 1280x720 DEPTH32F over a 640x360 one, its sun shadow maps as they resize), and for one frame a few times a minute // the shadowing went fully dark. At the replay's position vk_exec_inline_begin's barrier orders the clear after them. static int depth_init_inline(void) { static int on = -1; if (on < 0) on = rt_knob("DEPTH_INIT_INLINE", "1") != 0; return on; } // A new entry's first transition. A reused image goes through it at the replay's current position: earlier commands // of the part being recorded may still use the image as the texture it was, and the upload batch runs before the part. // So does a new depth or stencil image (depth_init_inline). static void init_new(Tex *e) { int ds = !e->reused && (e->f.depth || e->f.stencil) && depth_init_inline(); if (e->reused || ds) { int reopen; VkCommandBuffer cb = vk_exec_inline_begin(&reopen); if (cb) { init_image(e, cb); vk_exec_inline_end(reopen); if (ds) n_dinit_inline++, n_dinit_inpass += reopen != 0; return; } } if (ds) n_dinit_upload++; init_image(e, vk_upload_begin()); vk_upload_end(); } // NX2NAVI_SHADOW_SCALE (1 to 4): the game's shadow maps are made this many times larger on each side. NX2navi renders its // sun cascades into a 960x960 DEPTH16 2D array of 4 layers and its static shadows into 1440x1440 and 720x720 arrays of 2, // whatever the output resolution, and shrinks the cascades under load (872, 848, 800, 480 and so on). Those are made // larger here: render passes into them scale the game's viewports and scissors (vk_exec.c resolve_rt, pass_scale), // copies scale their regions (vk_exec_xfer.c sub_resolve), draws into them scale the game's slope-scaled depth bias // (vk_exec_draw.c slope_bias), and shaders sample them with normalised coordinates, so the game sees nothing of it. A map whose storage the CPU writes (mirrored) or that aliases pool memory keeps its size, // as its texels must match the guest layout. static int shadow_scale(void) { static int scale = -1; if (scale < 0) { int s = cfg_int("NX2NAVI_SHADOW_SCALE"); s = s < 1 ? 1 : s > 4 ? 4 : s; rt_log("vk: shadow map scale %d", s); scale = s; } return scale; } static int shadow_scale_of(const nvn_texdesc *d, const TexVkFormat *f, uint32_t samples, int mirrored) { int scale = shadow_scale(); if (scale == 1 || mirrored || samples > 1 || d->format != NVN_FORMAT_DEPTH16 || !f->depth) return 1; if (d->target != NVN_TEXTURE_TARGET_2D_ARRAY || d->width != d->height || d->width < 256 || d->depth < 2 || d->depth > 4) return 1; uint32_t most = vk_ctx()->props.limits.maxImageDimension2D; return d->width * (uint32_t)scale <= most ? scale : 1; } // NX2NAVI_SHADOW_PCF (default 0): every depth-compare sampler filters linearly. Through a nearest filter each of the // game's shadow compares tests one texel and returns 0 or 1, so a shadow edge is a staircase of shadow map texels, and // the sun cascades follow the camera, so the stairs crawl along the ground as it moves (Hyrule Field's hills). Through a // linear filter the GPU tests the 2x2 texels around the coordinate and blends the four results (percentage-closer // filtering), which the game's own four compares then smooth further. With NX2NAVI_SHADOW_SCALE above 1 the samplers // filter linearly anyway (make_sampler). Read once: samplers live until shutdown. static int shadow_pcf(void) { static int on = -1; if (on < 0) { on = cfg_bool("NX2NAVI_SHADOW_PCF"); rt_log("vk: depth-compare samplers: %s (NX2NAVI_SHADOW_PCF=%d)", on ? "linear, each compare blends the 2x2 texels around it" : "the game's filters", on); } return on; } // The filters of the depth-compare samplers made, each kind once (8 at most): whether NX2NAVI_SHADOW_PCF changes // anything shows in the log (with every one already linear it does not). Caller holds the lock. static void compare_sampler_note(const samp_par *d) { static uint32_t seen[8]; static int n; uint32_t kind = (uint32_t)d->minf | (uint32_t)d->magf << 8 | (uint32_t)d->cmp_func << 16 | 1u << 24; for (int i = 0; i < n; i++) if (seen[i] == kind) return; if (n == 8) return; seen[n++] = kind; rt_log("vk: depth-compare sampler: min filter %u, mag filter %u (0 nearest, 1 linear, more: mipmapped), function %u", (unsigned)d->minf, (unsigned)d->magf, (unsigned)d->cmp_func); } // ---- ASTC as BC1 (NX2NAVI_ASTC_BC1, vk_astc.c) // A texture the CPU provides whose ASTC blocks all decode opaque is held as BC1, not BC3: the encoder then writes only // the colour half of the BC3 blocks it would have written (bc3_encode.comp), which BC1 decodes to the same texels, in // half the memory. Every ASTC texture is made BC3 (create_tex); its first upload checks its blocks inside the copy into // staging it makes anyway (astc_copy_opaque, tex_astc.c astc_blocks_opaque), and when all are opaque the entry gets a // BC1 image in place of the BC3 one before the decode (astc_reformat). A check of its own where the texture is made // cost 1.7 s of the render thread over a seven-minute teleport tour; the image made again costs a fraction of that. // The check covers every 16-byte block of the storage that is copied, the padding between levels and layers included // (zeros there decode as error blocks, opaque magenta; other bytes can only make it stricter). Each later upload into a // BC1 image checks again: the game may load other content into the same storage, and content with alpha gets a BC3 // image back before it is decoded. Blocks of the endpoint modes with alpha are decoded to tell; past // ASTC_OPAQUE_DECODES of them in one texture it counts as not opaque, as the check would cost more than it saves. // Blocks the CPU untiles first (not astc_bl_regions) are checked in the storage before the copy (astc_opaque). #define ASTC_OPAQUE_DECODES 4096 static uint64_t n_bc1_made, n_bc1_made_bytes, n_bc1_widened, n_bc1_checks, n_bc1_check_ns, n_bc1_check_bytes; // 1 when every block of t's storage at addr decodes opaque; 0 when one does not or the storage is not mapped. static int astc_opaque(const tex_t *t, uint64_t addr, const TexVkFormat *f) { uint64_t t0 = kern_now_ns(), n = t->L.size / 16; if (!t->mem || !n || !mem_read_hold(addr, t->L.size)) return 0; uint32_t decoded = 0; int ok = astc_blocks_opaque_mt((const uint8_t *)t->mem, n, f->astc_bw, f->astc_bh, f->astc_srgb != 0, ASTC_OPAQUE_DECODES, &decoded, NULL); mem_read_release(); n_bc1_checks++, n_bc1_check_ns += kern_now_ns() - t0, n_bc1_check_bytes += t->L.size; return ok; } // ---- texture detail (NX2NAVI_TEXTURE_DETAIL = auto | full | reduced | low, texture_detail in nx2navi.ini; read at // the first texture, so a change waits for a restart) // The material textures of a loaded area are what fills a small card, and three quarters of a mipmapped texture is its // largest level. Over the budget Windows moves device memory into system memory and the frame rate falls to a // fraction (a tester's RTX 3060 Laptop, 6 GiB with a budget of 4,730 MiB, ran the world at 5 to 15 fps). // // Measured on an RTX 5090 with the budget taken as those 4,730 MiB (NX2NAVI_VRAM_BUDGET_MB), a walk through Castle // Town at 1440p with DLSS Balanced, device memory in use at its peak and the textures in it: // full 6,907 MiB, textures 4,302 MiB (2,852 MiB of them ASTC as BC3), 2,589 M ASTC texels decoded // reduced 5,472 MiB, textures 2,357 MiB (908 MiB), 826 M texels // low 4,987 MiB, textures 1,997 MiB (548 MiB), 503 M texels // About 1.1 GiB of each is outside the textures and buffers (DLSS alone reports 0.7 GiB). What a level does not touch // stays: about 0.8 GiB of BC4, BC5 and BC7 textures the game ships in those formats, 0.65 GiB of render targets and // 0.9 GiB of buffer mirrors. // // reduced makes such a texture without its largest level, low without its two largest when it is 1024x1024 or more // (detail_skip decides). The image then holds the game's levels from VkTexImage.skip on: image level i is the game's // level i + skip, at that level's own size, so a level and a texel rectangle inside it mean the same in both and only // the level index moves. Every path from a game level to the image subtracts skip: the upload and its ASTC regions // (upload, astc_bl_regions), WriteTexels and the write-back (image_texels, write_back_n), the views (vk_exec_draw.c // get_view) and the sampler's LOD clamp (vk_sampler_detail_by_id_locked). Shaders sample with normalised coordinates // and derivatives, which pick the same game level as before wherever it is kept. // // Only a texture for which that holds is reduced: an ASTC texture this runtime decodes (not one the device samples as // ASTC, whose copies run on the image, vk_exec_xfer.c sub_resolve), a 2D texture or 2D array, block-linear, one // sample, mipmapped, DETAIL_MIN texels or more each way, whose content the CPU provides (tex_mirrored). That leaves // out render targets and storage images (no ASTC format can be either), 3D and cube textures, the small and unmipped // ones (interface art, glyph atlases, lookup tables, ramps) and whatever the GPU copies, as texture copies of a // decoded ASTC texture are not replayed at all. Two more are left whole: // - a texture made over a live one that holds GPU-written content, which start_at copies in texel for texel; // - a texture a shader reads by texel position or whose size it asks for (exact, below): a fetch at a level the image // lacks, or coordinates worked out from the size of another texture, would read the wrong texels. The translated // shaders do that only through fixed bindings (gfx_shader.c reflect_exact; none of 44,081 prepared shaders of game // 1.1.1 does it through the texture pool), so the replay sees the texture at the binding and asks for it whole // (vk_texture_exact_by_id_locked). The key then stays whole for the session. // The size of a 2D array is also asked for by the layer clamp the translator puts before array samples; that reads // the layer count alone, which no level changes, and does not count (reflect_exact). // // auto: full from DETAIL_FULL_MB of device-local budget on, low under DETAIL_LOW_MB, reduced between. Textures give // way from 80% of the budget on (sweep_pressure), so a level wants its use at or under that: full, 6.9 to 7.5 GiB in // the town and in Hyrule Field; reduced, 5.5 GiB, on 8 GiB cards (their budget is about 7.2 GiB); and every 6 GiB // card, whose budget stays under 5.5 GiB, gets low. // Full was chosen from 8,192 MiB, which is the cards of 10 GiB (their budgets start near 9 GiB). What full takes is // more than its textures, though: after a tour of eleven areas at 2560x1440 the driver counted 12.7 GiB in use on a // 24 GiB card, 4.6 GiB of it free space inside the allocator's blocks, which the pressure check leaves out and the // driver does not (vk_mem_vram_live). With that card filled to the room of an 8 GiB one, memory went to system memory // and the same tour ran at 29.6 fps with a 1% low of 11.4; with reduced, 42.8 and 36.4, and nothing spilled. So full // starts at 10,240 MiB of budget, the cards of 12 GiB (a 12 GiB card's budget was 11,313 MiB in a player's log). // An integrated GPU's budget is a share of the system's memory, not memory of its own: one reported 10,361 MiB on a // handheld with 16 GiB of memory in all, auto chose full, and the game ended when a save was loaded until its player // set reduced. auto never chooses full there. // The choice itself is vk_tex_detail.h's tex_detail_auto, with the three budgets (tests/tex_detail_check.c). #define DETAIL_MIN 256 #define DETAIL_LOW_MIN 1024 // The levels above assume the ASTC textures end as BC3 (or stay ASTC), a byte a texel. Where they stay RGBA8 (astc = // cpu, a card without the decoder, astc_bc = 0) they take four times that, which is what one level left out gives // back: under this budget auto then goes one step lower. A card with 8 GiB that decoded on the CPU held two texture // arrays of 852 MiB each, was 560 MiB over its budget and lost its device. #define EXACT_MAX 65536 static nvn_hmap exact = {.klen = sizeof(TexKey)}; // keys made whole whatever the setting; value 1 static uint64_t n_exact; // textures made whole again so far static int alias_has_gpu_content(const struct Tex *src); static int64_t detail_budget_mb = -1; // the device-local budget auto went by, in MiB; 0: the device reports none // The device's video memory is the system's own (an integrated GPU): see DETAIL_FULL_MB. static int detail_shared(void) { const VkCtx *c = vk_ctx(); return c && c->props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; } int vk_texture_detail_auto(void) { int64_t mb = __atomic_load_n(&detail_budget_mb, __ATOMIC_RELAXED); if (mb < 0 && !vk_ctx()) return 0; // asked before there is a device: nothing is kept if (mb < 0) { // read once: the budget moves with what else uses the card, the choice must not uint64_t used = 0, budget = 0; int exact_budget = 0; mb = vk_mem_vram(&used, &budget, &exact_budget) ? (int64_t)(budget >> 20) : 0; __atomic_store_n(&detail_budget_mb, mb, __ATOMIC_RELAXED); } return tex_detail_auto(mb, detail_shared(), vk_astc_mode() != VK_ASTC_NATIVE && !vk_astc_bc()); } static const char *const detail_names[3] = {"full", "reduced", "low"}; static int detail_chosen = -1; // detail_most's answer once it is made // The texture detail in effect, for a fatal error's summary line. No lock, any thread. const char *vk_texture_detail_name(void) { int most = __atomic_load_n(&detail_chosen, __ATOMIC_RELAXED); return most < 0 ? "not chosen yet (no texture made)" : detail_names[most]; } static int detail_most(void) { // the most levels a texture is made without: 0 full, 1 reduced, 2 low static int most = -1; if (most >= 0) return most; const char *v = cfg_value("NX2NAVI_TEXTURE_DETAIL"); most = !strcmp(v, "full") ? 0 : !strcmp(v, "reduced") ? 1 : !strcmp(v, "low") ? 2 : -1; if (most >= 0) { rt_log("vk: texture detail: %s (texture_detail)", v); } else { most = vk_texture_detail_auto(); if (!detail_budget_mb) rt_log("vk: texture detail: full (texture_detail = %s; the device reports no video memory budget)", v); else rt_log("vk: texture detail: %s, chosen for a video memory budget of %lld MiB (texture_detail = %s; full from " "%d MiB, low under %d MiB%s%s)", detail_names[most],(long long)detail_budget_mb, v, DETAIL_FULL_MB, DETAIL_LOW_MB, detail_shared() && detail_budget_mb >= DETAIL_FULL_MB ? "; not full on an integrated GPU, whose budget is a share of the system's memory" : "", vk_astc_mode() != VK_ASTC_NATIVE && !vk_astc_bc() && detail_budget_mb < DETAIL_RGBA8_MB ? "; one step lower because ASTC textures stay uncompressed here" : ""); } __atomic_store_n(&detail_chosen, most, __ATOMIC_RELAXED); return most; } // The top mip levels the image of t is made without (the rule above). w, h, levels: the image as it would be whole. static uint32_t detail_skip(const tex_t *t, const TexKey *k, const TexVkFormat *f, uint32_t w, uint32_t h, uint32_t levels, uint32_t samples) { int most = detail_most(); const nvn_texdesc *d = &t->d; if (!most || !f->astc_bw || samples > 1 || levels < 2 || w < DETAIL_MIN || h < DETAIL_MIN) return 0; if ((d->target != NVN_TEXTURE_TARGET_2D && d->target != NVN_TEXTURE_TARGET_2D_ARRAY) || nvn_tex_is_linear(d)) return 0; if (!tex_mirrored(t) || (exact.n && nvn_hm_get(&exact, k))) return 0; for (const struct Tex *p = nvn_hm_get(&by_addr, &k->addr); p; p = p->next_at) if (!p->released && alias_has_gpu_content(p)) return 0; return most >= 2 && levels >= 4 && w >= DETAIL_LOW_MIN && h >= DETAIL_LOW_MIN ? 2 : 1; } // Where texture images go when video memory runs short. An RX 580 (8 GiB, AMD 23.10) shows its memory as a 6 GiB // device-local heap and a 2 GiB device-local, host-visible one. Asked for device-local memory without a budget, VMA // took the first heap every time: a save load put 5.6 GiB into it against a budget of 5.1 GiB while the other heap // held 69 MiB, and the next submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY. Each step below allocates within the // heaps' budgets (VK_EXT_memory_budget): device-local memory of any heap; the second allocates as before, past the // budget, where the driver keeps it in video memory or moves it out itself. System memory is never asked for: on a GPU // with one device-local heap (an RTX 3050 Laptop, 4 GiB) a step for any memory within the budget put every texture past // the budget in system memory, 4096 and more images, which the process's commit then held: with 16 GiB of RAM the // system's commit ran out, host memory imports failed, and the GPU wrote through the address of a pool left without a // buffer (device lost). static uint64_t placed[VK_MAX_MEMORY_HEAPS], n_spilled; static void note_placement(VkCtx *c, VmaAllocation a, int step) { // tex_lock held VmaAllocationInfo info; vmaGetAllocationInfo(c->vma, a, &info); uint32_t heap = c->mem_props.memoryTypes[info.memoryType].heapIndex; if (heap >= VK_MAX_MEMORY_HEAPS) return; uint64_t n = ++placed[heap]; int first_heap = 1; for (uint32_t h = 0; h < heap; h++) if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) first_heap = 0; if (step == 0 && first_heap && (c->mem_props.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)) return; if (step) n_spilled++; if (n <= 4 || !(n & (n - 1))) rt_log("vk: texture memory: %llu images in heap %u (%s)%s", (unsigned long long)n, heap, c->mem_props.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "video memory" : "system memory", step == 2 ? ", past the budget" : ""); } static VkResult image_alloc(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocationCreateFlags own, VkImage *img, VmaAllocation *a) { static const struct { VkMemoryPropertyFlags req; VmaAllocationCreateFlags flags; } step[2] = { {VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT}, {0, 0}}; VkResult r = VK_ERROR_OUT_OF_DEVICE_MEMORY; for (int i = 0; i < 2; i++) { VmaAllocationCreateInfo ai = {.flags = step[i].flags | own, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, .requiredFlags = step[i].req}; r = vmaCreateImage(c->vma, ci, &ai, img, a, NULL); if (r == VK_SUCCESS) { note_placement(c, *a, i ? 2 : 0); // 2: past the budget return r; } } return r; } // A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: // the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load // (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that // 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom // thread keeps ahead (vk_headroom.c), within the budget: past it, the next device-local memory type that has room // (image_alloc in vma_image). Render targets keep vmaCreateImage and the driver's preference (vma_image). // VK_NULL_HANDLE when the image does not take that memory. enum { SUBALLOC_MAX = 32 << 20 }; static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocation *alloc) { VkImage img; if (vkCreateImage(c->device, ci, NULL, &img) != VK_SUCCESS) return VK_NULL_HANDLE; // an image the driver requires (not only prefers) dedicated memory for keeps it: vma_image VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; VkMemoryRequirements2 mr2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr}; VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; vkGetImageMemoryRequirements2(c->device, &ri, &mr2); VkMemoryRequirements mr = mr2.memoryRequirements; VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && vmaAllocateMemory(c->vma, &mr, &ai, alloc, NULL) == VK_SUCCESS; if (ok) note_placement(c, *alloc, 0); if (ok && vmaBindImageMemory(c->vma, *alloc, img) != VK_SUCCESS) { vmaFreeMemory(c->vma, *alloc); ok = 0; } if (!ok) { vkDestroyImage(c->device, img, NULL); img = VK_NULL_HANDLE, *alloc = NULL; } return img; } // An image made with vmaCreateImage, by image_alloc's steps. One of DEDICATED_MIN or more gets memory of its own: // freed inside one of VMA's 256 MiB blocks it left a hole the driver went on counting, with the rest of the block, // against the budget and the system's commit (0.5 to 1.8 GiB of such free space in a session at 5120x2160), until the // block emptied; its own memory goes back when it does. static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *ik, VkImage *img, VmaAllocation *alloc) { const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, ik); VmaAllocationCreateFlags own = (known ? known->size : image_size(ci)) >= DEDICATED_MIN ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0; return image_alloc(c, ci, own, img, alloc); } // e has memory of its own (e->alloc): its size, and the block of VMA's it lies in, which the block table reads // without asking VMA for every entry (compact_table). An allocation keeps its place for as long as it lives. static void own_memory(VkCtx *c, Tex *e) { VmaAllocationInfo2 own; vmaGetAllocationInfo2(c->vma, e->alloc, &own); e->bytes = own.allocationInfo.size; e->blk_mem = (uint64_t)(uintptr_t)own.allocationInfo.deviceMemory, e->blk_size = own.blockSize; e->blk_type = (uint8_t)own.allocationInfo.memoryType, e->blk_own = own.dedicatedMemory != 0; } // defer_init: the caller records the image's first transition (init_image or alias_copy). // ---------------------------------------------------------------- overwritten pool memory (NX2NAVI_TEX_CLOBBER) // The game places transient render targets over each other in its memory pools, and the replay binds their images // into the same memory (bind_pool_memory). An image of another format or size that renders over an image's memory // leaves it holding the other image's texels in the GPU's own tiled and compressed layout, which reads back as // unrelated numbers. Ocarina's volumetric fog reads its 360x202 RGBA32F fog-volume texture at the start of a frame, // before it draws it again; at the end of the previous frame the 1440x810 R11G11B10F bloom target had been drawn over // that memory, and the fog lighting took those numbers as volume distances and types: teal splotches in the fog of // Kokiri Forest, on AMD and NVIDIA. An image read while in that state gets memory of its own from then on // (private_keys), so it keeps what it last drew; the read that found it takes an empty image once. // NX2NAVI_TEX_CLOBBER=0 keeps such images in pool memory. static Tex **rts; // pool-bound images the GPU wrote, sorted by address static int nrts, crts; static uint64_t max_rt; // largest storage size in rts[] static nvn_hmap private_keys = {.klen = sizeof(TexKey)}; static int clobber_fix = -1; static volatile int32_t n_unaliased; static uint64_t n_unaliased_period, n_unalias_kept; // since the last perf line: given own memory, left in pool memory static int rts_lower(uint64_t a) { int lo = 0, hi = nrts; while (lo < hi) { int m = (lo + hi) / 2; if (rts[m]->pub.addr < a) lo = m + 1; else hi = m; } return lo; } static void rt_add(Tex *e) { if (e->in_rt) return; if (nrts == crts) { crts = crts ? 2 * crts : 256; if (!(rts = realloc(rts, sizeof *rts * (size_t)crts))) rt_fatal("vk: out of memory for the render target list"); } int i = rts_lower(e->pub.addr); memmove(rts + i + 1, rts + i, sizeof *rts * (size_t)(nrts - i)); rts[i] = e; nrts++; if (e->t.L.size > max_rt) max_rt = e->t.L.size; e->in_rt = 1; } static void rt_remove(Tex *e) { if (!e->in_rt) return; for (int i = rts_lower(e->pub.addr); i < nrts && rts[i]->pub.addr == e->pub.addr; i++) if (rts[i] == e) { memmove(rts + i, rts + i + 1, sizeof *rts * (size_t)(nrts - i - 1)); nrts--; break; } e->in_rt = 0; } static int alias_compatible(const Tex *e, const Tex *src, const char **why); // e, bound into pool memory, was just written by the GPU: its own content is current, and the pool-bound images of // another layout whose storage overlaps it no longer hold theirs. static void mark_clobbered(Tex *e) { if (clobber_fix < 0) clobber_fix = cfg_bool("NX2NAVI_TEX_CLOBBER"); e->clobbered = 0; if (!clobber_fix || !e->bound_to) return; rt_add(e); uint64_t lo = e->pub.addr, hi = lo + e->t.L.size; for (int i = rts_lower(lo > max_rt ? lo - max_rt : 0); i < nrts && rts[i]->pub.addr < hi; i++) { Tex *o = rts[i]; if (o == e || o->clobbered || o->bound_to != e->bound_to || o->pub.addr + o->t.L.size <= lo) continue; const char *why; // the same texels through both: a compatible image of the same shape (another size tiles apart) if (o->pub.image == e->pub.image || (alias_compatible(o, e, &why) && o->pub.width == e->pub.width && o->pub.height == e->pub.height && o->pub.depth == e->pub.depth)) continue; o->clobbered = 1; } } static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { static volatile int32_t n_unmapped, n_unsupported, n_failed, n_unbound, n_thick; VkCtx *c = vk_ctx(); const nvn_texdesc *d = &t->d; if (!c || d->target == NVN_TEXTURE_TARGET_BUFFER) return NULL; TexVkFormat f; if (!tex_vk_format(d->format, &f)) { log_limited(&n_unmapped, "texture format %#x has no Vulkan format: %s", d->format, f.reason); return NULL; } if (!tex_vk_format_supported(&f)) { log_limited(&n_unsupported, "texture format %#x: VkFormat %d lacks features %#x", d->format, (int)f.vk, (unsigned)f.need); return NULL; } // ASTC as BC1: a texture the CPU provides, not one that lies in pool memory, may turn BC1 at its first upload int bc1_try = f.astc_bc == 1 && tex_mirrored(t) && !nvn_gpu_only(k->addr) && vk_astc_bc1(); nvn_texlayout L; nvn_tex_layout(d, &L); uint32_t w = umax(d->width, 1), h = umax(d->height, 1), depth = 1, layers = 1; VkImageType type = VK_IMAGE_TYPE_2D; switch (d->target) { case NVN_TEXTURE_TARGET_1D: type = VK_IMAGE_TYPE_1D; h = 1; break; case NVN_TEXTURE_TARGET_1D_ARRAY: type = VK_IMAGE_TYPE_1D; h = 1; layers = L.layers; break; case NVN_TEXTURE_TARGET_3D: type = VK_IMAGE_TYPE_3D; depth = umax(d->depth, 1); break; case NVN_TEXTURE_TARGET_2D_ARRAY: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY: case NVN_TEXTURE_TARGET_CUBEMAP: case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: layers = L.layers; break; default: break; } int scale = shadow_scale_of(d, &f, d->samples > 1 ? (uint32_t)d->samples : 1, tex_mirrored(t)); w *= (uint32_t)scale, h *= (uint32_t)scale; uint32_t levels = umax(d->levels, 1), most = 1; for (uint32_t s = umax(umax(w, h), depth); s > 1; s >>= 1) most++; if (levels > most) levels = most; uint32_t samples = d->samples > 1 ? (uint32_t)d->samples : 1; if (samples > 1) levels = 1; uint32_t skip = detail_skip(t, k, &f, w, h, levels, samples); // texture detail: the image starts at this level if (skip) w = mip(w, skip), h = mip(h, skip), levels -= skip; VkFormatFeatureFlags ff = features_of(f.vk); VkImageUsageFlags usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; int ds = f.depth || f.stencil; if (ds && (ff & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT; if (!ds && (ff & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; if ((d->flags & NVN_TEXTURE_FLAGS_IMAGE) && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1) usage |= VK_IMAGE_USAGE_STORAGE_BIT; // NVN2 titles (API 200+) number the texture flags differently: Ocarina's compute targets (the title movie's RGBA8 // frames) carry 0x1, NVN1's DISPLAY. Every single-sample colour texture whose format allows it can be a storage // image there, so a compute write into one is never dropped. if (rt_nvn_api_version[0] >= 200 && !ds && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1) usage |= VK_IMAGE_USAGE_STORAGE_BIT; // contact shadows (vk_contact.c) darken the game's B10G11R11 scene colour in place from a compute shader if (f.vk == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) && samples == 1 && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && w >= 640 && vk_contact_strength()) usage |= VK_IMAGE_USAGE_STORAGE_BIT; VkImageCreateFlags flags = ds ? 0 : VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; // format-reinterpreting views // Cube compatible only for cube targets: the replay makes cube views of nothing else (vk_exec_draw.c get_view checks // the image's own view type). The flag on every square array of 6+ layers changed how AMD's Windows driver lays such // an image out, and terrain material arrays sampled through it came out in rainbow blocks that moved with the mip // level (seen on an RX 580 and a 7800 XT). if (type == VK_IMAGE_TYPE_2D && layers >= 6 && w == h && samples == 1 && (d->target == NVN_TEXTURE_TARGET_CUBEMAP || d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY)) flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT; // NVN renders into a 3D texture slice by slice (a view's min layer is the depth plane, nvn_mtl.m), e.g. the // grading LUT of the menu background blur. Vulkan needs a 2D view of the slice for that. if (type == VK_IMAGE_TYPE_3D && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT; VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = flags, .imageType = type, .format = f.vk, .extent = {w, h, depth}, .mipLevels = levels, .arrayLayers = layers, .samples = (VkSampleCountFlagBits)samples, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = usage, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; int bw, bh, block_views = 0; // NX2NAVI_BLOCK_VIEWS_3D (default 0): a 3D BC image also gets block views. On RDNA2 cards (driver 26.10.1, issue #5) // the 32x32x32 BC4 volume the game creates at the logo, made BLOCK_TEXEL_VIEW_COMPATIBLE and 2D_ARRAY_COMPATIBLE, // took a GPU write just past the end of its memory: device lost at the first submit after it when the image ended // its memory block. Draws into a 3D BC image go through a stand-in instead (vk_exec.c bc_target). static int bv3d = -1; if (bv3d < 0) { bv3d = rt_knob("BLOCK_VIEWS_3D", "0"); rt_log("vk: 3D BC textures %s (NX2NAVI_BLOCK_VIEWS_3D=%d)", bv3d ? "get block views (renderable)" : "have no block views", bv3d); } if (!ds && samples == 1 && f.vk == f.ideal && nvn_fmt_bpb(d->format, &bw, &bh) && bw == 4 && bh == 4 && (type != VK_IMAGE_TYPE_3D || bv3d)) { VkImageCreateInfo bc = ci; // BC formats: renderable through views of their blocks (block_views_ok) bc.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; bc.flags |= VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; if (type == VK_IMAGE_TYPE_3D) bc.flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT; VkImageUsageFlags hidden = 0; // creation usage the replay never uses (views restrict theirs, vk_exec.c) int ok = block_views_ok(&bc); if (ok && type == VK_IMAGE_TYPE_3D && !image_size(&bc)) { // STORAGE usage keeps RADV on GFX9 from the thin layout (image_size), so agl's BC4 encoder still renders // into the volume; else the image is made without block views and its encoder passes are left out bc.usage |= VK_IMAGE_USAGE_STORAGE_BIT, hidden = VK_IMAGE_USAGE_STORAGE_BIT; ok = block_views_ok(&bc) && image_size(&bc); log_limited(&n_thick, "texture fmt %#x %ux%ux%u: the device cannot lay out a 3D image with block views; %s", d->format, w, h, depth, ok ? "made with storage usage (a thick layout)" : "made without them"); } if (ok) ci = bc, usage = bc.usage & ~hidden, flags = bc.flags, block_views = 1; } // Feedback loops (vk.h VkCtx.feedback): every attachable image may be sampled by a draw of a pass that renders it, // so it gets the usage now; adding it when a loop shows up would mean recreating the image (that lost the // grass tile maps once). int fb_usage = c->feedback == VK_FEEDBACK_LAYOUT && !block_views && (usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)); if (fb_usage) ci.usage = usage |= VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; Tex *e = calloc(1, sizeof *e); if (!e) rt_fatal("vk: out of memory for a texture"); e->key = *k; e->t = *t; e->f = f; e->pub.addr = k->addr; img_key_of(&ci, &e->ik); VkImage img = VK_NULL_HANDLE; // storage in a physical or virtual pool: alias the pool memory (section 3.2), unless an image of another layout // overwrote it there before it was read (private_keys) if (scale == 1 && nvn_gpu_only(k->addr) && !nvn_hm_get(&private_keys, k)) { const VkMemoryRequirements *q = nvn_hm_get(&img_reqs, &e->ik); VkMemSpan sp; if (q && !pool_fits(c, e, q, L.size, &sp)) { // it would not bind there (img_reqs) n_bind_skips++; log_limited(&n_unbound, "texture %#llx (%ux%u fmt %#x) cannot alias its pool memory; own memory instead", (unsigned long long)k->addr, w, h, d->format); } else if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS && !bind_pool_memory(c, e, img, L.size)) { vkDestroyImage(c->device, img, NULL); img = VK_NULL_HANDLE; log_limited(&n_unbound, "texture %#llx (%ux%u fmt %#x) cannot alias its pool memory; own memory instead", (unsigned long long)k->addr, w, h, d->format); } } if (!img && tex_mirrored(t) && samples == 1) img = suballoc_image(c, &ci, &e->alloc); int gpu_only = !tex_mirrored(t); // an entry split fill is to fill gets a new image: start_at uploads into it in the upload batch (see "split fill") int to_fill = gpu_only && split_wants(t, &f) && L.size <= SPLIT_FILL_MAX; for (const Tex *p = to_fill ? nvn_hm_get(&by_addr, &k->addr) : NULL; p; p = p->next_at) if (alias_has_gpu_content(p)) { to_fill = 0; break; } // start_at copies that in instead if (!img && gpu_only && !to_fill && tex_reuse_on()) { // the image of a released entry with these parameters (reuse, above) // Not one released in this frame or the one before: Ocarina releases its render targets and registers them // again every frame, and DLSS's scene target (2560x1440 at a fixed address in Kokiri Forest) lost its image // to a moving target of the same size in between, and DLSS its history (vk_dlaa.c evaluate), several times a // second. Not one at this address either: its own descriptor comes back for it (start_at). Of the others the // one nearest in address, so a target the allocator moves keeps the image it had; an overwritten one first // (overwrite_others): nothing can ask for its content again. uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); Tex *a = NULL, *kept = NULL; uint64_t best = UINT64_MAX, best_kept = UINT64_MAX; int n = 0; // An overwritten one goes by what drew with it or sampled it (sampled_lately), whatever slot still holds or // names it. The game finalizes the texture object of a target it has moved only when it needs the object // again, and until then the object's slot holds the entry: the pool stamps such an entry as used in every // submission (pool_touch), and the slot names it (slot_names). By those two an overwritten target's image // could not go to the target that took its place. While a shadow map in Kokiri Forest changed its size in // every frame (1141x1141 to 1917x1917, a texel or two a frame, as the camera moved), every target laid out // behind it moved with it: 945 images were made in 60 frames and 17 reused, the released entries reached // 1,931 with 8.6 GiB within five seconds of the area's load at 3416x1440, and a player's card of 20 GiB was // full twice in five minutes (image creation failed, 4 fps, then the game ended). The slot of an entry whose // image went shows its stand-in, as it does once drop_overwritten took the entry. int marks = marks_tell(); for (Tex *q = nvn_hm_get(&reuse, &e->ik); q && n < REUSE_LOOK; q = q->ru_next, n++) { if (q->key.addr == k->addr) continue; if (q->rel_epoch == submit_epoch || now - q->rel_frame < 2) { n_reuse_young++; continue; } int over = q->overwritten && marks; // still drawn with or sampled: its content is asked for if (over ? sampled_lately(q, now, over_share ? REUSE_UNUSED_FULL : REUSE_UNUSED) : used_lately(q, now, REUSE_UNUSED)) { n_reuse_used++; continue; } uint64_t dist = q->key.addr > k->addr ? q->key.addr - k->addr : k->addr - q->key.addr; // not one a current slot still names (slot_names), asked only of a candidate that would be the best if (q->overwritten) { if (dist >= best) continue; if (!over && slot_names(q)) { n_reuse_slot++; continue; } best = dist, a = q; } else if (dist < best_kept) { if (slot_names(q)) { n_reuse_slot++; continue; } best_kept = dist, kept = q; } } if (!a) a = kept; if (a) { img = a->pub.image, e->alloc = a->alloc; vk_exec_image_gone(img); // the views and records of the texture it was a->pub.image = VK_NULL_HANDLE, a->alloc = NULL; drop_tex(a, DROP_REUSED); e->reused = 1, e->og_reimg = 1; n_reused++; } } if (!img) { if (gpu_only) n_reuse_new++; VkResult r = vma_image(c, &ci, &e->ik, &img, &e->alloc); if (r != VK_SUCCESS) { log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r, d->format, w, h, depth, d->target, levels, layers); free(e); return NULL; } } if (e->alloc) own_memory(c, e); e->bc1_try = (uint8_t)bc1_try; e->reusable = e->alloc && gpu_only; e->gw_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (e->reusable && nvn_gpu_only(k->addr)) gwin_add(e); // A new image's handle can be one a destroyed image had. Views are purged with the image (destroy_tex), so none // should be cached under it: one that is was made from a destroyed texture's record after its purge, and would // reach this image's memory through the dead one's. Dropped here, and counted in the log. if (!e->reused) { static volatile int32_t n_stale; int stale = vk_exec_image_gone(img); if (stale) log_limited(&n_stale, "new image %#llx had %d cached views of a destroyed image with its handle; dropped", (unsigned long long)(uintptr_t)img, stale); } VkTexImage *p = &e->pub; p->image = img; p->format = f.vk; p->type = type; p->usage = usage; p->aspect = aspect_of(&f); p->width = w, p->height = h, p->depth = depth, p->levels = levels, p->layers = layers, p->samples = samples; p->view_type = type == VK_IMAGE_TYPE_1D ? (d->target == NVN_TEXTURE_TARGET_1D_ARRAY ? VK_IMAGE_VIEW_TYPE_1D_ARRAY : VK_IMAGE_VIEW_TYPE_1D) : type == VK_IMAGE_TYPE_3D ? VK_IMAGE_VIEW_TYPE_3D : d->target == NVN_TEXTURE_TARGET_CUBEMAP && (flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) ? VK_IMAGE_VIEW_TYPE_CUBE : d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY && (flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) ? VK_IMAGE_VIEW_TYPE_CUBE_ARRAY : layers > 1 || d->target == NVN_TEXTURE_TARGET_2D_ARRAY || d->target == NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D; p->alpha_one = f.alpha_one; memcpy(p->chan, f.chan, sizeof p->chan); p->swz[0] = (uint8_t)d->swizzle_r, p->swz[1] = (uint8_t)d->swizzle_g; p->swz[2] = (uint8_t)d->swizzle_b, p->swz[3] = (uint8_t)d->swizzle_a; if (!(p->swz[0] | p->swz[1] | p->swz[2] | p->swz[3])) // descriptors built without nvnTextureBuilderSetDefaults p->swz[0] = NVN_TEXTURE_SWIZZLE_R, p->swz[1] = NVN_TEXTURE_SWIZZLE_G, p->swz[2] = NVN_TEXTURE_SWIZZLE_B, p->swz[3] = NVN_TEXTURE_SWIZZLE_A; p->mirrored = (uint8_t)tex_mirrored(t); p->bound = e->bound_to != NULL; p->block_views = (uint8_t)block_views; p->fb_usage = (uint8_t)fb_usage; p->scale = (uint8_t)scale; p->skip = (uint8_t)skip; if (!defer_init) init_new(e); return e; } static int owned_lower(uint64_t a) { int lo = 0, hi = nowned; while (lo < hi) { int m = (lo + hi) / 2; if (owned[m]->pub.addr < a) lo = m + 1; else hi = m; } return lo; } static void own_add(Tex *e) { if (e->listed) return; plat_mutex_lock(&owned_lock); if (nowned == cowned) { cowned = cowned ? 2 * cowned : 256; if (!(owned = realloc(owned, sizeof *owned * (size_t)cowned))) rt_fatal("vk: out of memory for the owned-texture list"); } int i = owned_lower(e->pub.addr); memmove(owned + i + 1, owned + i, sizeof *owned * (size_t)(nowned - i)); owned[i] = e; __atomic_store_n(&nowned, nowned + 1, __ATOMIC_RELEASE); if (e->t.L.size > max_owned) max_owned = e->t.L.size; e->listed = 1; plat_mutex_unlock(&owned_lock); } static void own_remove(Tex *e) { if (!e->listed) return; plat_mutex_lock(&owned_lock); for (int i = owned_lower(e->pub.addr); i < nowned && owned[i]->pub.addr == e->pub.addr; i++) if (owned[i] == e) { memmove(owned + i, owned + i + 1, sizeof *owned * (size_t)(nowned - i - 1)); __atomic_store_n(&nowned, nowned - 1, __ATOMIC_RELEASE); break; } e->listed = 0; plat_mutex_unlock(&owned_lock); } static uint64_t rel_bytes; // own memory of the released entries (Tex.bytes) static void ru_link(Tex *e) { Tex *head = nvn_hm_get(&reuse, &e->ik); e->ru_prev = NULL, e->ru_next = head; if (head) head->ru_prev = e; nvn_hm_put(&reuse, &e->ik, e); e->ru_on = 1; } static void ru_unlink(Tex *e) { if (!e->ru_on) return; if (e->ru_prev) e->ru_prev->ru_next = e->ru_next; else if (e->ru_next) nvn_hm_put(&reuse, &e->ik, e->ru_next); else nvn_hm_del(&reuse, &e->ik); if (e->ru_next) e->ru_next->ru_prev = e->ru_prev; e->ru_prev = e->ru_next = NULL; e->ru_on = 0; } static void rel_append(Tex *e) { e->rel_next = NULL; e->rel_prev = rel_tail; if (rel_tail) rel_tail->rel_next = e; else rel_head = e; rel_tail = e; rel_bytes += e->bytes; if (e->reusable && tex_reuse_on()) ru_link(e); } static Tex *sweep_at; // where sweep_released goes on static Tex *aged_at; // where drop_aged goes on static Tex *press_at; // where sweep_pressure's walk of the released entries goes on static void rel_unlink(Tex *e) { if (sweep_at == e) sweep_at = e->rel_next; if (aged_at == e) aged_at = e->rel_next; if (press_at == e) press_at = e->rel_next; ru_unlink(e); rel_bytes -= e->bytes; if (e->rel_prev) e->rel_prev->rel_next = e->rel_next; else rel_head = e->rel_next; if (e->rel_next) e->rel_next->rel_prev = e->rel_prev; else rel_tail = e->rel_prev; e->rel_prev = e->rel_next = NULL; } // The entries the bindless texture pool's slots resolved to, for a submission's freshness check (make_fresh_snap once // per entry), which looked at every slot instead: 33,000 slots, 1.5 ms of the replay thread a submission in Kokiri // Forest. vk_exec_draw.c pool_fill notes a slot's entry after it resolved the slot; only entries whose content the CPU // provides are listed (the check does nothing for the others; one the GPU took over, gpu_owned, never uploads again // and leaves), and an entry stays listed until it is destroyed: the list holds every entry a slot resolves to, and // some that no slot does any more. Each keeps its storage range and the page generation of its last check here, so // that the check of an entry whose 2 MiB regions took no write since (vk_mem_unchanged_locked, one shared lock for the // whole list) touches neither the entry nor the lock: 5,600 entries took 0.77 ms a submission one by one. Texture lock // held. // rel is the entry's released state (mark_released, entry_of's take-back): the check does nothing for a released // entry, and with the state here it does not touch the entry to find that out. On a card with room the sweep keeps up // to 8,192 released entries, they stay listed, and since stays as it was at their release, so each one whose regions // took a write later missed the region check at every submission from then on: 1,000 to 6,300 entries a check in // Kakariko and Castle Town, a third to a half of the list. typedef struct { Tex *e; uint64_t addr, size, since; uint8_t ok, rel; } PEnt; // ok: uploaded as of since static PEnt *pent; static int npent, cpent, *pq, cpq; static uint64_t n_pool_checks, n_pool_skips, n_pool_rel; // The region check's misses (vk_mem_unchanged_why_locked): not uploaded, a region without this epoch's snapshot yet, one // untracked, one written since, nothing to check; and the snapshot queries its own pass made (NX2NAVI_POOL_PRESNAP). static uint64_t n_pool_miss[5], n_presnap_q; static void make_fresh_snap(Tex *e); static void pool_unlist(Tex *e) { if (!e->pidx) return; int i = e->pidx - 1; pent[i] = pent[--npent]; pent[i].e->pidx = i + 1; e->pidx = 0; } // The entries the pool's slots hold: for each, how many slots' last fill resolved to it (vk_exec_draw.c pool_hold; a fill // that shared another slot's lookup counts for the entry that lookup found). The walk over every slot looked each slot // up in each submission, which left its entry's use stamp (touch): the idle clock reads it, and start_at's alias rule // reads the frame of an entry with GPU content. The check by list is no lookup, so it leaves the stamps from these // counts (pool_touch): in each submission for the GPU-owned entries, in turn for the others, each once in // POOL_TOUCH_DIV submissions (IDLE_ROUNDS is 240 of them). An entry keeps its place until it is destroyed and no slot // holds it any more: a destroyed one is refilled out of its slots by the bump of its bucket. #define POOL_TOUCH_DIV 64 typedef struct { Tex *e; uint32_t refs; int next; uint8_t gown; } PRef; // e NULL: destroyed, or free (refs 0, next) static PRef *pref; static int npref, cpref, pref_free; // pref_free: 1 + the first free place static void touch(Tex *e); static uint32_t pool_ref(Tex *e) { if (e->ph) return (uint32_t)e->ph; int i; if (pref_free) { i = pref_free - 1, pref_free = pref[i].next; } else { if (npref == cpref) { int n = cpref ? cpref * 2 : 1024; PRef *q = realloc(pref, sizeof *q * (size_t)n); if (!q) rt_fatal("vk: out of memory for the texture pool's entry counts"); pref = q, cpref = n; } i = npref++; } pref[i] = (PRef){e, 0, 0, e->pub.gpu_owned}; return (uint32_t)(e->ph = i + 1); } void vk_texture_pool_hold_locked(uint32_t old, uint32_t ent) { if (ent) pref[ent - 1].refs++; if (old && !--pref[old - 1].refs && !pref[old - 1].e) pref[old - 1].next = pref_free, pref_free = (int)old; } static void pool_forget(Tex *e) { // destroy_tex if (!e->ph) return; PRef *q = &pref[e->ph - 1]; q->e = NULL; if (!q->refs) q->next = pref_free, pref_free = e->ph; e->ph = 0; } static void pool_touch(void) { static int at; for (int i = 0; i < npref; i++) if (pref[i].gown && pref[i].refs && pref[i].e) touch(pref[i].e); for (int k = (npref + POOL_TOUCH_DIV - 1) / POOL_TOUCH_DIV; k-- > 0;) { if (at >= npref) at = 0; const PRef *q = &pref[at++]; if (!q->gown && q->refs && q->e) touch(q->e); } } // e becomes GPU-owned: the list's check does nothing for it any more, and its use stamp is left in each submission. static void pool_gpu_owned(Tex *e) { pool_unlist(e); if (e->ph) pref[e->ph - 1].gown = 1; } int vk_texture_pool_note_locked(int id, uint32_t *ent) { *ent = 0; if (id < 0 || id >= ntslots || !tslots[id].used) return 0; // The caller's lookups left the slot resolved under its bucket's generation, unless it made none (a fill without a // token or a kind): an entry cached under another generation may be destroyed, so the slot is looked up first. if (tslots[id].ce && tslots[id].cgen != agen[sinfo[id] & 0xffff]) vk_texture_by_id_locked(id, NULL, NULL); Tex *e = tslots[id].ce; if (!e) return 0; if (!e->pidx && from_storage(e) && !e->pub.gpu_owned) { if (npent == cpent) { int n = cpent ? cpent * 2 : 1024; PEnt *p = realloc(pent, sizeof *p * (size_t)n); int *q = realloc(pq, sizeof *q * (size_t)n); if (!p || !q) rt_fatal("vk: out of memory for the texture pool's entry list"); pent = p, pq = q, cpent = cpq = n; } pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0, (uint8_t)e->released}; e->pidx = npent; } *ent = pool_ref(e); return 1; } // Released entries are left out: a slot resolves to none of those whose content the CPU provides (slot_resolve's stale // rule), and the game may be unmapping their storage. // NX2NAVI_POOL_PRESNAP=1 (default), 0, ab: the check takes the write-watch snapshot of an entry's regions that have // none in this submission yet before it looks at them (one exclusive hold of the generation table's lock for the list), // so an unchanged entry in such a region passes there. Without it 97 % of the 5,600 entries missed the region check for // want of a snapshot and went the long way, one entry and one lock at a time. With it 97 % pass, 3.5 snapshot queries // a submission: the check 0.50 to 0.19 ms a submission, the replay thread -0.37 ms against the control. static int pool_presnap(void) { static int mode = -1; if (mode < 0) mode = rt_knob("POOL_PRESNAP", "1"); return RT_KNOB_ON(mode); } // NX2NAVI_POOL_PRESNAP_RUNS=1 (default), 0, ab: those snapshots are taken for the whole list first, one write-watch // query per run of adjacent regions (vk_memory.c vk_mem_snap_noted_locked), instead of one per region as the list comes // to it. The log's "snapshot queries" of the pool freshness check count either. static int pool_presnap_runs(void) { static int mode = -1; if (mode < 0) mode = rt_knob("POOL_PRESNAP_RUNS", "1"); return RT_KNOB_ON(mode); } static void pool_made(uint64_t t0); // below, with the pool's creation budget // The submission being recorded checked the pool's entries by this list: for the idle clock it walked the pool // (vk_texture_new_submit). static int pool_by_list; int vk_texture_pool_fresh_locked(void) { int nq = 0, ps = pool_presnap(); uint64_t rel0 = n_pool_rel; pool_by_list = 1; if (ps) vk_mem_gen_lock(); else vk_mem_gen_lock_shared(); int by_range = ps; // each entry's regions are asked for as the list comes to it if (ps && pool_presnap_runs()) { for (int i = 0; i < npent; i++) if (pent[i].ok && !pent[i].rel) vk_mem_snap_note_locked(pent[i].addr, pent[i].size, submit_epoch); n_presnap_q += vk_mem_snap_noted_locked(submit_epoch, &by_range); } for (int i = 0; i < npent; i++) { const PEnt *p = &pent[i]; if (p->rel) { // released: nothing to check (the loop below would pass it by) n_pool_rel++; continue; } if (p->ok) { if (by_range) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch); int why = vk_mem_unchanged_why_locked(p->addr, p->size, p->since, submit_epoch); if (!why) continue; n_pool_miss[why]++; } else { n_pool_miss[0]++; } pq[nq++] = i; } if (ps) vk_mem_gen_unlock(); else vk_mem_gen_unlock_shared(); n_pool_skips += (uint64_t)(npent - nq); n_pool_skips -= n_pool_rel - rel0; for (int k = nq; k-- > 0;) { // from the end: an entry leaving swaps the last one into its place int i = pq[k]; PEnt *p = &pent[i]; Tex *e = p->e; if (e->pub.gpu_owned) { pool_unlist(e); continue; } if (e->released) continue; if (!e->uploaded || e->checked != submit_epoch) { uint64_t m0 = e->uploaded ? 0 : kern_now_ns(); // a first upload is part of making the entry make_fresh_snap(e); if (m0) pool_made(m0); e->checked = submit_epoch; n_pool_checks++; } p->since = e->wgen, p->ok = e->uploaded; } pool_touch(); return npent; } // slot_resolve's last answer was none because of its stale rule (vk_texture_slot_retry_locked) static int resolve_stale; // NX2NAVI_TRACE=perf, since the last perf line: slots a finalize marked (its own texture object's), slots at the storage // with the same parameters registered with another texture object, which a finalize leaves current // (vk_texture_release), and lookups the stale rule answered with the stand-in (slot_resolve). _hm: of them, the // terrain's streamed heightmaps (R32F 260x260), whose stand-in draws a chunk of ground at height zero. static uint64_t n_fin_own, n_fin_other, n_fin_other_hm, n_stale, n_stale_hm, n_stale_kept; static int terrain_heightmap(const tex_t *t) { return t->d.format == NVN_FORMAT_R32F && t->d.width == 260 && t->d.height == 260; } int vk_texture_slot_retry_locked(int id) { if (id < 0 || id >= ntslots || !tslots[id].used || !tslots[id].t.mem) return -1; resolve_stale = 0; if (vk_texture_by_id_locked(id, NULL, NULL)) return 1; return resolve_stale ? -1 : 0; } static void retire_remove(Tex *e); // ---- a flat texture sampled as a 3D one // Maxwell samples a texture by its own descriptor, whatever the shader's instruction says: a 2D texture under a 3D // sample gives its texel at (u, v), and the third coordinate does nothing. Vulkan has no 3D view of a 2D image, so // get_view (vk_exec_draw.c) found no view for such a binding and the draw sampled a stand-in. Such a texture now gets a // 3D image of one slice with its texels, all of its levels, made in the upload batch and copied again when the texture // was uploaded or drawn into since (n_up, wr_fence) or another layer is asked for; a 3D view of that gives the 2D // texel for any w. The copy is recorded in the upload batch, which runs before the part that samples it, so a texture // the GPU draws every frame is sampled as it was a part or a frame before. Small plain colour textures only: // FLAT3D_MAX texels, one sample, not block-compressed (BC rejects 3D on some drivers). NX2NAVI_TEX_FLAT3D=0: the // stand-in, as before. Texture lock held; NULL when the texture does not qualify or the image cannot be made. #define FLAT3D_MAX (1024u * 1024u) static volatile int32_t n_flat3d; const VkTexImage *vk_texture_flat3d_locked(const VkTexImage *img, uint32_t layer) { static int on = -1; if (on < 0) on = rt_knob("TEX_FLAT3D", "1") == 1; Tex *e = (Tex *)img; // pub is the first member if (!on || img->gone || !img->image || img->type != VK_IMAGE_TYPE_2D || img->samples > 1 || img->aspect != VK_IMAGE_ASPECT_COLOR_BIT || img->block_views || format_compressed(img->format) || (uint64_t)img->width * img->height > FLAT3D_MAX || layer >= img->layers || img->levels > 16) return NULL; VkCtx *c = vk_ctx(); VkCommandBuffer cb = VK_NULL_HANDLE; if (!e->flat3d) { VkTexImage *f = malloc(sizeof *f); if (!f) return NULL; *f = *img; VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_3D, .format = img->format, .extent = {img->width, img->height, 1}, .mipLevels = img->levels, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; if (vmaCreateImage(c->vma, &ci, &ai, &f->image, &e->flat3d_alloc, NULL) != VK_SUCCESS) { free(f); return NULL; } f->type = VK_IMAGE_TYPE_3D, f->view_type = VK_IMAGE_VIEW_TYPE_3D, f->depth = 1, f->layers = 1; f->usage = ci.usage, f->bound = 0, f->block_views = 0, f->fb_usage = 0, f->fb_learned = 0, f->gone = 0; e->flat3d = f; e->flat3d_up = ~0u; cb = vk_upload_begin(); VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_NONE, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = f->image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, 1}}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; vkCmdPipelineBarrier2(cb, &di); log_limited(&n_flat3d, "texture %#llx (%ux%u, %u levels, format %d) is sampled as a 3D texture: a one-slice 3D " "copy of it is made", (unsigned long long)img->addr, img->width, img->height, img->levels, (int)img->format); } if (e->flat3d_up != e->n_up || e->flat3d_wr != e->wr_fence || e->flat3d_layer != layer) { if (!cb) cb = vk_upload_begin(); VkImageCopy rg[16]; for (uint32_t l = 0; l < img->levels; l++) { uint32_t w = img->width >> l, h = img->height >> l; rg[l] = (VkImageCopy){.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, l, layer, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, l, 0, 1}, .extent = {w ? w : 1, h ? h : 1, 1}}; } barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); vkCmdCopyImage(cb, img->image, VK_IMAGE_LAYOUT_GENERAL, e->flat3d->image, VK_IMAGE_LAYOUT_GENERAL, img->levels, rg); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); e->flat3d_up = e->n_up, e->flat3d_wr = e->wr_fence, e->flat3d_layer = layer; } if (cb) vk_upload_end(); return e->flat3d; } // ---- origins (NX2NAVI_TRACE=perf; a run without it tests the flag and counts the drops, nothing more) // A texture drawn without its content leaves nothing in the log: no call fails and the image exists. What can still be // told afterwards is where the entry came from, so each entry keeps how it was made and how its content started // (og_*, had), and the last DROPS_N destroyed entries keep their descriptor and the reason they went. An entry made // for a descriptor that is still in that ring says which drop it follows (origin_made), and the lines about one // address list what was dropped there (others_at). The users are marks_check, which names an entry a shader sampled // while nothing had ever put content into its image, and the lines of start_at. #define DROPS_N 4096 typedef struct { uint64_t kh, addr, frame; uint32_t format, flags, w, h; uint8_t why, mirrored, released; } DropRec; static DropRec drops[DROPS_N]; static uint64_t ndrops; static uint64_t n_drop_by[DROP_N]; // entries destroyed since the last perf line, by reason static const char *const drop_names[DROP_N] = { "", "past RELEASED_MAX released entries", "past the released entries' size cap", "under commit pressure", "under VRAM pressure", "idle, under VRAM pressure", "idle, under commit pressure", "unsampled (disuse)", "overwritten at another address", "replaced by another descriptor at its address", "its image went to another texture", "its storage went", "made whole again (texture detail)", "shutdown", "released, in a block a compaction empties", "released and unused for longer than the hold", "past RELEASED_HARD released entries"}; static const char *const start_names[START_N] = { "started on its own", "filled from storage over a live entry", "content copied from a live entry", "content copied from a released entry", "started empty over a live entry", "started empty over a released entry", "content copied from the released CPU-written entry it replaced (split fill)", "filled from its storage (split fill)"}; const char *vk_texture_start_name(int start) { return start >= 0 && start < START_N ? start_names[start] : "?"; } const char *vk_texture_drop_name(int why) { return why >= 0 && why < DROP_N ? drop_names[why] : "?"; } // An entry's event for the flight recorder (flight.h flight_tex, FLIGHT_TF_*): a no-op while the recorder is off. static void flight_ev(const Tex *e, int kind, uint32_t detail, uint32_t us) { if (!flight_on) return; uint32_t fl = (e->f.depth ? FLIGHT_TF_DEPTH : 0) | (e->f.stencil ? FLIGHT_TF_STENCIL : 0) | (e->pub.mirrored ? FLIGHT_TF_MIRRORED : 0) | (e->bound_to ? FLIGHT_TF_BOUND : 0) | (e->f.astc_bw ? FLIGHT_TF_ASTC : 0) | (e->pub.gpu_owned ? FLIGHT_TF_GPU : 0); flight_tex(kind, e->key.addr, (uint32_t)e->t.d.format, e->pub.width, e->pub.height, e->pub.layers, fl, detail, us); } static uint64_t frames_now(void) { return __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); } // For the flight recorder, after an upload of e: FNV-1a over 64 probes of 64 bytes spread over its storage, and how many // of them were all zero. Two uploads of one texture with the same hash read the same bytes at the probes; a first one // with zero probes read storage the game had not filled. static void flight_bytes(const Tex *e) { if (!flight_on || !e->pub.mirrored || e->t.L.size < 64 * 64) return; if (!mem_read_hold(e->pub.addr, e->t.L.size)) return; const uint8_t *b = (const uint8_t *)(uintptr_t)e->pub.addr; uint64_t step = (e->t.L.size - 64) / 63; uint32_t h = 0x811c9dc5u, zero = 0; for (int i = 0; i < 64; i++) { const uint8_t *p = b + (uint64_t)i * step; uint8_t any = 0; for (int k = 0; k < 64; k++) h = (h ^ p[k]) * 0x01000193u, any |= p[k]; zero += !any; } mem_read_release(); flight_ev(e, FLIGHT_TEX_BYTES, h, zero); } static uint64_t key_hash(const TexKey *k) { // FNV-1a; make_key zeroes the key's padding uint64_t h = 0xcbf29ce484222325ull; const uint8_t *b = (const uint8_t *)k; for (size_t i = 0; i < sizeof *k; i++) h = (h ^ b[i]) * 0x100000001b3ull; return h; } // Drops for a low commit that come back (tex_lock held throughout). A texture dropped because the system's commit ran // low frees its memory only for as long as the game does not ask for it again; when it does, it is made and uploaded // again and holds the same memory as before. With other programs holding 49 of an 84 GB limit, the commit sat at the // low line for whole sessions: 340 to 590 released textures a second went, most were made again within seconds (178 // re-uploads a second against 21 without the drops), and the uploads inside the draws held the texture lock and the // game's threads behind it, a 35-45 ms frame several times a second, while the commit stayed where it was. // So each drop under the commit rule notes its texture (by its key, CB_SLOTS slots, the newest wins), and a texture // made again within CB_WITHIN presents counts as come back. Over each CB_PERIOD, when at least CB_MIN_DROPS went and // half or more came back, the commit drops (released entries and idle live ones) pause for CB_HOLD_MIN presents, // doubled at each such period up to CB_HOLD_MAX; a period in which fewer came back ends the doubling. Below half the // low line (commit_critical) nothing pauses: there the commit running out ends the game. #define CB_SLOTS 8192 #define CB_WITHIN PRESENTS(600) #define CB_PERIOD PRESENTS(240) #define CB_MIN_DROPS 64 #define CB_HOLD_MIN PRESENTS(600) #define CB_HOLD_MAX PRESENTS(7200) static struct { uint64_t kh, frame; } cb_slot[CB_SLOTS]; static uint64_t cb_drops, cb_backs, cb_t0, cb_hold, cb_hold_until; static uint64_t n_commit_back, n_commit_held; // since the last perf line static int commit_critical, cb_logs; static uint64_t cb_key(const Tex *e) { return key_hash(&e->key) ^ (e->key.addr * 0x9e3779b97f4a7c15ull); } static void commit_drop_note(const Tex *e) { uint64_t kh = cb_key(e); cb_slot[kh & (CB_SLOTS - 1)].kh = kh, cb_slot[kh & (CB_SLOTS - 1)].frame = frames_now() | 1; cb_drops++; } static void commit_back_note(const Tex *e) { // a new entry uint64_t kh = cb_key(e), now = frames_now(); typeof(cb_slot[0]) *s = &cb_slot[kh & (CB_SLOTS - 1)]; if (s->kh != kh || !s->frame || now - s->frame > CB_WITHIN) return; s->frame = 0; cb_backs++, n_commit_back++; } // 1 while the commit drops pause (above); judges the period that ended. static int commit_drops_held(void) { uint64_t now = frames_now(); if (now - cb_t0 >= CB_PERIOD) { if (cb_drops >= CB_MIN_DROPS && cb_backs * 2 >= cb_drops) { cb_hold = cb_hold ? (cb_hold * 2 < CB_HOLD_MAX ? cb_hold * 2 : CB_HOLD_MAX) : CB_HOLD_MIN; cb_hold_until = now + cb_hold; if (cb_logs < 12) { cb_logs++; rt_log("vk: commit low, but %llu of the %llu textures dropped for it were made again within %llu presents: " "the commit drops pause for %llu presents", (unsigned long long)cb_backs, (unsigned long long)cb_drops, (unsigned long long)CB_WITHIN, (unsigned long long)cb_hold); } } else if (cb_drops >= CB_MIN_DROPS) { cb_hold = 0; } cb_t0 = now, cb_drops = cb_backs = 0; } if (commit_critical || now >= cb_hold_until) return 0; n_commit_held++; return 1; } // destroy_tex with the reason, which every caller gives. // An entry the GPU drew into since its last write-back holds texels its storage lacks. Dropped for a reason that leaves // the storage to the game (a budget, a commit or video memory squeeze, its release, a replacement at its address), a // texture the game makes again from that storage (the drawn-once maps the fence write-back is for, moved or remade) // would start from what the storage held before the GPU drew. Such an entry goes back first when the fence write-back // would take it (wb_wanted) and the CPU has not rewritten its storage since; the commit drop pause above counts it the // same either way. Not an entry another target drew over (DROP_OVERWRITTEN: the storage is the newer one's), nor at // shutdown. static int wb_wanted(const Tex *e); static void wb_cfg_init(void); static int storage_rewritten(const Tex *e, uint64_t since, uint64_t *now); static void write_back_n(Tex **v, int n); static uint64_t n_drop_wb; // entries written back before their drop since the last perf line static void drop_tex(Tex *e, int why) { flight_ev(e, FLIGHT_TEX_DROP, (uint32_t)why, 0); if (why == DROP_COMMIT || why == DROP_IDLE_COMMIT) commit_drop_note(e); if (e->gpu_dirty && why != DROP_OVERWRITTEN && why != DROP_SHUTDOWN) { uint64_t now; wb_cfg_init(); if (storage_rewritten(e, e->gpu_gen, &now) == 1) e->gpu_dirty = 0; else if (wb_wanted(e)) write_back_n(&e, 1), n_drop_wb++; } n_drop_by[why]++; reuse_dropped(e, why); if (rt_trace & RT_TRACE_PERF) drops[ndrops++ % DROPS_N] = (DropRec){key_hash(&e->key), e->key.addr, frames_now(), (uint32_t)e->t.d.format, (uint32_t)e->t.d.flags, e->t.d.width, e->t.d.height, (uint8_t)why, e->pub.mirrored, e->released != 0}; destroy_tex(e); } // A new entry: the frame, and the drop of its descriptor's last entry when the ring still holds it. static void origin_made(Tex *e) { e->og_frame = frames_now(); commit_back_note(e); if (!(rt_trace & RT_TRACE_PERF)) return; uint64_t kh = key_hash(&e->key), n = ndrops < DROPS_N ? ndrops : DROPS_N; for (uint64_t i = 1; i <= n; i++) { // newest first const DropRec *r = &drops[(ndrops - i) % DROPS_N]; if (r->kh != kh || r->addr != e->key.addr) continue; e->og_drop = r->why, e->og_drop_frame = r->frame; return; } } // "fmt 0x25 (vk 37) 256x256x1, 9 levels, 1 layers, target 1, samples 0, flags 0x1, pool flags 0x22" static int desc_text(char *out, size_t n, const Tex *e) { const nvn_texdesc *d = &e->t.d; return snprintf(out, n, "fmt %#x (vk %d) %ux%ux%u, %u levels, %u layers, target %d, samples %d, flags %#x, pool flags %#x", d->format, (int)e->f.vk, d->width, d->height, d->depth, e->pub.levels + e->pub.skip, e->pub.layers, d->target, d->samples, (unsigned)d->flags, (unsigned)e->t.pool_flags); } // "made at frame 100 again after a drop at frame 90 (under commit pressure); started empty over a released entry; ..." static int origin_text(char *out, size_t n, const Tex *e) { int k = snprintf(out, n, "made at frame %llu", (unsigned long long)e->og_frame); if (e->og_drop && k < (int)n) k += snprintf(out + k, n - (size_t)k, " again after a drop at frame %llu (%s)", (unsigned long long)e->og_drop_frame, drop_names[e->og_drop]); if (k < (int)n) k += snprintf(out + k, n - (size_t)k, "; %s", start_names[e->og_start]); if (e->og_reimg && k < (int)n) k += snprintf(out + k, n - (size_t)k, "; with a released entry's image"); if (e->og_backs && k < (int)n) k += snprintf(out + k, n - (size_t)k, "; taken back %u times, last at frame %llu", e->og_backs, (unsigned long long)e->og_back_frame); return k < (int)n ? k : (int)n - 1; } // Whether nothing ever put content into e's image: an entry the CPU writes that was not uploaded, or one it does not // write, at an address the CPU can see, that the GPU never wrote and that got no copy, no texels and no split fill. static int no_content(const Tex *e) { if (e->pub.gpu_owned) return 0; if (e->pub.mirrored) return !e->uploaded; return !e->had && !(e->sfill && e->uploaded) && e->key.addr && !nvn_gpu_only(e->key.addr); } // Lines per address for one kind of event: up to `most` each, in a table of the first SAID_N addresses that had one // (a later address always gets its line), so a session's limit is by address, not a count of lines. #define SAID_N 4096 typedef struct { uint64_t addr[SAID_N]; uint16_t n[SAID_N]; } SaidAt; static int said_at(SaidAt *s, uint64_t addr, unsigned most) { uint32_t i = (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); for (int probe = 0; probe < 64; probe++, i = (i + 1) % SAID_N) { if (s->addr[i] && s->addr[i] != addr) continue; s->addr[i] = addr; return s->n[i] < most ? (s->n[i]++, 1) : 0; } return 1; } // The other descriptors at e's address: every entry in the cache with its state, and what the ring knows was dropped // there, newest first. skip: an entry the caller describes itself (NULL: none). static int others_at(char *out, size_t n, const Tex *e, const Tex *skip) { int k = 0, any = 0; out[0] = 0; for (const Tex *p = nvn_hm_get(&by_addr, &e->key.addr); p && k < (int)n - 1; p = p->next_at) { if (p == e || p == skip) continue; k += snprintf(out + k, n - (size_t)k, "%s[%s%s%s%s: fmt %#x flags %#x %ux%u, %s, %s]", any++ ? " " : "", p->released ? "released" : "live", p->superseded ? ", superseded" : "", p->retiring ? ", retiring" : "", p->pub.gpu_owned ? ", GPU-written" : "", p->t.d.format, (unsigned)p->t.d.flags, p->t.d.width, p->t.d.height, p->pub.mirrored ? (p->uploaded ? "CPU-written, uploaded" : "CPU-written, not uploaded") : (no_content(p) ? "not CPU-written, no content" : "not CPU-written"), start_names[p->og_start]); } uint64_t have = ndrops < DROPS_N ? ndrops : DROPS_N; int listed = 0; for (uint64_t i = 1; i <= have && listed < 4 && k < (int)n - 1; i++) { const DropRec *r = &drops[(ndrops - i) % DROPS_N]; if (r->addr != e->key.addr) continue; k += snprintf(out + k, n - (size_t)k, "%s[dropped at frame %llu (%s): fmt %#x flags %#x %ux%u, %s%s]", any++ ? " " : "", (unsigned long long)r->frame, drop_names[r->why], r->format, r->flags, r->w, r->h, r->mirrored ? "CPU-written" : "not CPU-written", r->released ? ", released then" : ", live then"); listed++; } if (!any) k = snprintf(out, n, "none"); return k < (int)n ? k : (int)n - 1; } // NX2NAVI_LOG_TEX_NEW= (diagnostics): from that presented frame on, the first LOG_NEW_MAX textures made are // logged with what was at their address and, when the same key was destroyed before, how many frames ago and from // where (destroy_tex's caller, an address the map file names: "destroy_tex at" gives the reference). For the steady // creations a still scene shows (about seven a frame in Kokiri Forest), each with its image, memory and upload. #define LOG_NEW_MAX 400 #define GONE_N 4096 static struct { uint64_t h, frame; void *ra; } gone[GONE_N]; static uint32_t gone_n; static long long log_new_at = -2; static int log_new_n; static void log_new(const Tex *e, const TexKey *k) { // tex_lock held if (log_new_at == -2) { const char *v = cfg_get("NX2NAVI_LOG_TEX_NEW"); log_new_at = v && *v ? atoll(v) : -1; } uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (log_new_at < 0 || now < (uint64_t)log_new_at || log_new_n >= LOG_NEW_MAX) return; log_new_n++; int live = 0, rel = 0; for (const Tex *p = nvn_hm_get(&by_addr, &k->addr); p; p = p->next_at) if (p != e) p->released ? rel++ : live++; uint64_t h = key_hash(k); char was[160] = "never destroyed before"; for (uint32_t i = 0; i < GONE_N && i < gone_n; i++) { uint32_t j = (gone_n - 1 - i) & (GONE_N - 1); if (gone[j].h == h) { snprintf(was, sizeof was, "the same key destroyed %llu frames ago from %p (destroy_tex at %p)", (unsigned long long)(now - gone[j].frame), gone[j].ra, (void *)destroy_tex); break; } } rt_log("vk: new texture frame %llu: %#llx %ux%ux%u nvn fmt %#x target %d levels %u%s%s; at its address %d live, %d " "released; %s", (unsigned long long)now, (unsigned long long)k->addr, e->pub.width, e->pub.height, e->pub.depth, k->d.format, k->d.target, e->pub.levels, e->pub.mirrored ? ", mirrored" : "", e->alloc ? ", own memory" : "", live, rel, was); } __attribute__((noinline)) static void destroy_tex(Tex *e) { if (log_new_at >= 0) { // NX2NAVI_LOG_TEX_NEW: who destroyed which key, for the log of new textures uint32_t j = gone_n++ & (GONE_N - 1); gone[j].h = key_hash(&e->key), gone[j].frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); gone[j].ra = __builtin_return_address(0); } if (e->retiring) retire_remove(e); if (e->compacting) compact_forget(e); pool_unlist(e); pool_forget(e); own_remove(e); gwin_remove(e); rt_remove(e); n_site[0]++; tex_gen_bump(e->key.addr); nvn_hm_del(&by_key, &e->key); Tex *head = nvn_hm_get(&by_addr, &e->key.addr); if (head == e) { if (e->next_at) nvn_hm_put(&by_addr, &e->key.addr, e->next_at); else nvn_hm_del(&by_addr, &e->key.addr); } else { for (Tex *p = head; p; p = p->next_at) if (p->next_at == e) { p->next_at = e->next_at; break; } } if (e->released) nreleased--, rel_unlink(e); if (e->pub.image) vk_exec_image_gone(e->pub.image); // none when another texture took it (create_tex, reuse) vk_defer_image(e->pub.image, e->alloc); if (e->flat3d) { // its 3D copy goes the same way vk_exec_image_gone(e->flat3d->image); vk_defer_image(e->flat3d->image, e->flat3d_alloc); e->flat3d->gone = 1; vk_defer_free(e->flat3d); } free(e->snap); // The record goes once the GPU is past the submission being recorded, as its image: the replay keeps &e->pub as a // render target for the rest of a command buffer (vk_exec.c resolve_target) and DLSS for the frame, and this runs on // whichever thread released the texture. Freed here, a pass that began again read the freed record, cached a view of // the dead image (vk_exec_target_view), and a later image given the same handle drew through it into freed memory. e->pub.gone = 1; vk_defer_free(e); } // Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries one // goes at each release (it comes back on demand, a mirrored one re-uploaded from its storage). NX2navi // finalizes and initializes transient textures every frame, so a released entry is mostly taken back within frames. // Dropping all of them at once when the count passed 4096 destroyed thousands of images in one frame: 333-422 ms // frames every 40-50 s on a GTX 1050 Ti, then the textures still in use were created and uploaded again. // // The one that goes is the oldest release that nothing sampled or drew into lately (samp_frame) or whose content is // gone (overwritten), not the oldest release as such. A released entry is not dead to the game (drop_stale, below, has the // reasons): a descriptor registered for it still resolves to it. Ocarina reads its scene picture in every frame through // a second descriptor at the picture's address (RGB9E5, 1920x1080, the input of its own upscaler) whose entry was // released once, a few seconds into the area, and was looked up from then on: it headed the list for the rest of // the session. One frame at the limit, as it was: // 1. the count of released entries passes RELEASED_MAX at a release, and the head of the list, that entry, is // destroyed although the frame before sampled it; // 2. later in the frame the game's descriptor is looked up, finds no entry, and one is made again; its content is // copied from the released scene target at its address (start_at); // 3. the frame comes out with the 3D scene black and the interface intact; the next one is whole again. Why the // frame with the remade entry is black was not traced. // The count reaches the limit where released entries pile up: with the game's own upscaler in use (upscaler = fsr) // its render targets lie at new addresses in every frame, 2 to 13 new entries a frame, and the count went from 540 // to 2,040 within 2,400 frames in Hyrule Field. Measured there (RTX 5090, 2026-11-11, Link running, 13,630 frames // kept of each run): 8 black frames in 5 of 7 runs. Where the remake was logged (3 runs, 4 black frames) each is the // frame after it, and where the drop was logged (2 runs) it is this sweep taking that entry as the count first // reaches the limit; in one of the two runs without a black frame that came after the frames kept. With the rule // below a run of 18,730 frames dropped 834 released entries past the limit from frame 11,600 on and showed no black // frame. A session with DLSS as the upscaler had the same black frame at Lon Lon Ranch while its log showed 2,032 to // 2,047 released entries, kept there by a low system commit; that it is the same drop is read from those counts, not // logged. // "Lately" is by sampling, not by lookups. The first form of this rule spared every entry a lookup still reached // (idle), and the texture pool's refresh looks up every registered slot in each submission that draws through the // pool: an entry whose descriptor the game merely leaves registered counts as looked up for good. With the game's own // upscaler in use its render targets move to new addresses in every frame and each leaves such an entry behind, 3 to // 7 a frame: over a town all 2,049 released entries were "looked up", the sweep fell back to the oldest lookup, which // among entries all stamped in the same submission is no order at all, and took entries still drawn: 200 to 450 a // window of 60 frames, with buildings black or wrongly lit for single frames. samp_frame is stamped when a marked // shader samples the entry's slot (vk_pool_marks_read) and by every lookup that is not the pool refresh (touch: a // draw's fixed binding, a render target, a copy), so a target the game draws into or reads is recent by it and one // it only keeps registered is not. // With this rule a player's session of 30,000 frames through eight areas kept the list at its limit, dropped // 50,182 entries past it, and showed none of those frames (2026-11-11, a hand test; no scripted run was made). // When every released entry was sampled within SWEEP_UNSAMPLED presents, the one sampled longest ago goes. // // That rule holds from RELEASED_HARD entries on. Between RELEASED_MAX and RELEASED_HARD only an entry that loses // nothing goes: one that comes back intact or an overwritten one, released STALE_FRAMES presents ago or earlier (a // younger one is mostly taken back within frames). An entry whose content exists only in its image can be used again // long after anything sampled it. In Kokiri Forest the count stays at RELEASED_MAX, and two render targets at one // address (a 40x24x64 volume and a 2240x1260 picture), released at the area's load and last sampled within seconds // of it, went as unsampled about two minutes later; the game used them again after that, they were made again empty, // and one frame came out with the scene black. On an 85 s route there (Link running, six runs a batch): a black // frame in 10 of 24 runs with the oldest release going; in 1 of 12 with the unsampled one going (the two targets went // in 7 of those runs and were made again empty in 4; the black frame is one in which the sweep took a row of render // targets); in 0 of 18 with such entries kept. Kept, the count reached 6,200 in the runs that pass the busiest part // of the route early and 2,300 to 2,500 in the others; frame times and the peak of the device memory in use were the // same in all three forms, the device memory at the end of a run 0.3 to 0.5 GiB higher with 6,200 kept. // A kept entry bound into pool memory owns no memory and goes with its pool (vk_texture_range_gone); the memory of // the others is bounded by NX2NAVI_TEX_RELEASED_MB (drop_stale). // Each call looks at up to SWEEP_LOOK entries from where the last one stopped (sweep_at; rel_unlink moves it off an // entry that leaves): with thousands of entries that stay, a walk from the head at every release would cross them all. // RELEASED_MAX is the floor of that limit; released_max raises it by the card (NX2NAVI_TEX_RELEASED_MAX sets it). #define RELEASED_MAX 2048 #define RELEASED_MAX_TOP 8192 #define RELEASED_HARD 16384 #define SWEEP_LOOK 64 #define SWEEP_UNSAMPLED PRESENTS(120) static int comes_back_intact(const Tex *e); static uint64_t stale_after(const Tex *e); static void sweep_hard(void); // The limit of released entries past which the sweep drops those that lose nothing: one per 2 MiB of the video memory // budget, from RELEASED_MAX to RELEASED_MAX_TOP. In Castle Town market on a 16 GB card (RTX 5060 Ti) with 2048 the // sweep dropped 412 entries in a session and a few textures blinked black for a moment; with 8192 (and // NX2NAVI_TEX_RELEASED_MB = 4096) it dropped none, 6,863 were kept and none blinked. Their memory stays bounded by // NX2NAVI_TEX_RELEASED_MB and the VRAM pressure. Read once the budget is known. static int released_max(void) { static int max; if (max) return max; const char *s = cfg_get("NX2NAVI_TEX_RELEASED_MAX"); if (s && *s) { long long n = atoll(s); max = n < 1 ? 1 : n > RELEASED_HARD ? RELEASED_HARD : (int)n; } else { uint64_t vused = 0, vbudget = 0; int exact = 0; if (!vk_mem_vram(&vused, &vbudget, &exact) || !vbudget) return RELEASED_MAX; long long n = (long long)(vbudget >> 21); max = n < RELEASED_MAX ? RELEASED_MAX : n > RELEASED_MAX_TOP ? RELEASED_MAX_TOP : (int)n; } rt_log("vk: released textures: the sweep keeps up to %d entries (NX2NAVI_TEX_RELEASED_MAX)", max); return max; } static void sweep_released(void) { int lim = released_max(); if (nreleased <= lim) return; { uint64_t now = frames_now(); const uint64_t stale = PRESENTS(120); // STALE_FRAMES, defined below Tex *e = sweep_at ? sweep_at : rel_head; for (int n = 0; n < SWEEP_LOOK && nreleased > lim; n++) { if (!e || now - e->rel_frame < stale) { e = rel_head; break; } // the rest is younger: from the oldest again Tex *next = e->rel_next; // an entry the game took back before waits out GAP_TIMES the longest of those releases (stale_after) // nor one a draw or a shader used since (used_lately): a finalized slot may still show it (idle, below) if ((comes_back_intact(e) || e->overwritten) && now - e->rel_frame >= stale_after(e) && !used_lately(e, now, stale)) drop_tex(e, DROP_MAX); e = next; } sweep_at = e; } if (nreleased > RELEASED_HARD) sweep_hard(); } // Past RELEASED_HARD the count is a hard bound: entries go whatever they cost. The rule was the first entry in release // order that was overwritten or that nothing sampled for SWEEP_UNSAMPLED presents, of whatever kind, render targets // included, whose drop the comment above ties to a black frame; and each drop walked the list from its head again. // Most entries there cost little to make again: at the end of a session of 2026-11-13, when the count sat at 16,384 // for a minute, 12,811 of 15,734 released entries owned no memory. So the entries now go cheapest first, by the // classes of vk_tex_release.h trl_hard_class: overwritten ones; ones that come back intact, released longer ago than // their stale_after and not looked up lately; other ones that come back intact; ones whose content only the GPU made; // the last three only when nothing sampled them for SWEEP_UNSAMPLED presents; while every entry was sampled lately, the // one sampled longest ago, as before. Within a class in release order. One walk of the list takes a batch down to // RELEASED_HARD - HARD_SLACK entries, so the next HARD_SLACK releases need no walk. NX2NAVI_TEX_HARD_ORDER=0: the // rule before. #define HARD_SLACK 256 #define HARD_BATCH 1024 static uint64_t n_hard_by[TRL_HC_N]; // entries dropped past RELEASED_HARD since the last perf line, by class // Each call of sweep_hard is timed: it walks every released entry and drops a batch under the texture lock, on the // thread that releases (a guest thread's finalize, the replay's overwrite_others), and nothing measured what a batch // costs. A call of HARD_LOG_NS or more gets a line (16 at most); the texture origins perf line sums them. #define HARD_LOG_NS 2000000ull static uint64_t n_hard_calls, ns_hard, ns_hard_max; // since the last perf line static int hard_order(void) { static int on = -1; if (on < 0) on = rt_knob("TEX_HARD_ORDER", "1") == 1; return on; } static int e_hard_class(const Tex *e, uint64_t now, uint64_t stale) { int intact = comes_back_intact(e); return trl_hard_class(e->overwritten, now - e->samp_frame < SWEEP_UNSAMPLED, intact, intact && now - e->rel_frame >= stale_after(e) && !used_lately(e, now, stale)); } static void hard_lru_drop(Tex *lru) { static volatile int32_t n_all_used; log_limited(&n_all_used, "all %d released textures were sampled or drawn into lately: the one longest ago goes " "(%#llx fmt %#x %ux%u, last in frame %llu)", nreleased, (unsigned long long)lru->key.addr, lru->t.d.format, lru->t.d.width, lru->t.d.height, (unsigned long long)lru->samp_frame); n_hard_by[TRL_HC_LATELY]++; drop_tex(lru, DROP_HARD); } static void sweep_hard(void) { // tex_lock held static Tex **walk; static uint8_t *cls; static int *out, cap; const uint64_t stale = PRESENTS(120); // STALE_FRAMES const uint64_t t0 = plat_now_ns(), drops0 = n_drop_by[DROP_HARD], wb0 = n_drop_wb; uint64_t walked = 0; while (nreleased > RELEASED_HARD && rel_head) { uint64_t now = frames_now(); Tex *lru = rel_head; if (nreleased > cap && hard_order()) { // room for the whole list; without it the rule before int want = nreleased + HARD_BATCH; Tex **w = realloc(walk, (size_t)want * sizeof *w); if (w) walk = w; uint8_t *c = w ? realloc(cls, (size_t)want) : NULL; if (c) cls = c; int *o = c ? realloc(out, (size_t)want * sizeof *o) : NULL; if (o) out = o, cap = want; } if (!hard_order() || nreleased > cap) { Tex *gone = NULL; for (Tex *e = rel_head; e && !gone; e = e->rel_next) { // in release order walked++; if (e->overwritten || now - e->samp_frame >= SWEEP_UNSAMPLED) gone = e; else if (e->samp_frame < lru->samp_frame) lru = e; } if (!gone) hard_lru_drop(lru); else n_hard_by[e_hard_class(gone, now, stale)]++, drop_tex(gone, DROP_HARD); continue; } int n = 0; for (Tex *e = rel_head; e && n < cap; e = e->rel_next) { // in release order walk[n] = e, cls[n] = (uint8_t)e_hard_class(e, now, stale), n++; if (e->samp_frame < lru->samp_frame) lru = e; } walked += (uint64_t)n; int need = nreleased - (RELEASED_HARD - HARD_SLACK); int k = trl_hard_pick(cls, n, need < HARD_BATCH ? need : HARD_BATCH, out); // The picks the GPU drew into go back to their storage first, in one upload batch with one wait // (write_back_n), as drop_tex would write each of them back for DROP_HARD with a GPU wait of its own: up to // HARD_BATCH waits on the thread that releases, the replay's overwrite_others included. write_back_n clears // gpu_dirty, so drop_tex then finds nothing to write back; the same entries go, in the same order. static Tex *wbv[HARD_BATCH]; int m = 0; wb_cfg_init(); for (int i = 0; i < k; i++) { Tex *e = walk[out[i]]; uint64_t t; if (!e->gpu_dirty) continue; if (storage_rewritten(e, e->gpu_gen, &t) == 1) e->gpu_dirty = 0; else if (wb_wanted(e)) wbv[m++] = e; } if (m) write_back_n(wbv, m), n_drop_wb += (uint64_t)m; for (int i = 0; i < k; i++) n_hard_by[cls[out[i]]]++, drop_tex(walk[out[i]], DROP_HARD); if (!k) hard_lru_drop(lru); } uint64_t d = plat_now_ns() - t0; n_hard_calls++, ns_hard += d; if (d > ns_hard_max) ns_hard_max = d; if (d >= HARD_LOG_NS) { static volatile int32_t n_slow; log_limited(&n_slow, "released textures past the hard count: a batch walked %llu entries and dropped %llu " "(%llu written back) in %.2f ms", (unsigned long long)walked, (unsigned long long)(n_drop_by[DROP_HARD] - drops0), (unsigned long long)(n_drop_wb - wb0), d / 1e6); } } // NX2NAVI_TEX_VRAM_PRESSURE (percent of the device-local budget, default VRAM_PRESSURE_DEFAULT; 0 = off): above it, each // submit drops up to PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released // images (each ASTC one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason // above. Only entries that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their // storage. A render target lives only in its image; dropping it brought the texture back empty (black flashes in a // Depths boss fight, ~170 "left uninitialised" aliases a minute). // // The default lies under the seven eighths of the budget past which a pool gets no VRAM mirror (vk_memory.c // mirror_create), and the evictions aim 2% under it, so textures give way before mirrors are refused: on an 8 GiB card // that leaves the mirrors about 0.7 GiB, on a 24 GiB one 2 GiB. With the default at 90 the order was the other way // round: at 88% of a 22 GiB budget, full of textures of areas left long before, 0.4 MiB mirrors were refused and the // draws that read their pools over PCIe brought the game to 11-20 fps, while no texture had been asked to go. #define VRAM_PRESSURE_DEFAULT 80 #define PRESSURE_STEP 32 #define EVICT_LOOK 1024 // released entries one pressure check looks at (sweep_pressure) static int vram_pressure = -1; static uint64_t n_pressure_drops; static int comes_back_intact(const Tex *e) { return e->pub.mirrored && !e->pub.gpu_owned && !e->gpu_dirty; } // When a lookup last reached an entry: entry_of (every texture get and slot resolution) and the per-submit check of a // slot lookup (vk_texture_by_id_locked). The texture pool refresh looks every registered slot up once in each // submission that draws through the pool (vk_exec_draw.c pool_sync_entries), so an entry no lookup reached for // IDLE_ROUNDS such submissions and STALE_FRAMES presents (below) has no live descriptor naming it: the game keeps the // texture object, nothing samples it. The clock is look_round, which only a submission whose slot checks reached half // of the registered pool slots moves on (vk_texture_new_submit): counted in submissions, a pause screen or a fade that // draws nothing through the pool made the whole world's textures idle within seconds. // With NX2NAVI_POOL_FAST the refresh checks the entries by list instead (vk_texture_pool_fresh_locked), which is no // lookup. Such a submission counts for the clock all the same, and the entries the pool's slots hold get their stamp // from the count of those slots (pool_touch), each at least once in POOL_TOUCH_DIV submissions: an entry a slot names // never comes near IDLE_ROUNDS. #define IDLE_ROUNDS 240 static uint32_t look_round = 1; // submissions whose lookups walked the texture pool static uint32_t round_checks; // slot checks of the submission being recorded (vk_texture_by_id_locked) static uint32_t pool_slots; // registered slots the pool covers (id < VK_POOL_TEX) static int pool_ctx; // the texture pool refresh is looking slots up (vk_texture_pool_ctx) static void touch(Tex *e) { e->use_round = look_round; e->use_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (!pool_ctx) e->samp_frame = e->use_frame; // a draw binds it, or the game or the replay reads it otherwise } void vk_texture_pool_ctx(int on) { pool_ctx = on; } static int idle(const Tex *e, uint64_t now); // NX2NAVI_TEX_RELEASED_MB (default a thirty-second of the video memory budget, 384 MiB to 1 GiB, released_cap_bytes; // 0 = no limit) bounds the own memory of the released entries. With the hold on (drop_aged, the default) they hold // what the last minute released, up to the bound; sweep_pressure lets young entries past it while the card and the // system have room (vk_tex_release.h trl_keep). Before the hold the bound was all that limited them: RELEASED_MAX alone kept 5-7 GiB of // released render targets and decoded textures after an hour of play at a 2160p game resolution, device memory the // driver also charges to the system's commit (the process's private bytes reached 31 GiB). A sixteenth (1 to 4 GiB) // still let them hold 1.3 GiB on a 32 GB PC whose commit then ran out at an area load (device lost); only entries // released STALE_FRAMES presents ago go, so the transient ones the game takes back within frames stay. // NX2NAVI_TEX_COMMIT_FREE_MB (default the larger of 4 GiB and a tenth of the commit // limit; 0 = off) lowers that bound to COMMIT_LOW_KEEP while less of the system's commit is free: an allocation that // fails there ends the game. On Linux nothing fails at the limit: the OOM killer ends the game when its cgroup or the // machine runs out of memory, and plat_commit_info reports the tighter of the two. Both drop only stale entries // (drop_stale), a few per check like the VRAM pressure. // // NX2NAVI_TEX_PROC_COMMIT_MB (default the installed memory, at least PROC_COMMIT_MIN_MB; 0 = off) is a bound on this // process's own commit that is logged when passed; it counted the commit as low too until it was found that no texture // dropped could bring the process back under it (commit_is_low, at its end). The system's figure depends on the page file and on what else // runs: with a 64 GB limit it turned low at 59 GB in use, when this process alone held 42 GB on a PC with 32 GB of // memory, about half of it device memory, with the textures of five areas left behind. A process is not meant to need // more commit than the PC has memory: beyond that it lives on the page file, whatever its size. The bound holds until // the process is a thirty-second under it, so that it does not flip with every texture freed. The floor keeps it out // of the way on PCs with less memory, where a bound at the installed memory could hold all through play: the system's // figure decides there, as before. On Linux the process's figure is its resident anonymous memory, which cannot pass // the installed memory, so only a bound set below that holds there; a cgroup's limit and the machine's RAM act through // the system's figure. // Where a discrete GPU's memory counts against the commit (plat_commit_counts_vram: Windows), the bound compares the // commit less the device-local memory the driver reports in use (vk_mem_vram): that memory is on the card, not in RAM // or the page file, and the VRAM pressure and tex_disuse already bound it. Counted in, it held the bound all through // Hyrule Field on a 16 GB PC with a 16 GB card (RTX 5060 Ti): the process at 28.9 to 30.7 GB, 9 to 10 GB of it video // memory, while the system had 8 to 10 GB of its 49 GB limit free, and the ground flickered white as textures gave way. // So on such a PC the 42 GB process above, half of it device memory, no longer reaches this bound: the system's // figure and the VRAM pressure are what act there. The part of a card's memory the driver has moved into shared // system memory is subtracted too, which it should not be; the driver does not report it apart. #define COMMIT_LOW_KEEP ((uint64_t)256 << 20) #define PROC_COMMIT_MIN_MB 24576 static uint64_t released_cap; // bytes; 0 = not read yet, UINT64_MAX = no limit static long long commit_free_mb = -2; // -2: not read yet, -1: the default static long long proc_commit_mb; // this process's bound in MB; 0 = none static int commit_low, commit_logs; // commit_low: either of the two below at the last check static int sys_low, proc_high; // the system's commit is low; this process's is above its bound static uint64_t n_cap_drops, n_commit_drops, n_commit_past; // The system's commit at the last check, for the room young released entries past the bound may take (trl_room): MB // free and the free space under which it counts as low; -1 free: not tracked. static double commit_free_now = -1, commit_low_mark; static uint64_t released_cap_bytes(void) { if (!released_cap) { const char *s = cfg_get("NX2NAVI_TEX_RELEASED_MB"); long long mb; const char *by = ""; // which term set the bound, for the line below if (s && *s) mb = atoll(s); else { // By the card: a thirty-second of its budget, 384 MiB to 1 GiB (vk_tex_release.h trl_cap_mb, which has the // run that set it). What the released textures hold at their fullest stays in use for the rest of the // session, as free space inside the allocator's blocks once they are dropped, so the bound is the price of // the cache; until 0.11.0 it was a quarter of the budget, up to 8 GiB. The two pressure rules still come // first: released entries go before anything else above NX2NAVI_TEX_VRAM_PRESSURE of the budget, and the // bound falls to COMMIT_LOW_KEEP while the commit is low. uint64_t vused = 0, vbudget = 0; int exact = 0; int known = vk_mem_vram(&vused, &vbudget, &exact) && vbudget; mb = trl_cap_mb(known ? vbudget >> 20 : 0); by = !known ? ", the default without the card's budget" : mb == TRL_CAP_MAX_MB ? ", the most the rule by the card gives" : mb == TRL_CAP_MIN_MB ? ", the least the rule by the card gives" : ", a thirty-second of the video memory budget"; } released_cap = mb > 0 ? (uint64_t)mb << 20 : UINT64_MAX; if (mb > 0) rt_log("vk: released textures keep at most %lld MiB of their own memory%s (NX2NAVI_TEX_RELEASED_MB)", mb, by); else rt_log("vk: released textures: no size limit"); } return released_cap; } static int commit_is_low(void) { if (commit_free_mb == -2) { const char *s = cfg_get("NX2NAVI_TEX_COMMIT_FREE_MB"); commit_free_mb = s && *s ? atoll(s) : -1; if (commit_free_mb < -1) commit_free_mb = -1; s = cfg_get("NX2NAVI_TEX_PROC_COMMIT_MB"); proc_commit_mb = s && *s ? atoll(s) : -1; if (proc_commit_mb < 0) { float ram_used, ram_total, ram_proc; plat_mem_info(&ram_used, &ram_total, &ram_proc); proc_commit_mb = ram_total > PROC_COMMIT_MIN_MB ? (long long)ram_total : PROC_COMMIT_MIN_MB; } if (!plat_commit_enforced()) commit_free_mb = proc_commit_mb = 0; // no limit known char note[2048]; // the platform layer's own lines go to stderr only, so the log file gets this through here if (plat_commit_note(note, sizeof note)) rt_log("platform: %s", note); if (proc_commit_mb) rt_log("vk: this process's commit%s is logged when it passes %lld MB (NX2NAVI_TEX_PROC_COMMIT_MB; 0 = off); " "textures give way only when the system's commit is low", plat_commit_counts_vram() ? ", less the video memory in use on a discrete card," : "", proc_commit_mb); } commit_free_now = -1; if (!commit_free_mb && !proc_commit_mb) return 0; double proc, used, limit; plat_commit_info(&proc, &used, &limit); int low = 0; commit_critical = 0; if (commit_free_mb && used >= 0 && limit > 0) { double want = commit_free_mb > 0 ? (double)commit_free_mb : limit / 10 > 4096 ? limit / 10 : 4096; low = limit - used < want; commit_free_now = limit - used, commit_low_mark = want; commit_critical = limit - used < want / 2; // the pause for drops that come back ends here } double dev = 0; // device-local memory in use, MB: in the commit (plat_commit_counts_vram), not in RAM (see above) VkCtx *c = vk_ctx(); uint64_t dev_used, dev_budget; int exact; if (proc_commit_mb && plat_commit_counts_vram() && c && c->props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU && vk_mem_vram(&dev_used, &dev_budget, &exact)) dev = dev_used / 1048576.0 < proc ? dev_used / 1048576.0 : proc; int high = proc_commit_mb && proc - dev > (double)(proc_high ? proc_commit_mb - proc_commit_mb / 32 : proc_commit_mb); if (high != proc_high && commit_logs < 40) { commit_logs++; rt_log("vk: this process's commit %s: %.0f MB less %.0f MB of video memory, the bound is %lld MB (system: %.0f " "of %.0f MB in use); released textures kept %d, %.0f MiB of own memory%s", high ? "high" : "no longer high", proc, dev, proc_commit_mb, used, limit, nreleased, rel_bytes / 1048576.0, high ? ": the host's own memory, which dropping textures does not lower; textures give way only when the " "system's commit is low" : ""); } proc_high = high; if (low != sys_low && commit_logs < 40) { // a few lines, however often it flips commit_logs++; rt_log("vk: system commit %s: %.0f of %.0f MB in use (this process %.0f MB); released textures kept %d, %.0f MiB " "of own memory%s", low ? "low" : "no longer low", used, limit, proc, nreleased, rel_bytes / 1048576.0, low ? ": dropping the stale ones" : ""); // A page file Windows sizes grows only slowly and by little, so the limit stays near the installed memory plus // what the page file held at the boot: on a 16 GB PC with a 16 GB card it ran low in Hyrule Field at 32 GB, // and textures gave way (white or black ground). A fixed page file of 32 GB or more raised the limit to 49 GB. static int hinted; if (low && !hinted && plat_page_file_managed() == 1) { hinted = 1; rt_log("vk: the page file is sized by Windows, which grows it only slowly: a fixed page file whose initial " "size is at least 32 GB keeps the system's commit from running low (System > About > Advanced system " "settings > Performance Settings > Advanced > Virtual memory > Change: untick the automatic " "management, Custom size with an initial size of 32768 MB or more, Set, then restart Windows)"); } } sys_low = low; // Only the system's figure makes textures give way. The process's bound compares its commit less the video memory // in use, and a texture dropped lowers both by the same amount: past the bound nothing a drop does could bring it // back under, and textures went one after the other for as long as the rest of the process stayed large, the ground // flickering white with the system's commit to spare (#434). What is above it is the host's own memory (the // game's heap, pipelines, caches); the bound is logged, and the system's figure still acts when the commit runs low. return commit_low = low; } int vk_texture_commit_low(void) { return __atomic_load_n(&commit_low, __ATOMIC_RELAXED); } // Drops up to PRESSURE_STEP released entries that own memory and were released STALE_FRAMES presents ago or earlier, // oldest first and the intact ones before the others, while the released entries own more than keep. A transient // texture is taken back within frames, so dropping it frees nothing: the game initializes it again and it is created // again, empty if it is a render target. Under commit pressure that a system-managed page file never relieves, dropping // young ones destroyed 2160p render targets about every frame for whole sessions. Frame generation also reads a frame's // depth target at the host present (vk_dlssg.c vk_dlssg_frame tags it), after the blit deferred destruction waits for. // // A released entry is not dead to the game. A descriptor registered for a render target still resolves to its released // entry (slot_resolve: a finalized slot with no live texture at its storage gets its own), and the game can sample it // or take it back long after the release. In Kakariko Village, with the system's commit low, render targets dropped at // STALE_FRAMES came back empty and the scene was drawn black for one frame every two to three seconds (interface // intact), for as long as the commit stayed low. So no entry goes while lookups still reach it (idle), and one whose // content exists only in its image waits GPU_STALE_FRAMES, or GAP_TIMES the longest it stayed released before the game // took it back when that is longer. An overwritten entry has no content left to wait for (overwrite_others): it goes // with the intact ones, whether or not a descriptor still names it. #define STALE_FRAMES PRESENTS(120) #define GPU_STALE_FRAMES PRESENTS(1800) #define GAP_TIMES 8 // Nor is an entry idle that a shader sampled within STALE_FRAMES presents (samp_frame, vk_pool_marks_read): a // finalized slot keeps showing its released entry (stale_kept), and the pool's check passes released entries by, so no // lookup reaches such an entry while every frame samples it. Counted idle, it was dropped past the released entries' // size cap, and the slot showed the stand-in until the game registered it again: in a player's recording of Castle // Town, finalized slots lost their picture in the very frames such entries were dropped. static int idle(const Tex *e, uint64_t now) { return look_round - e->use_round >= IDLE_ROUNDS && now - e->use_frame >= STALE_FRAMES && now - e->samp_frame >= STALE_FRAMES; } // NX2NAVI_TEX_KEEP_TAKEN (default 1; 0: as before): an entry that comes back intact waits out GAP_TIMES the longest it // stayed released before the game took it back, as one whose content lives only in its image does, and not // STALE_FRAMES alone. Coming back intact is not free: the descriptor finds no entry, the frame samples its stand-in // while the entry is made and uploaded again, and the next frame has it. Kakariko's hazy air is drawn with ASTC // textures the game releases and takes back every few hundred frames; with the released entries at RELEASED_MAX the // sweep dropped about 30 of them as soon as their 120 presents were up, and the frame that needed them again came out // without the haze, darker and sharper, once every few seconds. static int keep_taken(void) { static int on = -1; if (on < 0) on = rt_knob("TEX_KEEP_TAKEN", "1") == 1; return on; } static uint64_t stale_after(const Tex *e) { uint64_t gap = (uint64_t)e->back_gap * GAP_TIMES; if (e->overwritten) return STALE_FRAMES; if (comes_back_intact(e)) return keep_taken() && gap > STALE_FRAMES ? gap : STALE_FRAMES; return gap > GPU_STALE_FRAMES ? gap : GPU_STALE_FRAMES; } static int drop_stale(uint64_t keep, uint64_t *count, int why) { // returns the entries dropped: PRESSURE_STEP, more may follow uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); int dropped = 0; const uint64_t stale = STALE_FRAMES; for (int any = 0; any < 2; any++) for (Tex *e = rel_head, *next; e && now - e->rel_frame >= stale; e = next) { // in release order if (dropped == PRESSURE_STEP || rel_bytes <= keep) return dropped; next = e->rel_next; if (!e->bytes || (!any && !comes_back_intact(e) && !e->overwritten)) continue; if (now - e->rel_frame < stale_after(e) || (!e->overwritten && !idle(e, now))) continue; drop_tex(e, why); dropped++, (*count)++; } return dropped; } // A draw, a read or the pool's marks used e within `within` presents of now (touch, vk_pool_marks_read). static int used_lately(const Tex *e, uint64_t now, uint64_t within) { return now - e->use_frame < within || now - e->samp_frame < within; } // A draw bound e, the game or the replay read it, or a marked shader sampled a slot that shows it, within `within` // presents of now (samp_frame). used_lately also counts the pool's own stamp, which every GPU-owned entry a slot still // holds gets in each submission whether or not anything samples the slot (pool_touch): for such an entry "used" says // that the game has not finalized the slot's texture object yet. Asked only while the marks tell what is sampled // (marks_tell); without them a slot's sampling is not known and used_lately stands. static int sampled_lately(const Tex *e, uint64_t now, uint64_t within) { return now - e->samp_frame < within; } // Drops up to PRESSURE_STEP entries that were marked overwritten OVERWRITTEN_KEEP presents ago or earlier // (overwrite_others): the image reuse found no texture to give their images to in that time. // Not one that was used since (NX2NAVI_TEX_OVERWRITTEN_USED=0: those too, as before): the mark says another target was // drawn over the entry's storage, not that nothing reads the entry. In a player's recording the entry of a second // descriptor at the scene target's address (format 0x40 beside the target's 0x3f, both 2400x1350) was dropped here // and made again in the same frame, its content copied from a released entry at that address, whose bits are the // scene in the other format. That frame's picture was nearly black with its colours exchanged (brightness 17 against // 102 in the frames around it); the frames before and after were as usual. Whatever looks such an entry up after its // drop makes it again that way, so an entry something still uses stays; so does one the image reuse would take // (create_tex, REUSE_UNUSED). // full: the card is past the VRAM pressure's share of its budget by the driver's own figure (sweep_pressure). The // wait is then OVERWRITTEN_FULL presents, four times what the image reuse leaves an entry alone for there: an image no // texture took in that time holds nothing the game can ask for, and the memory it keeps for ten seconds more is what // the next image fails for. Not sooner: at ten presents this took the images the reuse was about to hand on, 7,301 in // a run at 3840x2160 that then made 7,672 images and reused 5,226. #define OVERWRITTEN_FULL (REUSE_UNUSED_FULL * 4) static uint64_t n_over_full; // since the last perf line: of n_over_drops, dropped with the card that full static void drop_overwritten(int full) { static int used_too = -1; static uint64_t n_kept; if (used_too < 0) used_too = !cfg_bool("NX2NAVI_TEX_OVERWRITTEN_USED"); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // used: drawn with or sampled (sampled_lately); a slot that merely still holds the entry does not keep it int dropped = 0, marks = marks_tell(); const uint64_t keep = full ? OVERWRITTEN_FULL : OVERWRITTEN_KEEP; for (Tex *e = rel_head, *next; e && now - e->rel_frame >= keep && dropped < PRESSURE_STEP; e = next) { next = e->rel_next; if (!e->overwritten || now - e->ow_frame < keep) continue; if (!used_too && (marks ? sampled_lately(e, now, keep) : used_lately(e, now, keep))) { if (e->kept_frame != now + 1 && (!n_kept++ || !(n_kept & (n_kept - 1)))) // the first, then at powers of two rt_log("vk: an overwritten texture stays, it is still used: %#llx fmt %#x %ux%u, last used %llu presents " "ago (%llu so far; NX2NAVI_TEX_OVERWRITTEN_USED=0 drops such a texture)", (unsigned long long)e->key.addr, e->t.d.format, e->t.d.width, e->t.d.height, (unsigned long long)(now - (marks || e->samp_frame > e->use_frame ? e->samp_frame : e->use_frame)), (unsigned long long)n_kept); e->kept_frame = now + 1; e->ow_frame = now; // looked at again OVERWRITTEN_KEEP presents from now continue; } drop_tex(e, DROP_OVERWRITTEN); dropped++, n_over_drops++, n_over_full += full != 0; } } // ---- released textures by age (vk_tex_release.h has the take-back figures behind the hold) // NX2NAVI_TEX_RELEASED_HOLD (seconds, default TRL_HOLD_S, at most TRL_HOLD_MAX_S; 0 = no hold): // a released entry that owns memory goes once it has gone unused that long, twice that for content only the GPU made, // whatever the size bound says. Unused counts from the release or the last sample of the released image, whichever is // later, in seconds of the wall clock (secs_since). The rules drop_stale keeps hold here too: not before stale_after // (an entry the game once took back after a long gap waits GAP_TIMES that gap) and not while it is not idle (a lookup // other than the pool's reached it, or a shader sampled it, lately). Nor an entry the GPU drew into since its last // write-back: its drop would first copy it back to the game's memory and wait for the GPU (drop_tex, write_back_n), // a stall the sweep is not to make in play; such an entry stays to the size bound, as before the hold. Each check // looks at up to AGED_LOOK entries from where the last one stopped (aged_at), in release order, and starts from the // oldest again at the first one released less than the hold ago: the rest are younger. #define AGED_LOOK 256 static int rel_hold = -1; // seconds; -1: not read yet static uint64_t n_age_drops, n_age_bytes; // since the last vk_texture_reuse_log line static uint32_t released_hold(void) { if (rel_hold < 0) { const char *s = cfg_get("NX2NAVI_TEX_RELEASED_HOLD"); long long v = s && *s ? atoll(s) : TRL_HOLD_S; rel_hold = v < 0 ? 0 : v > TRL_HOLD_MAX_S ? TRL_HOLD_MAX_S : (int)v; if (rel_hold) rt_log("vk: released textures go once unused for %d s, %d s for content only the GPU made " "(NX2NAVI_TEX_RELEASED_HOLD); young ones past the size bound stay while the card and the system " "have room, up to %d times the bound", rel_hold, (int)trl_hold((uint32_t)rel_hold, TRL_GPU), TRL_ROOM_TIMES); else rt_log("vk: released textures stay until the size bound or a pressure takes them " "(NX2NAVI_TEX_RELEASED_HOLD=0)"); } return (uint32_t)rel_hold; } static void drop_aged(void) { // tex_lock held uint32_t hold = released_hold(); if (!hold) return; uint64_t now = frames_now(); Tex *e = aged_at ? aged_at : rel_head; int dropped = 0; for (int n = 0; n < AGED_LOOK && dropped < PRESSURE_STEP; n++) { if (!e || secs_since(e->rel_frame) < hold) { // the rest is younger: from the oldest next time e = NULL; break; } Tex *next = e->rel_next; int cls = comes_back_intact(e) || e->overwritten ? TRL_STORAGE : TRL_GPU; uint64_t last = e->samp_frame > e->rel_frame ? e->samp_frame : e->rel_frame; if (e->bytes && !e->gpu_dirty && trl_aged(hold, cls, secs_since(last)) && now - e->rel_frame >= stale_after(e) && idle(e, now)) { n_age_drops++, n_age_bytes += e->bytes, dropped++; drop_tex(e, DROP_AGE); } e = next; } aged_at = e; } // NX2NAVI_TEX_EVICT (default on; 0 = only released entries go under VRAM pressure, as before): above the VRAM pressure // threshold, once the released entries that come back intact are gone, live entries go too, least recently looked up // first: those that come back intact and that no lookup reached for IDLE_ROUNDS pool walks and STALE_FRAMES presents // (touch: no live descriptor names them). A later lookup creates the image again and uploads it from its storage, as // for a texture seen the first time. Up to EVICT_STEP entries per check, until the device-local use is 2% of the budget // under the threshold. The use counts the free space inside VMA's blocks as free (vk_mem_vram_live): a freed texture // leaves a hole the driver still counts, and measured that way the evictions went on until nearly every idle texture // was gone. The next check waits until the GPU is past the submissions that may still name the images, as their // memory goes back to VMA only then (vk_defer_image). A VRAM mirror that did not fit asks for room the same way, below // the threshold too (vk_texture_want_room): a pool without its mirror costs far more than a texture uploaded again. // Its request stands as the use the mirror fits at (room_level) for ROOM_CHECKS checks or until the use is down there // (sweep_pressure): taken as bytes to free at the one check that read it, a request for 205 MiB got the EVICT_STEP // textures of that check, 26 MiB, or nothing when the check fell before evict_wait, and was gone. // A walk of the whole table that finds no candidate is not made again for EVICT_EMPTY_WAIT submissions (two pressure // checks; idle_empty_wait, which only evict_idle reads, so that evict_disused still runs): under a standing pressure // or mirror request with nothing idle, every eighth submission walked every texture again. An entry turns idle only // after IDLE_ROUNDS pool walks, so the wait delays an eviction by a few submissions at most. #define EVICT_STEP 64 #define EVICT_EMPTY_WAIT 16 #define ROOM_CHECKS 64 static int tex_evict = -1; static uint64_t evict_wait; // vk_sync_fence_done value the next eviction waits for static uint64_t idle_empty_wait; // vk_sync_issued value the next walk of evict_idle waits for static uint64_t n_evict, n_evict_bytes; // since the last perf line static uint64_t n_evict_empty, ns_evict_empty; // empty walks of evict_idle and their time, as long static int evict_logs; static volatile int64_t room_wanted; // bytes refused VRAM mirrors asked for (vk_texture_want_room) static uint64_t room_level; // tex_lock: the device-local use at which those mirrors fit static uint32_t room_checks; // tex_lock: checks the request still stands for; 0 = none stands void vk_texture_want_room(uint64_t bytes) { int64_t cur = __atomic_load_n(&room_wanted, __ATOMIC_RELAXED); // the largest request since the last check while ((int64_t)bytes > cur && !__atomic_compare_exchange_n(&room_wanted, &cur, (int64_t)bytes, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} } static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget, int why) { // tex_lock held; returns the bytes freed if (vk_sync_fence_done() < evict_wait || vk_sync_issued() < idle_empty_wait) return 0; uint64_t t0 = kern_now_ns(); Tex *v[EVICT_STEP]; int nv = 0; uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP oldest candidates, oldest first Tex *e = by_key.vals[i]; if (!e || e->released || e->retiring || !e->bytes || !comes_back_intact(e) || !idle(e, now)) continue; uint32_t age = look_round - e->use_round; int at = nv; while (at > 0 && look_round - v[at - 1]->use_round < age) at--; if (at == EVICT_STEP) continue; if (nv < EVICT_STEP) nv++; memmove(v + at + 1, v + at, sizeof *v * (size_t)(nv - 1 - at)); v[at] = e; } if (!nv) { // the walk of the whole table found nothing: none again for EVICT_EMPTY_WAIT submissions idle_empty_wait = vk_sync_issued() + EVICT_EMPTY_WAIT; n_evict_empty++, ns_evict_empty += kern_now_ns() - t0; return 0; } uint64_t freed = 0; int n = 0; for (; n < nv && freed < want; n++) freed += v[n]->bytes, drop_tex(v[n], why); if (!n) return 0; n_evict += (uint64_t)n, n_evict_bytes += freed; evict_wait = vk_sync_issued() + 2; if (evict_logs < 8 || ((rt_trace & RT_TRACE_PERF) && evict_logs < 64)) { evict_logs++; rt_log("vk: VRAM %.0f of %.0f MiB in use: %d idle textures evicted (%.1f MiB), least recently used first; they " "are uploaded again when the game uses them", used / 1048576.0, budget / 1048576.0, n, freed / 1048576.0); } return freed; } // ---- disuse (NX2NAVI_TEX_DISUSE, default on; vk_pipeline.c "texture pool marks", vk_exec_draw.c "pool marks") // Ocarina keeps the textures of the areas it has loaded registered in the pool, and the pool refresh looks every // registered slot up, so idle (above) never holds for them, and their memory, which the driver charges to the system's // commit, only grew. Marked shaders tell which slots they sample: vk_pool_marks_read sets samp_frame of the texture of // each marked slot. Where evict_idle frees too little, evict_disused takes live textures that come back intact and that // nothing sampled for DISUSE_FRAMES presents, least recently sampled first, in two steps: // - retired (retire): the entry stays whole, but the pool refresh no longer resolves a slot to it (slot_resolve), so // every slot that names it gets the kind's stand-in. Any use in the meantime takes it back as it is (retire_cancel): // a lookup other than the pool refresh, or a mark on one of its slots (the stand-in was sampled). // - destroyed (retire_sweep) once the texture pool has taken every set of its ring since, each brought up to date // when taken (vk_exec_draw.c pool_take_set, pool_catch_up), after a sync that left no slot behind wrote its stand-in // (vk_pool_takes counts only such takes), and the GPU has finished the submissions recorded until // then: no set can be bound with its view any more, and no submission that bound one is still running. Its key is // held then: a held key makes no entry in the pool refresh (held_out), so the slot keeps its stand-in; a mark on the // slot drops the hold and the next refresh makes the texture again, as does any other lookup. // An image freed at the eviction itself, as before, faulted the GPU once (a read of freed memory in a pass, 0.6.1.14) // without a path to it found in the code; the delay leaves no window in the pool sets, and the cancels count uses the // marks missed. NX2NAVI_TEX_DISUSE=2 retires and never destroys (stand-ins only, nothing freed): it tells the marks and // stand-ins from the freeing as the cause of a fault. Only while every module that indexes the pool was marked and two // reads arrived: an unmarked shader could sample what the marks call unused. // Two minutes, not the thirty seconds it was: a texture out of view that long and then seen again shows its stand-in // for the frames until its mark is read, which is a flash of missing ground or a dark object each time the camera // turns back, and a batch of such textures gave back 0.4 to 31 MiB in a recorded session. The textures of an area // left behind, which are what the memory grows by, still go two minutes after it. #define DISUSE_SECONDS 120 #define DISUSE_FRAMES PRESENTS(DISUSE_SECONDS * 60) #define HOLDS_MAX 65536 static nvn_hmap holds = {.klen = sizeof(TexKey)}; // value: 1 static uint32_t marks_reads; // Whether the pool's marks tell which textures the shaders sample: every module that indexes the pool is marked, no // read was missed and two have arrived. static int marks_tell(void) { return vk_pool_marks_enabled() && !__atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) && marks_reads >= 2; } // Whether the scene is being drawn, from the marks: marks_peak is the most slots one read marked lately (in 1/256, // halving in about a minute of reads), marks_live the reads in a row that marked at least a quarter of it. A pause // screen, a menu or a fade draws next to nothing through the pool for longer than DISUSE_SECONDS: every texture of the // scene behind it then counted as unsampled, went under memory pressure, and the scene came back as stand-ins. // evict_disused waits for two such reads in a row: the first one after a pause is what notes the scene's textures as // sampled again. static uint64_t marks_peak; static uint32_t marks_live; static int resolve_held; // slot_resolve left the slot out for a held key or a retired entry static uint64_t n_disuse, n_disuse_bytes, n_disuse_back; // since the last perf line // since the last perf line: eviction candidates kept because another descriptor is registered at their address // (shared_address), and holds dropped by a mark that another entry at the address had taken (vk_pool_marks_read) static uint64_t n_disuse_shared, n_disuse_diverted; static uint64_t n_retire_cancel, n_retire_freed; // since the last perf line static uint64_t n_retire_stalled; // retired textures taken back because they never came due #define RETIRE_PATIENCE PRESENTS(5 * 60) static int disuse_logs; static Tex **retiring; // retired entries not destroyed yet static int nretiring, cretiring; static uint64_t retiring_bytes; // Takes a retired entry back as it is: something uses it. Its slots resolve to it again (the bump). static void retire_cancel(Tex *e) { e->retiring = 0; // it leaves retiring[] at the next retire_sweep retiring_bytes -= e->bytes; if (holds.n) nvn_hm_del(&holds, &e->key); n_retire_cancel++, n_disuse_back++; // In use as of now: while retired the pool refresh left it out, so it carries no use stamp of that time and // counted as idle the moment it came back. A sweep over the budget (sweep_pressure calls retire_sweep and then // evict_idle) could evict it in the same call that took it back. e->use_round = look_round; e->use_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); tex_gen_bump(e->key.addr); } static void retire_remove(Tex *e) { // destroy_tex: the entry goes another way (released and dropped, shutdown) for (int i = 0; i < nretiring; i++) if (retiring[i] == e) { retiring[i] = retiring[--nretiring]; break; } if (e->retiring) e->retiring = 0, retiring_bytes -= e->bytes; } // Each submission: destroys the retired entries the pool sets and the GPU are done with (above). TEX_DISUSE=2 keeps them. static void retire_sweep(void) { if (!nretiring) return; static int keep = -1; if (keep < 0) keep = rt_knob("TEX_DISUSE", "1") == 2; uint64_t takes = vk_pool_takes(), done = vk_sync_fence_done(); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); int k = 0; for (int i = 0; i < nretiring; i++) { Tex *e = retiring[i]; if (!e->retiring) continue; // taken back if (!e->retire_due && takes - e->retire_takes > VK_POOL_SETS) e->retire_due = vk_sync_issued(); // The takes that count are those of syncs that left no slot behind, and a PC that has no creation time over // in most frames leaves some behind in nearly every sync: there a retired texture was never destroyed and its // slots showed the stand-in for as long as that lasted, minutes in one report. One that is still not due // after RETIRE_PATIENCE presents is taken back and counts as sampled now. if (!e->retire_due && now - e->retire_frame > RETIRE_PATIENCE) { retire_cancel(e); e->samp_frame = now; n_retire_stalled++; continue; } if (!keep && e->retire_due && done >= e->retire_due) { e->retiring = 0, retiring_bytes -= e->bytes; nvn_hm_put(&holds, &e->key, (void *)(uintptr_t)1); n_retire_freed++; drop_tex(e, DROP_DISUSE); continue; } retiring[k++] = e; } nretiring = k; static uint64_t stalled_logged; static int stalled_logs; if (n_retire_stalled != stalled_logged && stalled_logs < 8) { stalled_logs++; rt_log("vk: %llu textures evicted as unsampled were taken back: the texture pool had not been brought up to " "date completely in %d s, so they could not be destroyed and their slots showed stand-ins", (unsigned long long)(n_retire_stalled - stalled_logged), 5); stalled_logged = n_retire_stalled; } } // A lookup of key k that found no entry: 1 when the pool refresh asks for a held key (it gets the stand-in). Any other // lookup ends the hold: the game uses the texture. static int held_out(const TexKey *k) { if (!holds.n || !nvn_hm_get(&holds, k)) return 0; if (pool_ctx) return 1; nvn_hm_del(&holds, k); n_disuse_back++; return 0; } // tex_lock held. Whether a registered slot at e's address names another descriptor than e's: the same bytes through // another swizzle or format, or a view the game draws through. Such a texture is not retired by disuse. Its slots // do not all come back by the marks: a slot of the other descriptor that followed e gets an image of its own once // e is gone, an empty one when the descriptor is not a CPU-written one, and from then on that image takes the // marks of every slot at the address, so the held key of e was never dropped and the slots that named e showed // the stand-in for good: the ground of Hyrule Field and of Lon Lon Ranch drawn flat and see-through after 6,663 // textures went under low commit and 928 came back. static int slot_names_other(int32_t id, const Tex *e) { const TexSlot *s = &tslots[id]; if (!s->used || s->finalized || (uintptr_t)s->t.mem != e->key.addr) return 0; TexKey sk; make_key(&s->t, &sk); return memcmp(&sk, &e->key, sizeof sk) != 0; } static int shared_address(const Tex *e) { if (!sinfo || !slot_next) return 0; uint32_t b = agen_idx(e->key.addr); for (int32_t id = bucket_head[b]; id >= 0; id = slot_next[id]) if (slot_names_other(id, e)) return 1; for (int32_t g = bucket_ghead[b]; g >= 0; g = grp[g].gnext) for (int32_t id = grp[g].pub.head; id >= 0; id = slot_next[id]) if (slot_names_other(id, e)) return 1; return 0; } // NX2NAVI_TEX_DISUSE_EVICT (default 0; 1 = textures no shader sampled for a while are retired under pressure, as // before). Off since players' ground went flat and see-through after such evictions and stayed so (Hyrule Field, Lon // Lon Ranch; 1,441 retired on arriving in a village, 82 back): with the eviction off the same play kept its ground, // and neither of the two holes closed in the way back (shared_address, the hold drop in vk_pool_marks_read) acted // in the session that still lost it. What takes the retired texture's place for good is not known yet; a draw that // samples the stand-in once into a picture the game keeps would do it, and no mark can undo that. The shaders keep // their marks (NX2NAVI_TEX_DISUSE), so turning this on changes no pipeline. Released textures and the idle ones the // game no longer names still give way under pressure. static int disuse_evict_on(void) { static int on = -1; if (on < 0) { on = rt_knob("TEX_DISUSE_EVICT", "0") == 1; rt_log("vk: textures no shader sampled for a while %s under pressure (NX2NAVI_TEX_DISUSE_EVICT)", on ? "are retired" : "stay"); } return on; } // tex_lock held; returns the bytes retired (freed a few submissions later), counting those still retiring as freed static uint64_t evict_disused(uint64_t want) { if (!disuse_evict_on()) return 0; if (!want || !vk_pool_marks_enabled() || __atomic_load_n(&vk_pool_marks_missed, __ATOMIC_RELAXED) || marks_reads < 2 || marks_live < 2 || vk_sync_fence_done() < evict_wait) return 0; if (retiring_bytes >= want) return retiring_bytes; // what is on its way out covers it want -= retiring_bytes; Tex *v[EVICT_STEP]; int nv = 0; uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), disuse = DISUSE_FRAMES; for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP least recently sampled candidates, oldest first Tex *e = by_key.vals[i]; if (!e || e->released || e->retiring || !e->bytes || !comes_back_intact(e) || now - e->samp_frame < disuse) continue; int at = nv; while (at > 0 && v[at - 1]->samp_frame > e->samp_frame) at--; if (at == EVICT_STEP) continue; if (nv < EVICT_STEP) nv++; memmove(v + at + 1, v + at, sizeof *v * (size_t)(nv - 1 - at)); v[at] = e; } // A candidate whose address another descriptor shares stays (shared_address), and counts as sampled now, so the // walk over its bucket is made once per disuse time and the next candidates get their turn. int kept = 0; for (int i = 0; i < nv; i++) { if (shared_address(v[i])) v[i]->samp_frame = now, n_disuse_shared++; else v[kept++] = v[i]; } nv = kept; if (!nv) return 0; if (holds.n + (uint32_t)nv > HOLDS_MAX) { // keys of textures the game has let go of pile up: keep those a slot still holds out (cheld). Dropping them all // would make the pool refresh create every evicted texture again while memory is short. nvn_hm_clear(&holds, NULL); for (int id = 0; id < ntslots; id++) { if (!tslots[id].used || !tslots[id].cheld) continue; TexKey k; make_key(&tslots[id].t, &k); nvn_hm_put(&holds, &k, (void *)(uintptr_t)1); } rt_log("vk: holds of textures evicted by disuse past %d: kept the %u that registered slots still name", HOLDS_MAX, holds.n); } uint64_t freed = 0; int n = 0; uint64_t takes = vk_pool_takes(); for (; n < nv && freed < want; n++) { // retire: the pool refresh gives their slots the stand-in from now on Tex *e = v[n]; if (nretiring == cretiring) { int c = cretiring ? cretiring * 2 : 256; Tex **r = realloc(retiring, sizeof *r * (size_t)c); if (!r) break; retiring = r, cretiring = c; } retiring[nretiring++] = e; e->retiring = 1, e->retire_takes = takes, e->retire_due = 0, e->retire_frame = now; pool_unlist(e); // no slot resolves to it in the pool refresh; taken back, the refill of its slots lists it again retiring_bytes += e->bytes; freed += e->bytes; tex_gen_bump(e->key.addr); } n_disuse += (uint64_t)n, n_disuse_bytes += freed; evict_wait = vk_sync_issued() + 2; if (disuse_logs < 8 || ((rt_trace & RT_TRACE_PERF) && disuse_logs < 64)) { disuse_logs++; rt_log("vk: %d textures no shader sampled for %d s or more evicted (%.1f MiB), least recently sampled first; their " "images go once the texture pool sets no longer name them, and one sampled again is made again within a " "few frames", n, DISUSE_SECONDS, freed / 1048576.0); } return retiring_bytes; } // The registered slots at e's address, for marks_check's line: how many, those of e's own descriptor ("f": finalized), // those whose last lookup gave e (a slot of another descriptor among them follows e), and the other descriptors' with // format and flags; up to 8 ids of each. static void slot_note(int32_t id, const Tex *e, int *n, int *n_own, int *n_show, int *n_other, char *own, char *show, char *other, size_t own_cap, size_t other_cap) { const TexSlot *s = &tslots[id]; if (!s->used || (uintptr_t)s->t.mem != e->key.addr) return; ++*n; TexKey sk; make_key(&s->t, &sk); size_t at; if (!memcmp(&sk, &e->key, sizeof sk)) { if (++*n_own <= 8 && (at = strlen(own)) < own_cap - 1) snprintf(own + at, own_cap - at, " %d%s", (int)id, s->finalized ? "f" : ""); } else if (++*n_other <= 8 && (at = strlen(other)) < other_cap - 1) { snprintf(other + at, other_cap - at, " %d (fmt %#x flags %#x%s)", (int)id, s->t.d.format, (unsigned)s->t.d.flags, s->finalized ? ", finalized" : ""); } if (s->ce == e && s->cgen == agen[sinfo[id] & 0xffff] && ++*n_show <= 8 && (at = strlen(show)) < own_cap - 1) snprintf(show + at, own_cap - at, " %d", (int)id); } static void slots_text(char *out, size_t n, const Tex *e) { char own[112] = "", show[112] = "", other[360] = ""; int total = 0, n_own = 0, n_show = 0, n_other = 0; uint32_t b = agen_idx(e->key.addr); for (int32_t id = bucket_head[b]; id >= 0; id = slot_next[id]) slot_note(id, e, &total, &n_own, &n_show, &n_other, own, show, other, sizeof own, sizeof other); for (int32_t g = bucket_ghead[b]; g >= 0; g = grp[g].gnext) for (int32_t id = grp[g].pub.head; id >= 0; id = slot_next[id]) slot_note(id, e, &total, &n_own, &n_show, &n_other, own, show, other, sizeof own, sizeof other); snprintf(out, n, "%d registered; %d of this descriptor:%s; %d showing this entry:%s; %d of other descriptors:%s", total, n_own, n_own ? own : " none", n_show, n_show ? show : " none", n_other, n_other ? other : " none"); } // NX2NAVI_TRACE=perf, from vk_pool_marks_read: a shader sampled slot id, and nothing has ever put content into e, the // entry the slot shows or one credited with the mark (no_content): the picture drawn through it is an empty or // undefined image, which nothing else reports. One line per entry and up to MARK_LINES per address (said_at); the // marked slots that met such an entry are counted in the texture origins line whether or not a line was written. static uint64_t n_mark_empty, n_mark_none; // since the last perf line: marked slots that met such an entry; that show none #define MARK_LINES 6 #define MARK_LINES_ALL 20000 static void marks_check(Tex *e, uint32_t id, uint64_t now, int followed) { if (!no_content(e)) return; // an entry the CPU writes is uploaded by its first check, which one just made (again) has not had yet if (e->pub.mirrored && now - e->og_frame < 4) return; n_mark_empty++; if (e->said) return; e->said = 1; static SaidAt said; static uint32_t total; if (total >= MARK_LINES_ALL || !said_at(&said, e->key.addr, MARK_LINES)) return; total++; char d[224], og[320], state[160], slots[640], others[640]; desc_text(d, sizeof d, e); origin_text(og, sizeof og, e); slots_text(slots, sizeof slots, e); others_at(others, sizeof others, e, NULL); if (e->pub.mirrored) snprintf(state, sizeof state, "CPU-written, not uploaded%s", e->up_retry ? " (an upload that had no memory waits for its next try)" : ""); else snprintf(state, sizeof state, "not CPU-written (%s), never drawn into, copied into or filled", mirror_refused(&e->t)); rt_log("perf: frame %llu: sampled without content: texture %#llx (%llu bytes) [%s]: %s; %s; marked slot %u%s; slots " "at the address: %s; other descriptors there: %s", (unsigned long long)now, (unsigned long long)e->key.addr, (unsigned long long)e->t.L.size, d, state, og, id, followed ? " (of another descriptor, credited to this entry)" : "", slots, others); } void vk_pool_marks_read(const uint32_t *marks, uint32_t n) { lock(); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); int trace = (rt_trace & RT_TRACE_PERF) != 0; if (n > (uint32_t)ntslots) n = (uint32_t)ntslots; uint32_t marked = 0; for (uint32_t id = 0; id < n; id++) { if (!marks[id]) continue; if (!tslots[id].used) { // sampled, and not a registered slot now: what it showed is not known here if (flight_on) flight_tex(FLIGHT_TEX_UNREG, 0, 0, 0, 0, 0, 0, 0, id); continue; } marked++; TexSlot *s = &tslots[id]; if (s->ce && s->cgen == agen[sinfo[id] & 0xffff]) { // what the slot resolves to now if (s->ce->retiring) retire_cancel(s->ce); // not expected: retiring bumps the slot's generation s->ce->samp_frame = now; if (trace) marks_check(s->ce, id, now, 0); continue; } if (trace && !s->ce && s->cgen == agen[sinfo[id] & 0xffff]) n_mark_none++; // its last lookup gave no entry TexKey k; make_key(&s->t, &k); Tex *e = nvn_hm_get(&by_key, &k); if (e && !e->released) { if (e->retiring) retire_cancel(e); // retired, and its stand-in was sampled: back as it is e->samp_frame = now; if (trace) marks_check(e, id, now, 0); continue; } // Its own entry is released: the slot shows it where no live entry at the address is followed (slot_resolve), // so it counts as sampled too. The image reuse and drop_overwritten go by this stamp (sampled_lately). if (e) e->samp_frame = now; // The slot has no entry of its own: it samples another descriptor's image at its address (slot_resolve follows // one of the same target, as a second descriptor with another swizzle does). That image is what was sampled. // With the mark given to the slot's own key alone, a texture sampled only through such slots was never noted // as sampled while the slots' cached resolution was out of date, which it is at most reads in a bucket the // game keeps changing: it was retired as unused after two minutes, and a mark on its stand-in could not take // it back. While the pool's syncs kept leaving slots behind, a retired texture was never destroyed either // (retire_sweep waits for complete takes), so its slots kept the stand-in: the ground of the field drawn // without its textures and its height. int followed = 0; for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) { if (p->released || p->key.d.target != k.d.target) continue; if (p->retiring) retire_cancel(p); p->samp_frame = now; followed = 1; if (trace) marks_check(p, id, now, 1); } if (flight_on && !followed) // nothing of its own and nothing followed: its stand-in was on screen flight_tex(FLIGHT_TEX_BLANK, k.addr, (uint32_t)s->t.d.format, s->t.d.width, s->t.d.height, 0, tex_mirrored(&s->t) ? FLIGHT_TF_MIRRORED : 0, (s->finalized ? 1u : 0) | (holds.n && nvn_hm_get(&holds, &k) ? 2u : 0), id); // The slot's own key may be held although an entry at its address took the mark: the slot shows the stand-in // then (slot_resolve follows fewer entries than the loop above credits), and only this drop makes its texture // again. Left to the case where the loop found nothing, such a slot kept the stand-in for good. if (holds.n && nvn_hm_del(&holds, &k)) { // the stand-in was sampled: the next pool refresh makes it n_disuse_back++; n_disuse_diverted += followed != 0; s->cheld = 0; s->cwant = 1; // on screen as a stand-in: made at the next refresh, whatever the frame's creation time tex_gen_bump(k.addr); } } marks_reads++; marks_peak -= marks_peak >> 14; if ((uint64_t)marked << 8 > marks_peak) marks_peak = (uint64_t)marked << 8; if (marked >= 16 && ((uint64_t)marked << 10) >= marks_peak) marks_live += marks_live < 1000; else marks_live = 0; unlock(); } // NX2NAVI_TEX_COMMIT_EVICT (default on; 0 = off): while the commit is low (commit_is_low), idle live textures go as // under VRAM pressure (evict_idle), COMMIT_EVICT_STEP at a time, once the released entries gave what they could: they // are down to COMMIT_LOW_KEEP, or drop_stale found less than a full step to drop. The driver charges device memory to // the commit too, and the eviction above starts only at NX2NAVI_TEX_VRAM_PRESSURE of the device's budget: on a 16-32 GB // card the textures of every area visited stayed until the commit ran out (27-37 GB processes, then a failed // allocation or a device lost, #177 #203 #215), as nothing else gave them back while they stayed below that share of // the video memory. Waiting for the released entries to be down to COMMIT_LOW_KEEP, as before, never ended where the // game releases and takes back 200-300 textures a frame: 2,048 released entries held 1.4-2.0 GiB, none of them stale, // and with the commit low for minutes no live texture went (a 42 GB process). The entries the game takes back stay // either way: drop_stale leaves them, and the evictions below take live textures only. #define COMMIT_EVICT_STEP ((uint64_t)256 << 20) static int commit_evict_on = -1; static void commit_evict(void) { // tex_lock held, the commit is low and the released entries gave what they could if (commit_evict_on < 0) commit_evict_on = rt_knob("TEX_COMMIT_EVICT", "1") == 1; if (!commit_evict_on || !tex_evict) return; if (rel_bytes > COMMIT_LOW_KEEP) { static int logs; if (!n_commit_past++ && logs < 4) { logs++; rt_log("vk: commit low: the %d released textures keep %.0f MiB the game still takes back or looks up; " "live textures nothing sampled go without waiting for them", nreleased, rel_bytes / 1048576.0); } } uint64_t used = 0, budget = 0; vk_mem_vram_live(&used, &budget); uint64_t freed = evict_idle(COMMIT_EVICT_STEP, used, budget, DROP_IDLE_COMMIT); if (freed < COMMIT_EVICT_STEP) evict_disused(COMMIT_EVICT_STEP - freed); } // The released entries hold more than their bound (released_cap_bytes) after drop_stale took what it may: one line that // says what keeps the rest, at most every REL_CENSUS_SUBS submissions and REL_CENSUS_LINES times a run. The bound is // no bound for an entry a lookup still reaches, and the pool stamps every GPU-owned entry that a slot's last fill // resolved to in each submission (pool_touch), a finalized slot's released one too: at 5120x2160 a session's log had // 989 and then 2,172 released entries holding 9.2 and 10.1 GiB against a bound of 4,008 MiB, and its overwritten // render targets "last used 1 presents ago". Each entry counts once, under the first reason that holds. // The same line, at most every REL_CENSUS_ROOM_NS of wall time and REL_CENSUS_ROOM_LINES times a run, while they hold // more than the bound itself but not a quarter more than what they may keep (room 1), and only while trl_keep lets // them keep more than the bound: with room on the card it lets them keep twice the bound, and a session that held // 7,150 MiB against a bound of 4,008 MiB (2026-11-13) never logged what kept them. Without that room what they may // keep is the bound, and the line said "kept past it while the card has room" of entries drop_stale was taking. #define REL_CENSUS_SUBS 3600 #define REL_CENSUS_LINES 24 #define REL_CENSUS_ROOM_NS 60000000000ull #define REL_CENSUS_ROOM_LINES 60 static void rel_census(uint64_t cap, int room) { // tex_lock held static uint32_t last, lines, room_lines; static uint64_t room_at; if (room) { uint64_t t = plat_now_ns(); if (room_lines >= REL_CENSUS_ROOM_LINES || (room_at && t - room_at < REL_CENSUS_ROOM_NS)) return; room_at = t, room_lines++; } else { if (lines >= REL_CENSUS_LINES || (last && submit_epoch - last < REL_CENSUS_SUBS)) return; last = submit_epoch ? submit_epoch : 1, lines++; } uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); enum { YOUNG, SLOT, SLOT_OVER, SAMPLED, LOOKED, NEXT, KINDS }; uint64_t by[KINDS] = {0}, gpu = 0; int n[KINDS] = {0}, ngpu = 0; // the young ones by what they wait for: fewer than STALE_FRAMES since the release; content only the GPU made // (GPU_STALE_FRAMES or the gap); intact and waiting out the gap (stale_after) with nothing reaching them, which // the VRAM pressure takes; intact, waiting it out and still reached enum { Y_NEW, Y_GPU, Y_GAP, Y_REACHED, YKINDS }; uint64_t yby[YKINDS] = {0}, gap_max = 0; int yn[YKINDS] = {0}; for (const Tex *e = rel_head; e; e = e->rel_next) { if (!e->bytes) continue; int shown = e->ph && pref[e->ph - 1].refs; int k = now - e->rel_frame < stale_after(e) ? YOUNG : now - e->samp_frame < STALE_FRAMES ? SAMPLED : shown ? (e->overwritten ? SLOT_OVER : SLOT) : !e->overwritten && !idle(e, now) ? LOOKED : NEXT; by[k] += e->bytes, n[k]++; if (!comes_back_intact(e)) gpu += e->bytes, ngpu++; if (k == YOUNG) { int y = now - e->rel_frame < STALE_FRAMES ? Y_NEW : !comes_back_intact(e) && !e->overwritten ? Y_GPU : idle(e, now) ? Y_GAP : Y_REACHED; yby[y] += e->bytes, yn[y]++; if (y >= Y_GAP && stale_after(e) > gap_max) gap_max = stale_after(e); } } const double M = 1048576.0; rt_log("vk: released textures over their bound%s: %d entries, %.0f MiB of %.0f MiB (%d of them, %.0f MiB, hold content " "only the GPU made): %d, %.0f MiB, released too lately to go; %d, %.0f MiB, sampled or drawn into lately; %d, " "%.0f MiB, shown by a texture pool slot and not sampled, and %d more, %.0f MiB, that another target was drawn " "over; %d, %.0f MiB, reached by another lookup; %d, %.0f MiB, next to go", room ? " (kept past it while the card has room)" : "", nreleased, rel_bytes / M, cap / M, ngpu, gpu / M, n[YOUNG], by[YOUNG] / M, n[SAMPLED], by[SAMPLED] / M, n[SLOT], by[SLOT] / M, n[SLOT_OVER], by[SLOT_OVER] / M, n[LOOKED], by[LOOKED] / M, n[NEXT], by[NEXT] / M); if (n[YOUNG]) rt_log("vk: the ones released too lately wait: %d, %.0f MiB, the first %llu presents after a release (2 s at the " "%u presents a second the waits count); %d, %.0f MiB, with content only the GPU made, %llu presents or their own " "gap; %d, %.0f MiB, intact, for 8 times their longest gap before a take-back (%llu presents at most) with " "nothing reaching them: the VRAM pressure takes these; %d, %.0f MiB, intact, for their gap and still " "reached", yn[Y_NEW], yby[Y_NEW] / M, (unsigned long long)STALE_FRAMES, tex_rate * 60 >> 8, yn[Y_GPU], yby[Y_GPU] / M, (unsigned long long)GPU_STALE_FRAMES, yn[Y_GAP], yby[Y_GAP] / M, (unsigned long long)gap_max, yn[Y_REACHED], yby[Y_REACHED] / M); } static void sweep_pressure(void) { // tex_lock held if (vram_pressure < 0) { const char *s = cfg_get("NX2NAVI_TEX_VRAM_PRESSURE"); vram_pressure = s && *s ? atoi(s) : VRAM_PRESSURE_DEFAULT; if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = VRAM_PRESSURE_DEFAULT; tex_evict = rt_knob("TEX_EVICT", "1"); rt_log("vk: textures under VRAM pressure: %s%d%%%s", vram_pressure ? "dropped above " : "off", vram_pressure, !vram_pressure ? "" : tex_evict ? ", released ones first, then idle live ones (NX2NAVI_TEX_EVICT=0: released only)" : ", released ones only (NX2NAVI_TEX_EVICT=0)"); if (vram_pressure > 87) rt_log("vk: NX2NAVI_TEX_VRAM_PRESSURE=%d is above the seven eighths of the budget past which a new VRAM " "mirror is refused: textures make room only once a mirror was refused", vram_pressure); } retire_sweep(); if (submit_epoch & 7) return; // the budget and commit queries every 8 submits uint64_t used, budget; int exact, dropped = 0; uint64_t wanted = tex_evict ? (uint64_t)plat_xchg64(&room_wanted, 0) : 0; int known = vram_pressure && (rel_head || tex_evict || wanted) && (tex_evict ? vk_mem_vram_live(&used, &budget) : vk_mem_vram(&used, &budget, &exact)); uint64_t over = 0; if (known) { uint64_t limit = budget / 100 * (uint64_t)vram_pressure; if (used > limit) over = used - limit + budget / 50; if (wanted) { // the use at which the refused mirror fits; the lowest of the requests that stand uint64_t level = used > wanted ? used - wanted : 0; if (!room_checks || level < room_level) room_level = level; room_checks = ROOM_CHECKS; } if (room_checks && used <= room_level) room_checks = 0; // made: the pool gets its mirror when it asks again if (room_checks) { room_checks--; if (used - room_level > over) over = used - room_level; } } if (over) { // Released entries that come back intact, by the rule drop_stale keeps: released STALE_FRAMES ago and reached by // no lookup since (idle). The game releases and takes back hundreds of textures a frame in towns, and the // release list is mostly those: dropped, each was made and uploaded again within frames and freed nothing. // With the budget taken as 7.5 GiB (NX2NAVI_VRAM_BUDGET_MB), teleports from Hyrule Field through Kakariko // Village, the Graveyard, Death Mountain Trail and Kokiri Forest dropped 82,288 under the pressure and the card // stayed over its budget throughout; with this rule 34,699 went, a fifth fewer textures were made again, the // render thread waited a third less on textures, and 4 draws met a texture gone where 11 had. // Each check looks at up to EVICT_LOOK entries from where the last one stopped (press_at; rel_unlink moves it // off an entry that leaves), and starts from the oldest again at the first entry released less than // STALE_FRAMES ago or at the list's end: a walk from the head passed every entry it could not drop, up to // RELEASED_HARD of them, at every check. uint64_t freed = 0, now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); Tex *e = press_at ? press_at : rel_head; for (int n = 0; n < EVICT_LOOK && dropped < PRESSURE_STEP; n++) { if (!e || now - e->rel_frame < STALE_FRAMES) { // the rest is younger: from the oldest next time e = NULL; break; } Tex *next = e->rel_next; if (comes_back_intact(e) && idle(e, now)) { freed += e->bytes; drop_tex(e, DROP_VRAM); dropped++, n_pressure_drops++; } e = next; } press_at = e; if (tex_evict && freed < over) freed += evict_idle(over - freed, used, budget, DROP_IDLE_VRAM); if (tex_evict && freed < over) evict_disused(over - freed); } int low = commit_is_low(), more = 0; // more: the released entries have stale ones left for the next check int held = low && commit_drops_held(); // the commit drops came back (commit_back_note) if (rel_head) { // The driver's figure, holes in VMA's blocks counted: the pressure above goes by the memory in use without // them (vk_mem_vram_live), and a new block or an image with memory of its own cannot use a hole. A card // at 100 % of its budget by this figure was at 81 % by the other, 3.1 GiB of it holes, and nothing gave way. uint64_t cap = released_cap_bytes(), keep = cap, vused = 0, vbudget = 0; int vexact, vknown = vk_mem_vram(&vused, &vbudget, &vexact); over_share = vknown && vram_pressure && vused > vbudget / 100 * (uint64_t)vram_pressure; drop_overwritten(over_share); drop_aged(); // the bound, or more of it while the hold empties the released entries within minutes and the card and the // system have room (vk_tex_release.h trl_keep): a young entry dropped there is one the game may still take back if (rel_bytes > cap && released_hold() && vknown) keep = trl_keep(released_hold(), cap, trl_room(vused, vbudget, vram_pressure ? vram_pressure : VRAM_PRESSURE_DEFAULT, commit_free_now, commit_low_mark)); if (rel_bytes > keep) drop_stale(keep, &n_cap_drops, DROP_CAP); if (keep != UINT64_MAX && rel_bytes > keep + keep / 4) rel_census(keep, 0); // well over it: what holds the rest else if (keep > cap && rel_bytes > cap) rel_census(cap, 1); // kept past the bound while the card has room if (rel_bytes > COMMIT_LOW_KEEP && low && !held) more = drop_stale(COMMIT_LOW_KEEP, &n_commit_drops, DROP_COMMIT) == PRESSURE_STEP; } if (low && !more && !held) commit_evict(); } // ---------------------------------------------------------------- per-frame texture work (vk.h VK_TS_*) // Sums for the frame being made, by the render and replay threads (atomics: both may add at once). vk_texstat_frame, // called at each present (perf.c ft_frame), takes them; a frame whose texture work (creation, upload and upload flush) // reaches NX2NAVI_TEXUP_LOG_MS gets a "texup:" line, so an area change's hitch shows where its time went (default 0: // no lines; 16 is a useful value). static volatile int64_t ts_ns[VK_TS_N], ts_n[VK_TS_N], ts_bytes[VK_TS_N]; static int ts_log_ms = -1; static int ts_thread(void) { uint32_t tid = plat_thread_id(); return tid == prof_render_tid || tid == prof_replay_tid; } void vk_texstat_time(int kind, uint64_t t0) { if (!t0 || !ts_thread()) return; plat_add64(&ts_ns[kind], (int64_t)(kern_now_ns() - t0)); plat_inc64(&ts_n[kind]); } void vk_texstat_add(int kind, uint64_t n, uint64_t bytes) { if (!ts_thread()) return; plat_add64(&ts_n[kind], (int64_t)n); plat_add64(&ts_bytes[kind], (int64_t)bytes); } static void compact_frame_time(uint64_t d); void vk_texstat_frame(uint64_t frame, uint64_t d) { if (compact_on()) compact_frame_time(d); vk_astc_scratch_trim(); if (ts_log_ms < 0) { ts_log_ms = cfg_int("NX2NAVI_TEXUP_LOG_MS"); if (ts_log_ms < 0) ts_log_ms = 0; } int64_t ns[VK_TS_N], n[VK_TS_N], b[VK_TS_N]; for (int k = 0; k < VK_TS_N; k++) ns[k] = plat_xchg64(&ts_ns[k], 0), n[k] = plat_xchg64(&ts_n[k], 0), b[k] = plat_xchg64(&ts_bytes[k], 0); double work = (ns[VK_TS_CREATE] + ns[VK_TS_UPLOAD] + ns[VK_TS_FLUSH]) / 1e6; static int32_t n_lines; if (!ts_log_ms || work < ts_log_ms || ++n_lines > 400) return; rt_log("texup: frame %llu: %.1f ms frame, texture work %.1f ms; created %lld in %.1f ms; uploads %lld (%.1f MiB) in " "%.1f ms: staging %.1f ms (%lld new buffers, %.1f MiB), storage copy %.1f ms, ASTC record %.1f ms (%lld " "scratch buffers made in %.1f ms); upload flush %lld in %.1f ms; new device memory of 32 MiB or more %lld " "(%.0f MiB)", (unsigned long long)frame, d / 1e6, work, (long long)n[VK_TS_CREATE], ns[VK_TS_CREATE] / 1e6, (long long)n[VK_TS_UPLOAD], b[VK_TS_UPLOAD] / 1048576.0, ns[VK_TS_UPLOAD] / 1e6, ns[VK_TS_STAGE] / 1e6, (long long)n[VK_TS_STG_NEW], b[VK_TS_STG_NEW] / 1048576.0, ns[VK_TS_COPY] / 1e6, ns[VK_TS_ASTC] / 1e6, (long long)n[VK_TS_SCRATCH], ns[VK_TS_SCRATCH] / 1e6, (long long)n[VK_TS_FLUSH], ns[VK_TS_FLUSH] / 1e6, (long long)n[VK_TS_DEVMEM], b[VK_TS_DEVMEM] / 1048576.0); } // ---------------------------------------------------------------- uploads static uint32_t texel_alpha_one(int format, uint32_t *elem) { switch (format) { case 0x19: *elem = 1; return 0xFF; case 0x1A: *elem = 1; return 0x7F; case 0x1B: case 0x1C: *elem = 1; return 1; case 0x1D: *elem = 2; return 0x3C00; case 0x1E: *elem = 2; return 0xFFFF; case 0x1F: *elem = 2; return 0x7FFF; case 0x20: case 0x21: *elem = 2; return 1; case 0x22: *elem = 4; return 0x3F800000; default: *elem = 4; return 1; // 0x23, 0x24 } } // 3-component texels to 4 (n texels). static void expand_rgb(int format, const uint8_t *src, uint8_t *dst, size_t n) { uint32_t e, one = texel_alpha_one(format, &e); for (size_t i = 0; i < n; i++, src += 3 * e, dst += 4 * e) { memcpy(dst, src, 3 * e); memcpy(dst + 3 * e, &one, e); } } static void shrink_rgb(int format, const uint8_t *src, uint8_t *dst, size_t n) { uint32_t e; texel_alpha_one(format, &e); for (size_t i = 0; i < n; i++, src += 4 * e, dst += 3 * e) memcpy(dst, src, 3 * e); } // NX2NAVI_ASTC=check: the compute decoder's texels of n regions (gpu: the regions one after the other) against // astc_decode_ldr of the same raw blocks (raw: the staging bytes the regions point into). static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar, int n, const uint8_t *gpu) { static volatile int32_t n_bad; static uint64_t n_tex, n_texels, n_diff_tex; uint32_t bw = e->f.astc_bw, bh = e->f.astc_bh; uint64_t diff = 0, first = UINT64_MAX; uint32_t want = 0, got = 0; int first_k = 0; for (int k = 0; k < n; k++) { const VkAstcRegion *r = &ar[k]; size_t sb = (size_t)((r->w + bw - 1) / bw) * ((r->h + bh - 1) / bh) * 16, st = (size_t)r->w * r->h * 4; uint8_t *cpu = malloc(st); for (uint32_t z = 0; z < r->slices; z++, gpu += st) { astc_decode_ldr(raw + r->src + z * sb, r->w, r->h, bw, bh, e->f.astc_srgb, cpu); if (!memcmp(cpu, gpu, st)) continue; for (size_t i = 0; i < st / 4; i++) if (memcmp(cpu + 4 * i, gpu + 4 * i, 4)) { if (first == UINT64_MAX) { first = i, first_k = k; memcpy(&want, cpu + 4 * i, 4), memcpy(&got, gpu + 4 * i, 4); } diff++; } } free(cpu); n_texels += (uint64_t)r->w * r->h * r->slices; } n_tex++; if (diff) { n_diff_tex++; log_limited(&n_bad, "ASTC check: texture %#llx fmt %#x %ux%u: %llu texels differ, first region %d texel %llu " "cpu %08x gpu %08x", (unsigned long long)e->pub.addr, e->t.d.format, bw, bh, (unsigned long long)diff, first_k, (unsigned long long)first, want, got); } if (n_tex == 1 || !(n_tex & 63) || diff) rt_log("vk: ASTC check: %llu textures, %llu texels compared, %llu textures differ", (unsigned long long)n_tex, (unsigned long long)n_texels, (unsigned long long)n_diff_tex); } // Staging bytes of one part of a texture upload decoded or copied on the CPU (see upload). With NX2NAVI_ASTC=cpu a // large ASTC array decodes to RGBA8 in one go: a staging buffer of 676682336 bytes failed (-2) on an Intel UHD with // 8 GB of shared memory and the game closed before the title screen. #define UPLOAD_PART (32ull << 20) // fits the staging pool (vk_staging takes up to half of its 64 MiB block) // A texture of at most CHASH_MAX bytes keeps a hash of its storage as it was uploaded. A write to its pages then // re-uploads it only when its own bytes changed: Ocarina's frames rewrite hundreds of small textures' pages (shared // with data the CPU writes, or written with the same bytes), about 420 re-uploads a frame of 1.5 KiB each, and one // costs a staging buffer, a copy and a barrier, several microseconds, where hashing 1.5 KiB takes a fraction of one. // A CPU write during the hash sets the pages' write bits again, so the next check sees it. // // NX2NAVI_TEX_SAME_BIG (default 1 since 0.11.0; ab): textures up to CHASH_BIG_MAX keep the hash too. While the player runs through // a town the game moves about 32 textures to new storage every four or five frames, back to back in one block, and a // frame after their first upload every page of the block shows a write again. Of the ones the hash covers, 0 to 4 in a // hundred are uploaded again (128x128, 171x171, 256x256 BC1: 9 of 440 in one recording of 600 frames); of the others, // whatever their size and format, 75 to 94 in a hundred (256x256 BC3, 341x341 BC1, everything from 512x512 up): 3,034 // uploads in those 600 frames beside 4,233 first ones, their ASTC ones decoded and encoded on the GPU a second time. // The frames of those second uploads take 0.9 to 2.5 ms longer than the frames without uploads and the frames after // them wait 0.3 to 1.3 ms longer for the GPU (six recordings of one player, RTX 5090, 2026-11-12). // What the hash costs: the bytes are read once more at the first upload and once at the comparison, 15 to 18 us each // for 300 KiB (17 to 20 GB/s, below), together about what the second upload takes on the submitting thread (27 us // for 300 KiB). The gain is the GPU's work, and a run through Hyrule Field measured it (2026-11-14, RTX 5090, about // three minutes of running, the same route each time): without the hash 44,000 to 52,000 second uploads of 33 to 47 // GiB of storage in four runs of six (9,000 and 11,000 in the other two), one 1536x1536 texture 29 times within 300 // frames with the same bytes each time, 66 thousand million texels decoded on the GPU in a run; with it 592 second // uploads of 195 MiB, 24 GiB found unchanged, and 24 thousand million texels decoded, which is the first uploads. // The second uploads took 2.9 s of the submitting thread in such a run, the two hashes about 3 s. A texture the CPU // really rewrites would pay the hash for nothing each time: after CHASH_MISSES comparisons in a row that found other // bytes its uploads take no hash any more. An area load takes none either (vk_exec_pool_loading): it uploads // gigabytes in a few frames. NX2NAVI_TEX_SAME_BIG=0 hashes only the textures up to CHASH_MAX, as before 0.11.0. // The pool log's buckets line counts the comparisons and the hashed bytes. enum { CHASH_MAX = 64 << 10, CHASH_BIG_MAX = 8 << 20, CHASH_MISSES = 2 }; static uint64_t content_hash(const void *p, uint64_t n) { const uint8_t *b = p; uint64_t h = 0x9e3779b97f4a7c15ull ^ n, i = 0; // Eight chains side by side over 64 bytes a step, folded into one: a chain's multiply waits for the one before it, // so the chains, which wait for nothing of each other, read eight times what one reads in the same time. A step is // a bijection of its chain for the word it takes and of the word for the chain, as the single chain's is, and so // is each fold. Measured on textures read from memory (Ryzen 7 9850X3D, clang -O2): one chain 7 GB/s, four 10, // eight 17 to 20, sixteen no more; a copy of the same bytes runs at 70. if (n >= 512) { uint64_t c[8]; for (int k = 0; k < 8; k++) c[k] = h ^ (0xc2b2ae3d27d4eb4full * (uint64_t)(k + 1)); for (; i + 64 <= n; i += 64) { uint64_t w[8]; memcpy(w, b + i, 64); for (int k = 0; k < 8; k++) { c[k] = (c[k] ^ w[k]) * 0xbf58476d1ce4e5b9ull; c[k] ^= c[k] >> 29; } } for (int k = 0; k < 8; k++) { h = (h ^ c[k]) * 0x94d049bb133111ebull; h ^= h >> 29; } } for (; i + 8 <= n; i += 8) { uint64_t w; memcpy(&w, b + i, 8); h = (h ^ w) * 0xbf58476d1ce4e5b9ull; h ^= h >> 29; } for (; i < n; i++) h = (h ^ b[i]) * 0x94d049bb133111ebull; return h ^ (h >> 31); } // ASTC for the GPU decoder without the CPU deswizzle: the storage goes into staging as the game laid it out, in one // copy, and the decoder reads each block from its block-linear place (VkAstcRegion.bl, astc_decode.comp bl_block). // The CPU deswizzle moved every 16 bytes through the GOB arithmetic and was most of an ASTC upload's CPU time when an // area loads. Fills ar (n = levels * layers, level-major as upload's regions) and returns 1 when every level and layer // qualifies: block-linear, starting 16-byte aligned, its layout inside the storage. NX2NAVI_ASTC_GPU_DESWIZZLE=0 keeps // the CPU deswizzle. levels counts the image's levels: level lv of the image is the game's level lv + skip (texture // detail), which is where its blocks lie and what its size is. static int astc_bl_regions(const Tex *e, uint32_t levels, uint32_t layers, int bw, int bh, VkAstcRegion *ar) { static int on = -1; if (on < 0) { on = cfg_bool("NX2NAVI_ASTC_GPU_DESWIZZLE"); if (!on) rt_log("vk: ASTC blocks deswizzled on the CPU (NX2NAVI_ASTC_GPU_DESWIZZLE=0)"); } const nvn_texdesc *d = &e->t.d; if (!on || nvn_tex_is_linear(d)) return 0; uint32_t k = 0, sk = e->pub.skip; for (uint32_t lv = 0; lv < levels; lv++) for (uint32_t ly = 0; ly < layers; ly++) { uint32_t row_bytes, rows, slices, bpb; uint64_t base; unsigned th, td; if (nvn_deswizzle_geom(d, lv + sk, ly, &row_bytes, &rows, &slices, &base, &th, &td, &bpb) || (base & 15) || th > 5 || td > 5) return 0; ar[k] = (VkAstcRegion){.src = base, .w = mip(d->width, lv + sk), .h = mip(d->height, lv + sk), .slices = slices, .level = lv, .layer = ly, .bl = 1, .th = (uint8_t)th, .td = (uint8_t)td}; if (base + vk_astc_region_bytes(&ar[k], (uint32_t)bw, (uint32_t)bh) > e->t.L.size) return 0; k++; } return 1; } // NX2NAVI_ASTC_ARENA=1 (the default): the raw ASTC blocks for the GPU decoder stage in a // range of the staging arena (vk_staging_at) like the other formats, not in a VkBuffer of their own. vk_astc_record // reads its source only through a buffer copy at each region's src, so the range's offset is added to the regions. A // buffer per texture cost a vmaCreateBuffer and, after the batch, a vmaDestroyBuffer each; an area load uploads // thousands. The check mode keeps its own buffer (astc_check reads the blocks at the regions' src offsets). 0: a buffer // per texture. With the kept scratch, a Kakariko walk at 1440p (2026-11-11) went from 342 device memory allocations // and 6 frames over 50 ms to 233 and 3. static int astc_arena(void) { static int on = -1; if (on < 0) { on = cfg_bool("NX2NAVI_ASTC_ARENA"); rt_log("vk: ASTC blocks for the GPU decoder stage in %s (NX2NAVI_ASTC_ARENA=%d)", on ? "the staging arena" : "a buffer per texture", on); } return on; } // Copies the whole storage into the image (every level and layer the image has). An image made without its top levels // (texture detail, VkTexImage.skip) has the game's level lv + skip at its level lv: the loops below run over the image's // levels and read the storage at the game's. // How long an upload whose decode had no memory waits for its next try (upload, up_wait): half a second of frames. A // try costs the copy of the texture's blocks into staging memory, so not every frame. #define UP_RETRY_FRAMES 30 static uint64_t n_up_retry; uint64_t vk_texture_upload_retries(void) { return __atomic_load_n(&n_up_retry, __ATOMIC_RELAXED); } // 1 while a texture that is not uploaded waits for its next try (e->up_retry; 0 for one never tried). static int up_wait(const Tex *e) { return e->up_retry && __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) < e->up_retry; } // ASTC as BC1: e gets an image of the other format, made with the same parameters, in place of its own: BC1 (bc 2) for // the opaque content of its first upload, BC3 (bc 1) for content with alpha about to go into its BC1 image. The old // image and its views go once the GPU is past the submission being recorded, as a destroyed texture's do (destroy_tex), // so commands recorded with it keep it; the new one starts as a new image (init_image) in the upload batch, which runs // before the part being recorded (in cb, the batch upload has open, or its own), and the slots that resolve to e look // it up again (tex_gen_bump). 0 when no image could be made; e keeps its own. static int astc_reformat(Tex *e, VkCommandBuffer cb, int bc) { static volatile int32_t n_failed; VkCtx *c = vk_ctx(); const ImgKey *k = &e->ik; int s = e->f.astc_srgb; VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = k->flags, .imageType = (VkImageType)k->type, .format = bc == 2 ? (s ? VK_FORMAT_BC1_RGB_SRGB_BLOCK : VK_FORMAT_BC1_RGB_UNORM_BLOCK) : (s ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK), .extent = {k->w, k->h, k->d}, .mipLevels = k->levels, .arrayLayers = k->layers, .samples = (VkSampleCountFlagBits)k->samples, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = k->usage, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; ImgKey nk; img_key_of(&ci, &nk); VmaAllocation a = NULL; VkImage img = suballoc_image(c, &ci, &a); if (!img && vma_image(c, &ci, &nk, &img, &a) != VK_SUCCESS) { log_limited(&n_failed, "ASTC texture %#llx (%ux%u): no %s image could be made", (unsigned long long)e->pub.addr, e->t.d.width, e->t.d.height, bc == 2 ? "BC1" : "BC3"); return 0; } vk_exec_image_gone(e->pub.image); vk_defer_image(e->pub.image, e->alloc); vk_exec_image_gone(img); // views cached under the handle of a destroyed image (create_tex) int relink = e->ru_on; if (relink) ru_unlink(e); if (e->released) rel_bytes -= e->bytes; if (e->retiring) retiring_bytes -= e->bytes; e->pub.image = img, e->alloc = a, e->reused = 0; own_memory(c, e); if (e->released) rel_bytes += e->bytes; if (e->retiring) retiring_bytes += e->bytes; e->pub.format = e->f.vk = ci.format; e->f.astc_bc = (uint8_t)bc; e->ik = nk; if (relink) ru_link(e); if (cb) { init_image(e, cb); } else { init_image(e, vk_upload_begin()); vk_upload_end(); } tex_gen_bump(e->key.addr); if (bc == 2) n_bc1_made++, n_bc1_made_bytes += e->bytes; else n_bc1_widened++; return 1; } // The upload's copy of size bytes of ASTC storage into staging, with the opaque check of its blocks: 1 when every block // is opaque and *decoded stays within ASTC_OPAQUE_DECODES. A large texture's copy and check run on the ASTC decode // workers (tex_astc_mt.c astc_blocks_opaque_mt), a small one's here. static uint64_t n_bc1_up_checks, n_bc1_up_bytes; static int astc_copy_opaque(const Tex *e, uint8_t *dst, const uint8_t *src, uint64_t size, uint32_t *decoded) { uint64_t nb = size / 16; int ok = astc_blocks_opaque_mt(src, nb, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb != 0, ASTC_OPAQUE_DECODES, decoded, dst); if (size % 16) memcpy(dst + 16 * nb, src + 16 * nb, size % 16); n_bc1_up_checks++, n_bc1_up_bytes += size; return ok; } // Whether an upload of e takes the hash of its storage: always up to CHASH_MAX; above it, up to CHASH_BIG_MAX, with // NX2NAVI_TEX_SAME_BIG, outside an area load, and while the comparisons have not found other bytes CHASH_MISSES times // in a row ("NX2NAVI_TEX_SAME_BIG" above content_hash). static int chash_wanted(const Tex *e) { static int mode = -1; if (e->t.L.size <= CHASH_MAX) return 1; if (mode < 0) mode = rt_knob("TEX_SAME_BIG", "1"); return RT_KNOB_ON(mode) && e->t.L.size <= CHASH_BIG_MAX && e->chash_miss < CHASH_MISSES && !vk_exec_pool_loading(); } // The host temporaries of the CPU's texture work, counted for the memory report (vk_texture_host): the copies an upload // decodes ASTC or expands RGB from (upload), the tight ASTC blocks a texel write decodes (image_texels) and the texels // a write-back reads (write_back_n). Each lives for one upload or write-back; the report gives the bytes in // use when it asks and the most at once since it last asked. Two atomics per temporary, not per frame. static volatile int64_t tmp_now, tmp_peak; static void *tmp_alloc(size_t n) { void *p = malloc(n); if (!p) return NULL; int64_t now = __atomic_add_fetch(&tmp_now, (int64_t)n, __ATOMIC_RELAXED); int64_t pk = __atomic_load_n(&tmp_peak, __ATOMIC_RELAXED); while (now > pk && !__atomic_compare_exchange_n(&tmp_peak, &pk, now, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} return p; } static void tmp_free(void *p, size_t n) { if (!p) return; free(p); __atomic_sub_fetch(&tmp_now, (int64_t)n, __ATOMIC_RELAXED); } static void upload(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; int no_mem = 0; // a part's ASTC decode had no memory: the texture is uploaded again later const tex_t *t = &e->t; const nvn_texdesc *d = &t->d; int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); if (!bpb || !t->mem) return; // A stale slot (the bindless pools look every slot up): no upload; a later write to mapped storage uploads it // (make_fresh_snap). if (!mem_readable(e->pub.addr, e->t.L.size)) { static volatile int32_t n_unmapped; log_limited(&n_unmapped, "texture %#llx (%llu bytes): storage not mapped; not uploaded", (unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size); e->uploaded = 1; return; } e->chash_ok = (uint8_t)chash_wanted(e); // the hash of what this upload copies (make_fresh_snap compares it) if (e->chash_ok) { // read with the storage held mapped, as the copies below: the game unmaps a texture's memory on its own thread if (mem_read_hold(e->pub.addr, e->t.L.size)) { e->chash = content_hash((const void *)(uintptr_t)e->pub.addr, e->t.L.size); mem_read_release(); if (e->t.L.size > CHASH_MAX) big_hash_n++, big_hash_bytes += e->t.L.size; } else { e->chash_ok = 0; } } static __thread char ctx[160]; // what this thread copies, for a crash report snprintf(ctx, sizeof ctx, "texture upload %#llx (%llu bytes, fmt %#x %ux%u)", (unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size, d->format, d->width, d->height); uint64_t p0 = perf_begin(), u0 = kern_now_ns(); int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D; uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers; int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb; int agpu = astc && vk_astc_gpu(), acheck = agpu && vk_astc_mode() == VK_ASTC_CHECK; // raw blocks to the GPU uint32_t n = levels * layers, k = 0, sk = linear ? 0 : e->pub.skip; VkBufferImageCopy *rg = calloc(n, sizeof *rg); VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL; VkDeviceSize total = 0, decoded = 0; uint64_t src_bytes = 0; // the storage bytes the image's levels take (all of the storage unless levels are left out) for (uint32_t lv = 0; lv < levels; lv++) for (uint32_t ly = 0; ly < layers; ly++) { uint32_t w = mip(d->width, lv + sk), h = one_d ? 1 : mip(d->height, lv + sk), sl = is3d ? mip(d->depth, lv + sk) : 1; uint64_t texels = (uint64_t)w * h * sl, raw = (uint64_t)(w + bw - 1) / bw * bpb * ((h + bh - 1) / bh) * sl; uint64_t out = agpu ? raw : astc ? texels * 4 : rgb ? texels * (uint64_t)bpb / 3 * 4 : raw; rg[k++] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {e->pub.aspect, lv, ly, 1}, .imageExtent = {w, h, sl}}; total += (out + 15) & ~15ull; decoded += texels * 4; src_bytes += raw; } if (!sk) src_bytes = t->L.size; VkBuffer sb, rbuf = VK_NULL_HANDLE; VmaAllocation ralloc = NULL; uint8_t *rmap = NULL, *raw_copy = NULL; int checked = 0, parts = 1; // The GPU decoder reads the blocks in place (astc_bl_regions); the check mode compares against tight blocks. int gdsw = agpu && !acheck && !one_d && astc_bl_regions(e, levels, layers, bw, bh, ar); // ASTC as BC1: a BC1 image takes opaque content only, and a BC3 one that may turn BC1 (bc1_try) does at its // first upload when the content is opaque (astc_reformat). The blocks the GPU decoder takes as they lie (gdsw) are // checked as they are copied; the others are checked here first. Without memory for a BC3 image the upload waits as // one without memory for its decode does (no_mem); without memory for a BC1 one the texture stays BC3. int bc1 = e->f.astc_bc == 2, try1 = e->bc1_try && e->f.astc_bc == 1 && !e->n_up, opq = 1; uint32_t bc1_dec = 0; e->bc1_try = 0; if ((bc1 || try1) && !gdsw) { opq = astc_opaque(t, e->pub.addr, &e->f); if (bc1 && !opq && !astc_reformat(e, VK_NULL_HANDLE, 1)) { e->uploaded = 0; e->up_retry = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + UP_RETRY_FRAMES; __atomic_add_fetch(&n_up_retry, 1, __ATOMIC_RELAXED); free(ar), free(rg); return; } if (try1 && opq) astc_reformat(e, VK_NULL_HANDLE, 2); bc1 = try1 = 0; } int check = bc1 || try1; // A CPU decode or copy stages at most UPLOAD_PART bytes per staging buffer (a region larger than that alone) and // records one copy per buffer into the open upload batch (vk_upload_end may submit a batch that holds much staging // between two parts; queue order and the barriers keep them in order). The GPU decoder takes the raw blocks of // every region from one buffer. for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) { VkDeviceSize base = rg[k0].bufferOffset, size; for (k1 = k0 + 1; k1 < n && (agpu || (k1 + 1 < n ? rg[k1 + 1].bufferOffset : total) - base <= UPLOAD_PART);) k1++; size = (k1 < n ? rg[k1].bufferOffset : total) - base; if (gdsw) size = t->L.size; // the storage as it is (one part: the GPU decoder takes every region at once) // Without its top levels, only the levels the image has: each region's blocks as they lie in the storage, one // after the other (the top level is three quarters of the bytes, and an array keeps each layer's levels // together, so the kept levels are no single range). if (gdsw && sk) { size = 0; for (k = 0; k < n; k++) size += vk_astc_region_bytes(&ar[k], (uint32_t)bw, (uint32_t)bh); src_bytes = size; } if (k0) parts++; VkCommandBuffer cb = vk_upload_begin(); // The part stages in a range of the staging arena (vk_staging_at, its offset added to the regions below), or in // a buffer of its own at offset 0 when larger than an arena range; the GPU decoder takes one with // NX2NAVI_ASTC_ARENA=0 or in the check mode (astc_arena). VkDeviceSize soff = 0; uint64_t s0 = kern_now_ns(); uint8_t *stg = agpu && (acheck || !astc_arena()) ? vk_staging(size, &sb, NULL) : vk_staging_at(size, &sb, &soff); vk_texstat_time(VK_TS_STAGE, s0); uint64_t c0 = kern_now_ns(); // the storage copies and decodes of this part (VK_TS_COPY) if (acheck) rmap = vk_staging(decoded, &rbuf, &ralloc); // The part's copies out of the storage, with it held mapped (mem_read_hold): the game releases a texture's // memory on its own thread, and a decommit in the middle of a copy faulted (tex_bl_copy). Only the copies run // held, so a waiting unmap and the guest's memory calls behind it never wait for the staging block or a decode. // Storage released since the check above uploads as zeros, as the game no longer has that texture. uint8_t **tmps = calloc(k1 - k0, sizeof *tmps); int held = mem_read_hold(e->pub.addr, e->t.L.size); if (held) plat_crash_context(ctx); else { static volatile int32_t n_gone; log_limited(&n_gone, "texture %#llx (%llu bytes): storage released during the upload; uploaded as zeros", (unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size); // the image holds zeros, not the bytes hashed above: the same bytes mapped back there upload again e->chash_ok = 0; } if (gdsw && !sk) { if (held && check) opq = astc_copy_opaque(e, stg, t->mem, size, &bc1_dec); else if (held) par_memcpy(stg, t->mem, size); // large blocks over the par_jobs pool else memset(stg, 0, size); } else if (gdsw) { VkDeviceSize at = 0; for (k = 0; k < n; k++) { // astc_bl_regions checked that each lies inside the storage VkDeviceSize nb = vk_astc_region_bytes(&ar[k], (uint32_t)bw, (uint32_t)bh); if (held && check) opq &= astc_copy_opaque(e, stg + at, t->mem + ar[k].src, nb, &bc1_dec); else if (held) par_memcpy(stg + at, t->mem + ar[k].src, nb); else memset(stg + at, 0, nb); ar[k].src = at, at += nb; } } for (k = k0; k < k1 && !gdsw; k++) { uint32_t lv = rg[k].imageSubresource.mipLevel, ly = rg[k].imageSubresource.baseArrayLayer; uint32_t w = rg[k].imageExtent.width, h = rg[k].imageExtent.height, sl = rg[k].imageExtent.depth; uint32_t rb = (w + bw - 1) / bw * (uint32_t)bpb, rows = (h + bh - 1) / bh; size_t raw = (size_t)rb * rows * sl; rg[k].bufferOffset -= base; uint8_t *dst = stg + rg[k].bufferOffset, *tmp = (astc && !agpu) || rgb ? tmp_alloc(raw) : NULL; tmps[k - k0] = tmp; if (!held) { memset(tmp ? tmp : dst, 0, raw); } else if (nvn_deswizzle_mt(d, lv + sk, ly, t->mem, tmp ? tmp : dst, rb)) { // large levels: par_jobs.c log_limited(&n_bad, "texture %#llx fmt %#x target %d: level %u layer %u has no CPU layout; uploaded as zeros", (unsigned long long)e->pub.addr, d->format, d->target, lv + sk, ly); memset(tmp ? tmp : dst, 0, raw); } } if (held) { plat_crash_context(NULL); mem_read_release(); // the part's reads of the storage are done } for (k = k0; k < k1 && !gdsw; k++) { // the decodes, from the copies (gdsw: ar already names the blocks) uint32_t lv = rg[k].imageSubresource.mipLevel, ly = rg[k].imageSubresource.baseArrayLayer; uint32_t w = rg[k].imageExtent.width, h = rg[k].imageExtent.height, sl = rg[k].imageExtent.depth; uint32_t rb = (w + bw - 1) / bw * (uint32_t)bpb, rows = (h + bh - 1) / bh; uint8_t *dst = stg + rg[k].bufferOffset, *tmp = tmps[k - k0]; if (agpu) { ar[k] = (VkAstcRegion){.src = rg[k].bufferOffset, .w = w, .h = h, .slices = sl, .level = lv, .layer = ly}; } else if (astc) { uint64_t a0 = perf_begin(); for (uint32_t z = 0; z < sl; z++) astc_decode_ldr_mt(tmp + (size_t)z * rb * rows, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, dst + (size_t)z * w * h * 4); perf_end(PERF_ASTC, a0); rg[k].bufferRowLength = w, rg[k].bufferImageHeight = h; } else if (rgb) { expand_rgb(d->format, tmp, dst, (size_t)w * h * sl); rg[k].bufferRowLength = w, rg[k].bufferImageHeight = h; } else { rg[k].bufferRowLength = rb / (uint32_t)bpb * (uint32_t)bw, rg[k].bufferImageHeight = rows * (uint32_t)bh; } tmp_free(tmp, (size_t)rb * rows * sl); // the size the first loop asked for (raw) } free(tmps); vk_texstat_time(VK_TS_COPY, c0); if (acheck) { raw_copy = malloc(size); memcpy(raw_copy, stg, size); } barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); if (agpu) { for (k = 0; k < n; k++) ar[k].src += soff; // a range of the arena (astc_arena); 0 in a buffer of its own uint64_t a0 = perf_begin(); // content with alpha for a BC1 image: a BC3 one first, or no decode for now (-1); opaque content at a BC3 // image's first upload: a BC1 one (it stays BC3 when none can be made) int widen_failed = 0; if (bc1 && !opq) widen_failed = !astc_reformat(e, cb, 1); else if (try1 && opq) astc_reformat(e, cb, 2); checked = widen_failed ? -1 : vk_astc_record(cb, sb, size, ar, (int)n, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, e->pub.image, e->pub.aspect, rbuf); perf_end(PERF_ASTC, a0); vk_texstat_time(VK_TS_ASTC, a0); if (checked <= 0) { log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was%s", (unsigned long long)e->pub.addr, d->format, d->width, d->height, checked < 0 ? " and tried again later (no memory for the decode)" : ""); e->chash_ok = 0; // the image does not hold the bytes hashed above: the next write uploads, equal or not no_mem |= checked < 0; } } else { for (k = k0; k < k1; k++) rg[k].bufferOffset += soff; vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, k1 - k0, rg + k0); } barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_upload_end(); } if (parts > 1) log_limited(&n_split, "texture %#llx fmt %#x %ux%u, %u levels, %u layers: %.1f MiB staged in %d parts", (unsigned long long)e->pub.addr, d->format, d->width, d->height, levels, layers, total / 1048576.0, parts); if (acheck) { // waits for the GPU vk_upload_flush(1); if (checked > 0) { vmaInvalidateAllocation(vk_ctx()->vma, ralloc, 0, VK_WHOLE_SIZE); astc_check(e, raw_copy, ar, (int)n, rmap); } vmaDestroyBuffer(vk_ctx()->vma, rbuf, ralloc); } free(raw_copy); free(ar); free(rg); uint64_t dt = kern_now_ns() - u0; if (e->uploaded) { n_reuploads++, perf_count(PERF_CNT_TEX_REUP); n_split_refill += e->sfill != 0; ru_n++, ru_bytes += src_bytes, ru_ns += dt; if (e->wt_n) { ru_wt_n++, ru_wt_bytes += src_bytes, ru_wt_ns += dt; if (rt_trace & RT_TRACE_PERF) rt_log("perf: frame %llu: re-upload after %u WriteTexels: texture %#llx fmt %#x %ux%u, %.1f KiB, %.3f ms", (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), e->wt_n, (unsigned long long)e->pub.addr, d->format, d->width, d->height, src_bytes / 1024.0, dt / 1e6); } } else { n_uploads++, perf_count(PERF_CNT_TEX_UP); } flight_ev(e, e->uploaded ? FLIGHT_TEX_REUP : FLIGHT_TEX_UP, e->wt_n, (uint32_t)(dt / 1000)); flight_bytes(e); e->wt_n = 0; e->uploaded = 1; e->n_up++; // The decode had no memory (vk_astc_record: its scratch buffer failed while the video memory was used up), so the // image does not hold this content: a new image holds nothing, an older one what it held before. Counted as // uploaded, it stayed that way until the game next wrote the storage, which for most textures is never. It counts // as not uploaded instead, and make_fresh and make_fresh_snap upload it again UP_RETRY_FRAMES frames on, when // the evictions under memory pressure have had time to make room; again and again until one succeeds. if (no_mem) { e->uploaded = 0; e->up_retry = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + UP_RETRY_FRAMES; __atomic_add_fetch(&n_up_retry, 1, __ATOMIC_RELAXED); } else { e->up_retry = 0; } perf_end(PERF_TEX, p0); vk_texstat_time(VK_TS_UPLOAD, u0); vk_texstat_add(VK_TS_UPLOAD, 0, src_bytes); } // Whether e's storage fills no 4 KiB page of its own, so the write tracking can only see writes to pages it shares. static int fills_no_page(const Tex *e) { return ((e->pub.addr + e->t.L.size) & ~0xfffull) <= ((e->pub.addr + 0xfff) & ~0xfffull); } // A texture that fills no page keeps a copy of its storage bytes from when a GPU write to it was recorded // (vk_texture_gpu_written), up to SNAP_MAX bytes; wb_exact (NX2NAVI_WB_EXACT, default on) enables it. enum { SNAP_MAX = 4096 }; static int wb_exact = -1; static volatile int32_t n_kept; // vk_mem_changed over the pages wholly inside e's storage (all of its pages when it fills none): a neighbour that shares // the first or last page does not count as a write to e. A texture that fills no page shares all of its pages, so // there a page write counts only when e's own bytes differ from the copy taken at the GPU write. NX2navi's Purah Pad // camera renders the ID, screen rect and depth of what is under the reticle into 4x4 targets of 512 bytes, two // slots used in turn, and the CPU reads them a frame later. One slot's rect target shares its page with data the CPU // writes every frame: without the copy its renders were never written back, the game read a zero rect every other // frame and the target's name label blinked. NX2NAVI_WB_EXACT=0 restores the page-only decision. static int storage_rewritten(const Tex *e, uint64_t since, uint64_t *now) { uint64_t lo = (e->pub.addr + 0xfff) & ~0xfffull, hi = (e->pub.addr + e->t.L.size) & ~0xfffull; if (hi <= lo) lo = e->pub.addr, hi = e->pub.addr + e->t.L.size; int ch = vk_mem_changed(lo, hi - lo, since, now); if (ch != 1 || !since || !e->snap || memcmp(e->snap, (const void *)(uintptr_t)e->pub.addr, e->t.L.size)) return ch == 1; log_limited(&n_kept, "write-back of %#llx kept: fmt %#x %ux%u, %llu bytes; a neighbour wrote its page", (unsigned long long)e->pub.addr, e->t.d.format, e->t.d.width, e->t.d.height, (unsigned long long)e->t.L.size); return 0; } // The pages of an uploaded texture show a write (or are not tracked): whether its own bytes are still those it was // uploaded with (upload, content_hash). WriteTexels since the upload always counts as a change. static int same_content(Tex *e) { static int on = -1; // NX2NAVI_TEX_SAME=0: every write to the pages uploads again (diagnosis) if (on < 0) { const char *v = cfg_get("NX2NAVI_TEX_SAME"); on = !(v && !strcmp(v, "0")); } if (!on || !e->chash_ok || e->wt_n) return 0; // chash_ok: the upload took the hash (chash_wanted) if (!mem_read_hold(e->pub.addr, e->t.L.size)) return 0; // unmapped since: not the same; the upload deals with it uint64_t h = content_hash((const void *)(uintptr_t)e->pub.addr, e->t.L.size); mem_read_release(); int big = e->t.L.size > CHASH_MAX; if (h != e->chash) { if (big) big_diff_n++, e->chash_miss += e->chash_miss < 255; return 0; } ru_same_n++, ru_same_bytes += e->t.L.size; if (big) big_same_n++, big_same_bytes += e->t.L.size, e->chash_miss = 0; return 1; } // A mirrored texture is uploaded on first use and again when its storage pages carry a newer write generation. So is // one split fill fills (from_storage), while it is not released: the game may be unmapping a released one's storage, // and nothing keeps a slot from resolving to it as the stale rule does for a CPU-written one (slot_resolve). static void make_fresh(Tex *e) { if (!from_storage(e) || e->pub.gpu_owned || (e->sfill && e->released)) return; uint64_t size = e->t.L.size; if (!e->uploaded) { if (up_wait(e)) return; vk_mem_changed(e->pub.addr, size, 0, &e->wgen); // resets the write bits before the storage is read upload(e); return; } uint64_t now; int ch = vk_mem_changed(e->pub.addr, size, e->wgen, &now); e->wgen = now; e->last_ch = ch; if (ch && !same_content(e)) upload(e); // 1 written, -1 not tracked: both upload again, unless the bytes match } // make_fresh for draw bindings: the page check answers from this submission's write-watch snapshot (vk_memory.c // vk_mem_changed_snap) instead of a system call per texture. A write the snapshot of the texture's region missed (made // after it, during this replay) keeps its write-watch bit and is caught by the next submission's snapshot. static void make_fresh_snap(Tex *e) { if (!from_storage(e) || e->pub.gpu_owned || (e->sfill && e->released)) return; if (!e->uploaded && up_wait(e)) return; uint64_t now; n_checks++; int ch = vk_mem_changed_snap(e->pub.addr, e->t.L.size, e->uploaded ? e->wgen : 0, submit_epoch, &now); e->wgen = now; if (!e->uploaded) { upload(e); return; } e->last_ch = ch; if (ch && !same_content(e)) upload(e); } // ---------------------------------------------------------------- aliases (NX2NAVI_TEX_ALIAS) // Bytes per block and block size of the image's VkFormat (ASTC decodes to RGBA8, RGB expands to RGBA); 0 when the // VkFormat substitutes another one of unknown size. static int img_block(const Tex *e, int *bw, int *bh) { if (e->f.astc_bc) { *bw = *bh = 4; return e->f.astc_bc == 2 ? 8 : 16; } // BC1 or BC3 if (e->f.astc_bw) { *bw = *bh = 1; return 4; } int b = nvn_fmt_bpb(e->t.d.format, bw, bh); if (e->f.expand_rgb) { *bw = *bh = 1; return b / 3 * 4; } return e->f.vk == e->f.ideal ? b : 0; } // The new entry e can take src's content texel for texel: colour, one sample, the same image type and storage layout, // and image formats of one block size (0 with a reason in *why when not). Block-compressed images need the same shape; // uncompressed ones take the region both cover (alias_copy). Ocarina moves render targets whose size follows an effect // on screen, so a descriptor made over a live target's memory is often a few texels larger or smaller than it. The game // reads the bytes there as its content (the same memory on the console); started empty instead, the scene went black // for a frame about once a second while the camera turned, from 0.6.1 on (issue #253). static int alias_compatible(const Tex *e, const Tex *src, const char **why) { const VkTexImage *p = &e->pub, *q = &src->pub; if (e->f.depth || e->f.stencil || src->f.depth || src->f.stencil || p->samples > 1 || q->samples > 1) return *why = "depth, stencil or multisample", 0; if (p->type != q->type) return *why = "another image type", 0; if (p->skip || q->skip) return *why = "made without its top levels (texture detail)", 0; // level i is not level i if (nvn_tex_is_linear(&e->t.d) != nvn_tex_is_linear(&src->t.d)) return *why = "linear and block-linear", 0; int ew, eh, sw, sh, eb = img_block(e, &ew, &eh), sb = img_block(src, &sw, &sh); if (!eb || eb != sb || ew != sw || eh != sh) return *why = "another block size", 0; if (ew == 1 && eh == 1) return 1; if (p->width != q->width || p->height != q->height || p->depth != q->depth || e->t.L.size != src->t.L.size) return *why = "compressed, another shape", 0; return 1; } // src holds content the storage bytes do not: the GPU wrote it and, for a mirrored texture, it was not written back // and the CPU has not rewritten the storage since (the new entry's upload then reads the newer bytes). static int alias_has_gpu_content(const Tex *src) { if (!src->pub.gpu_owned) return 0; if (!src->pub.mirrored) return 1; uint64_t now; return src->gpu_dirty && !storage_rewritten(src, src->gpu_gen, &now); } // Maxwell reads the same bytes through both descriptors: src's content goes into the new image e at the replay's // current position, before any later command of the submission uses e. Through a buffer, since bound images share // their memory and image copies may not overlap; e leaves UNDEFINED between the two halves, so its first transition // cannot clobber src's bytes before they are read. 0 when no replay is recording (e is then not initialised yet). static int alias_copy(Tex *e, const Tex *src) { int reopen; VkCommandBuffer cb = vk_exec_inline_begin(&reopen); if (!cb) return 0; int bw, bh, bpb = img_block(e, &bw, &bh); uint32_t levels = e->pub.levels < src->pub.levels ? e->pub.levels : src->pub.levels; uint32_t layers = e->pub.layers < src->pub.layers ? e->pub.layers : src->pub.layers, n = levels * layers, k = 0; // the region both images cover (alias_compatible: the same shape for block-compressed ones); the rest of a larger // new image starts as init_image would leave it int partial = src->pub.width < e->pub.width || src->pub.height < e->pub.height || src->pub.depth < e->pub.depth || levels < e->pub.levels || layers < e->pub.layers; VkBufferImageCopy *rg = calloc(n, sizeof *rg); if (!rg) rt_fatal("vk: out of memory for an alias copy"); VkDeviceSize total = 0; for (uint32_t lv = 0; lv < levels; lv++) for (uint32_t ly = 0; ly < layers; ly++, k++) { uint32_t w = umin(mip(e->pub.width, lv), mip(src->pub.width, lv)); uint32_t h = umin(mip(e->pub.height, lv), mip(src->pub.height, lv)); uint32_t z = umin(mip(e->pub.depth, lv), mip(src->pub.depth, lv)); rg[k] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, lv, ly, 1}, .imageExtent = {w, h, z}}; total += ((VkDeviceSize)(w + bw - 1) / bw * ((h + bh - 1) / bh) * z * bpb + 15) & ~15ull; } VkBuffer buf = vk_exec_inline_buffer(total); if (!buf) { free(rg); vk_exec_inline_end(reopen); return 0; } barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); vkCmdCopyImageToBuffer(cb, src->pub.image, VK_IMAGE_LAYOUT_GENERAL, buf, n, rg); VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT}; VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = e->pub.image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; vkCmdPipelineBarrier2(cb, &di); if (partial && vk_fix_zero() && !e->bound_to && !e->pub.mirrored && !format_compressed(e->f.vk)) { VkClearColorValue zero = {0}; VkImageSubresourceRange all = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}; vkCmdClearColorImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &all); barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); } vkCmdCopyBufferToImage(cb, buf, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, n, rg); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_exec_inline_end(reopen); free(rg); return 1; } // Descriptors whose image creation failed (VRAM exhausted, an unsupported format): not tried again for 2 s, so a // draw that names one every frame does not repeat a failing allocation on every use. Texture lock held. #define FAILED_HOLD 32 static struct { TexKey k; uint64_t until_ns; } failed_hold[FAILED_HOLD]; static unsigned failed_next; static int create_held(const TexKey *k) { uint64_t now = plat_now_ns(); for (int i = 0; i < FAILED_HOLD; i++) if (failed_hold[i].until_ns > now && !memcmp(&failed_hold[i].k, k, sizeof *k)) return 1; return 0; } static void create_failed(const TexKey *k) { unsigned i = failed_next++ % FAILED_HOLD; failed_hold[i].k = *k, failed_hold[i].until_ns = plat_now_ns() + 2000000000ull; } // NX2NAVI_TRACE=perf: a line for each event of the two kinds that can leave a descriptor showing an image without // content, up to START_LINES per address and kind (said_at). // - over: e, which the CPU writes, was made over the live entry `over` and is filled from its storage. `over` stays // as it is, and a slot that names its descriptor goes on showing it, whatever it holds. // - gone: e, which the CPU does not write, was made at an address the CPU can see, and the released CPU-written entry // there was destroyed (gone: its description, gone_rel: the frame of its release). Nothing uploads e; a slot of e's // descriptor that showed the destroyed entry's image (slot_resolve follows it while it is live) shows e's from now on. // flag: e was taken back (over); how e started (gone): 0 empty, 1 with a copy of another entry's content, 2 with a // copy of the destroyed entry's, 3 filled from its storage (the last two: split fill). static uint64_t n_cpu_empty[3]; // since the last perf line: entries the CPU does not write, at an address it can see, // that started without content: on their own, over a released entry, over a live one #define START_LINES 8 #define START_LINES_ALL 20000 static void start_line(const Tex *e, const Tex *over, const char *gone, uint64_t gone_rel, int flag) { static SaidAt said[2]; static uint32_t total[2]; int kind = over ? 0 : 1; if (total[kind] >= START_LINES_ALL || !said_at(&said[kind], e->key.addr, START_LINES)) return; total[kind]++; char d[224], o[224], og[320]; desc_text(d, sizeof d, e); if (over) { const char *why = mirror_refused(&over->t); desc_text(o, sizeof o, over); origin_text(og, sizeof og, over); rt_log("perf: frame %llu: texture %#llx filled from storage over a live entry%s: new [%s]; live [%s], %s%s%s, %s", (unsigned long long)frames_now(), (unsigned long long)e->key.addr, flag ? " (taken back)" : "", d, o, why ? "not CPU-written: " : "CPU-written", why ? why : "", no_content(over) ? ", no content" : "", og); } else { static const char *const how[4] = {"started empty", "got another entry's content", "got the destroyed entry's content", "was filled from its storage"}; rt_log("perf: frame %llu: texture %#llx, not CPU-written (%s), %s after the released CPU-written entry at its " "address was destroyed: new [%s]; destroyed [%s], released at frame %llu", (unsigned long long)frames_now(), (unsigned long long)e->key.addr, mirror_refused(&e->t), how[flag & 3], d, gone, (unsigned long long)gone_rel); } } // Split fill from the storage: e, new and through its first transition, gets its storage's bytes in the upload batch // (make_fresh, as a CPU-written texture's first use) and is marked to follow them or not (Tex.sfill). 0 when e is left // as it is: storage over SPLIT_FILL_MAX, an image taken from a released entry (see "split fill"), storage not mapped, // or an upload that had no memory, which make_fresh tries again later. static int split_fill(Tex *e) { if (e->t.L.size > SPLIT_FILL_MAX) return n_split_big++, 0; if (e->reused) return n_split_reused++, 0; if (!mem_readable(e->pub.addr, e->t.L.size)) return 0; uint64_t t0 = kern_now_ns(); e->sfill = (uint8_t)split_mode; make_fresh(e); if (!e->uploaded) return 0; n_split_fill++, split_total[1]++, n_split_bytes += e->t.L.size, n_split_ns += kern_now_ns() - t0; e->had |= HAD_FILL; return 1; } static int depth_alias_log(void) { // NX2NAVI_DEPTH_ALIAS_LOG=1: start_at logs each new depth entry over a live texture static int on = -1; if (on < 0) on = rt_knob("DEPTH_ALIAS_LOG", "0"); return on; } // The start of e's content at its address, for a new entry and for a superseded one taken back (taken_back): another // descriptor's released entry there is superseded, and a live one holding GPU-written content is the alias source, // preferring the one that differs only in swizzle (same_but_swizzle). A superseded entry is kept for its own // descriptor, which takes it back here: Ocarina alternates descriptors at some addresses every frame (a 2560x1440 // target and a 480x270 one in Kokiri Forest), and destroying the released one made both images again each frame. It // starts over as a new entry would (alias copy or init_image) when a texture at its address holds GPU content written // since its release (newer), and else keeps its own: the memory still holds what it held then. Ocarina gives its scene // picture (r11g11b10f) a second descriptor at the same address that nothing draws into (rgb9e5, read raw by the post // pass), and the picture, taken back about once a second and restarted empty, went black for a frame (issue #253). // Released mirrored entries are still destroyed: their content is the storage's, uploaded when an entry is made. // A released entry that holds GPU content gives it to e when no live texture there holds any (rsrc): its memory on // the console still holds what it drew, and a new descriptor made for that picture after the old one was let go read // it there (a black frame with FSR 3.1 and its frame generation, two presents long). static void start_at(Tex *e, const TexKey *k, int taken_back) { int alias = 0; int trace = (rt_trace & RT_TRACE_PERF) != 0; int gone = 0, rel_other = 0; // released entries at the address: CPU-written ones destroyed here, others superseded char gone_desc[224] = ""; // trace: the last destroyed one, described while it existed uint64_t gone_rel = 0; Tex *src = NULL, *rsrc = NULL; // split fill: e is to start with its storage's bytes; msrc: the released CPU-written entry whose image holds them // (destroyed after the copy), msrc_gpu: it holds what the GPU wrote instead, msrc_gen: the storage's generation int fill = !taken_back && tex_alias && split_wants(&e->t, &e->f), msrc_gpu = 0; uint64_t msrc_gen = 0; Tex *msrc = NULL; const char *why = "no live texture with GPU content"; // for the log of an alias left empty const Tex *other = NULL; int newer = 0; // taken back: a texture at this address holds GPU content written since e's release int live_content = 0; // a live texture at this address holds GPU content for (Tex *p = nvn_hm_get(&by_addr, &k->addr), *nx; p; p = nx) { nx = p->next_at; if (p == e) continue; if (taken_back && p->use_frame >= e->rel_frame && alias_has_gpu_content(p)) newer = 1; if (p->released) { if (p->pub.mirrored) { if (trace) { int n = desc_text(gone_desc, sizeof gone_desc, p); if (n > 0 && n < (int)sizeof gone_desc) snprintf(gone_desc + n, sizeof gone_desc - (size_t)n, ", %s", p->pub.gpu_owned ? "GPU-written" : p->uploaded ? "uploaded" : "not uploaded"); gone_rel = p->rel_frame; } gone++; const char *no; uint64_t gen = 0; int gpu = alias_has_gpu_content(p); if (fill && !msrc && (gpu || (p->uploaded && !p->pub.gpu_owned && !vk_mem_changed(p->pub.addr, p->t.L.size, p->wgen, &gen))) && alias_compatible(e, p, &no)) { msrc = p, msrc_gpu = gpu, msrc_gen = gen; continue; } drop_tex(p, DROP_REPLACED); continue; } p->superseded = 1, rel_other = 1; const char *unused; if (tex_alias && alias_has_gpu_content(p) && alias_compatible(e, p, &unused) && (!rsrc || p->use_frame > rsrc->use_frame)) rsrc = p; continue; } alias = 1; other = p; if (alias_has_gpu_content(p)) live_content = 1; if (!tex_alias || (src && !(same_but_swizzle(&p->key, k) && !same_but_swizzle(&src->key, k)))) continue; if (!alias_compatible(e, p, &why)) continue; if (!alias_has_gpu_content(p)) { why = "the live texture holds no GPU content"; continue; } src = p; } int from_released = !src && rsrc && !live_content; if (from_released) src = rsrc; if (alias) perf_count(PERF_CNT_TEX_ALIAS); if (alias && (e->f.depth || e->f.stencil) && depth_alias_log()) { // each new depth or stencil entry over a live texture, with the frame and whether the two are bound into pool // memory (the same allocation: they share bytes), to line up with a dark frame static int n_dlog; if (n_dlog++ < 20000) rt_log("vk: depth alias: frame %llu %#llx %ux%ux%u fmt %#x%s%s over %ux%ux%u fmt %#x%s%s%s: %s", (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), (unsigned long long)k->addr, e->pub.width, e->pub.height, e->pub.layers, e->t.d.format, e->bound_to ? " bound" : " own memory", taken_back ? " (taken back)" : "", other->pub.width, other->pub.height, other->pub.layers, other->t.d.format, other->f.depth ? " depth" : "", other->bound_to ? " bound" : " own memory", e->bound_to && e->bound_to == other->bound_to ? ", same allocation" : "", taken_back && !newer && !e->pub.mirrored ? "kept its own" : e->reused || taken_back ? "cleared at the replay's position (reused)" : depth_init_inline() ? "cleared at the replay's position" : "cleared in the upload batch"); } if (taken_back && !newer && !e->pub.mirrored) { // its image still holds its content n_alias_kept++; return; } if (!tex_alias) { // create_tex initialised a new entry if (taken_back) init_new(e); return; } // split fill only where nothing live holds content the storage lacks: that content is what e should show if (src || live_content) fill = 0; int copied = src && alias_copy(e, src), filled = 0; if (fill && msrc && alias_copy(e, msrc)) { // from the released entry this one replaces copied = filled = 1; n_split_copy++, split_total[0]++; e->og_start = START_COPY_GONE; e->had |= HAD_COPY; if (!msrc_gpu && e->t.L.size <= SPLIT_FILL_MAX) { // the storage's bytes as of msrc_gen: followed from there e->sfill = (uint8_t)split_mode, e->uploaded = 1, e->wgen = msrc_gen; e->chash_ok = 0; } } if (msrc) drop_tex(msrc, DROP_REPLACED); if (filled) { // nothing more: e is not CPU-written } else if (copied) { n_alias_copied++; if (from_released) n_alias_released++; e->og_start = from_released ? START_COPY_RELEASED : START_COPY_LIVE; e->had |= HAD_COPY; if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen); e->uploaded = 1; pool_gpu_owned(e); e->pub.gpu_owned = 1; } e->chash_ok = 0; } else { init_new(e); if (fill) filled = split_fill(e); e->og_start = filled ? START_FILL : alias ? (e->pub.mirrored ? START_UPLOAD_LIVE : START_EMPTY_LIVE) : (gone || rel_other) && !e->pub.mirrored ? START_EMPTY_RELEASED : START_NEW; // an entry the CPU does not write, at an address the CPU can see, that starts without content if (!filled && !e->pub.mirrored && !nvn_gpu_only(k->addr)) n_cpu_empty[alias ? 2 : gone || rel_other ? 1 : 0]++; if (alias) { if (e->pub.mirrored) { n_alias_upload++; if (trace) start_line(e, other, NULL, 0, taken_back); } else if (!filled) { n_alias_uninit++; static int logged; // what an empty start over live memory was, to tell a black frame's cause; depth buffers start empty // over another target's memory at every area load, harmlessly, and are left out int depth = e->f.depth || e->f.stencil || other->f.depth || other->f.stencil; if (logged < 24 && other && !depth) { logged++; rt_log("vk: texture %#llx (%ux%u fmt %#x%s) starts empty over a live one (%ux%u fmt %#x): %s; frame %llu", (unsigned long long)k->addr, e->pub.width, e->pub.height, e->t.d.format, taken_back ? ", taken back" : "", other->pub.width, other->pub.height, other->t.d.format, src ? "no replay recording to copy in" : why, (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED)); } } } } if (trace && gone && !e->pub.mirrored && !nvn_gpu_only(k->addr)) start_line(e, NULL, gone_desc, gone_rel, e->og_start == START_COPY_GONE ? 2 : filled ? 3 : copied); } // A replay worker's chunk before its turn (vk_exec.c "replay workers"): a lookup that would make or take back an entry // returns nothing and sets this; the chunk replays the op again once every earlier chunk is done, so entries are made, // initialised and taken back in the order of the stream, as the replay would. static _Thread_local int tex_deferred; static volatile uint64_t n_deferred[3]; // by what the lookup found: no entry, a retired one, a released one void vk_texture_deferred_stats(uint64_t out[3]) { for (int i = 0; i < 3; i++) out[i] = __atomic_exchange_n(&n_deferred[i], 0, __ATOMIC_RELAXED); } int vk_texture_deferred(void) { return tex_deferred; } int vk_texture_deferred_take(void) { int d = tex_deferred; tex_deferred = 0; return d; } // A lookup of k that finds e (NULL: no entry) and needs no turn (vk_exec_turn_due): it returns nothing without changing // anything (the pool refresh asking for a held key or a retired entry, a creation that failed lately: the texture pool // refresh looks its unresolved slots up at every sync), or it is the submitting thread's past a group (vk_exec_behind) // and takes back a released entry whose image stayed its own. That keeps its content and is ordered after the // group's chunks anyway: one of them that looks the entry up later finds it in use, as it would have after taking it // back itself. A transient render target the frame releases and takes back at every pass that draws it did that after // nearly every group (Hyrule Field, 2026-11-12). static int lookup_quiet(const Tex *e, const TexKey *k) { if (e && e->retiring) return pool_ctx; if (e) return !e->superseded && vk_exec_behind(); if (holds.n && nvn_hm_get(&holds, k)) return pool_ctx; return create_held(k); } static Tex *entry_of(const tex_t *t) { TexKey k; make_key(t, &k); Tex *e = nvn_hm_get(&by_key, &k); if ((!e || e->retiring || e->released) && vk_exec_turn_due() && !lookup_quiet(e, &k)) { __atomic_add_fetch(&n_deferred[!e ? 0 : e->retiring ? 1 : 2], 1, __ATOMIC_RELAXED); tex_deferred = 1; return NULL; } if (e && e->retiring) { // retired by disuse: the pool refresh leaves it out, any other lookup takes it back if (pool_ctx) return NULL; retire_cancel(e); } if (!e && private_keys.n) // another swizzle of an image that has memory of its own (unalias): the same image for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) if (!p->released && !p->bound_to && nvn_hm_get(&private_keys, &p->key) && same_but_swizzle(&p->key, &k)) { touch(p); return p; } if (!e) { if (held_out(&k) || create_held(&k)) return NULL; uint64_t c0 = kern_now_ns(); // the entry's creation (VK_TS_CREATE): image, memory, first transition if (tex_alias < 0) { tex_alias = cfg_bool("NX2NAVI_TEX_ALIAS"); rt_log("vk: texture aliases: %s", tex_alias ? "a new image takes the live image's content" : "a new image starts empty"); } if (!(e = create_tex(t, &k, tex_alias))) return create_failed(&k), NULL; origin_made(e); reuse_made(e); perf_count(PERF_CNT_TEX_NEW); start_at(e, &k, 0); flight_ev(e, FLIGHT_TEX_NEW, e->og_start, 0); e->next_at = nvn_hm_get(&by_addr, &k.addr); nvn_hm_put(&by_addr, &k.addr, e); nvn_hm_put(&by_key, &k, e); n_site[1]++; tex_gen_bump(k.addr); log_new(e, &k); vk_texstat_time(VK_TS_CREATE, c0); e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // new: not disused before it is drawn } if (e->released) { flight_ev(e, FLIGHT_TEX_TAKEBACK, 0, 0); e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one uint64_t gap = e->samp_frame - e->rel_frame; if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; reuse_taken_back(e); e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); if (e->pidx) pent[e->pidx - 1].rel = 0; // the pool's check looks at it again e->overwritten = 0; e->og_backs++, e->og_back_frame = e->samp_frame; if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work e->superseded = 0, e->reused = 1; n_restarts++; start_at(e, &k, 1); } } e->t = *t; touch(e); return e; } // vk_dump: upload bookkeeping of an image vk_texture_get / vk_texture_by_id returned. pages_written: whether its // storage pages were written since the last check (1, 0, -1 not tracked); the query leaves that check's state alone. void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written) { lock(); const Tex *e = (const Tex *)img; // pub is the first member uint64_t now; *uploads = e->n_up; *last_change = e->last_ch; *pages_written = e->pub.mirrored && e->uploaded ? vk_mem_changed(e->pub.addr, e->t.L.size, e->wgen, &now) : 0; unlock(); } const VkTexImage *vk_texture_get(const tex_t *t) { if (!t || !t->mem) return NULL; lock(); Tex *e = entry_of(t); if (e) make_fresh(e); unlock(); return e ? &e->pub : NULL; } // Render targets the CPU reads without NVN's invalidate (vk_texture_write_back_small); small_pending: one may be dirty. // wb_limit: targets up to this size (bytes) always go back when the CPU waits for a fence, NX2NAVI_WB_FENCE_LIMIT, read // once. A larger one goes back unless the GPU wrote it in each of the last WB_RUN frames (wb_wanted). wb_trace: // NX2NAVI_TRACE=gfx, read together with it. enum { WB_SMALL_DEFAULT = 64 << 10, WB_RUN = 16 }; static int small_pending; static uint64_t wb_limit = WB_SMALL_DEFAULT; static int wb_trace; static int wb_cfg_read; static void wb_cfg_init(void) { if (wb_cfg_read) return; int v = cfg_int("NX2NAVI_WB_FENCE_LIMIT"); wb_limit = v < 0 ? 0 : (uint64_t)v; wb_trace = (rt_trace & RT_TRACE_GFX) != 0; wb_cfg_read = 1; if (wb_limit != WB_SMALL_DEFAULT) rt_log("vk: fence write-back limit %llu bytes", (unsigned long long)wb_limit); } // Whether the fence write-back takes a dirty target. A target the GPU draws every frame is scratch the CPU does not // read: in Hyrule Field three 256x256 RGBA8 targets and a 256x256 RG8 one, which cost 2.7 ms a frame to write back. // NX2navi's 260x260 RG8 grass maps, which its fire spread reads after a fence wait (#132), are drawn once per change, or // up to 9 frames in a row while a tile loads. static int wb_wanted(const Tex *e) { return e->t.L.size <= wb_limit || e->run < WB_RUN; } static void mark_released(Tex *e) { // tex_lock held; e is not released flight_ev(e, FLIGHT_TEX_RELEASE, 0, 0); e->released = 1; if (e->pidx) pent[e->pidx - 1].rel = 1; // the pool's check passes it by (vk_texture_pool_fresh_locked) e->rel_epoch = submit_epoch; e->rel_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); nreleased++; rel_append(e); n_site[3]++; tex_gen_flip(e, 0); } // e, in the GPU-only address window, is written by the GPU in frame fr: the entries its storage runs over are // overwritten (section "Moved render targets", above gwin). static void overwrite_others(const Tex *e, uint64_t fr) { uint64_t lo = e->key.addr, hi = lo + e->t.L.size; int released = 0; for (int i = gwin_lower(lo > gwin_span ? lo - gwin_span : 0); i < ngwin && gwin[i]->key.addr < hi; i++) { Tex *q = gwin[i]; uint64_t qlo = q->key.addr, qhi = qlo + q->t.L.size; if (q->overwritten || qlo == lo || qhi <= lo || (lo >= qlo && hi <= qhi)) continue; if (fr - q->gw_frame < OVERWRITE_FRAMES) continue; q->overwritten = 1, q->ow_frame = fr; n_overwritten++; if (!q->released) mark_released(q), released = 1; } if (released) sweep_released(); } void vk_texture_gpu_written(const tex_t *t) { if (!t || !t->mem) return; vk_exec_frame_written(t); lock(); Tex *e = entry_of(t); if (e) { make_fresh(e); // a partial render keeps what the storage held before uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); e->gw_frame = now; if (e->scan_frame != now + 1 && nvn_gpu_only(e->key.addr)) { // once per frame and entry e->scan_frame = now + 1; overwrite_others(e, now); } pool_gpu_owned(e); e->pub.gpu_owned = 1; mark_clobbered(e); e->chash_ok = 0; // the image now holds the GPU's content: matching storage bytes no longer mean "unchanged" if (e->pub.skip) { // texture detail leaves out what the GPU can write (detail_skip); said if it ever does static volatile int32_t n_skip_written; log_limited(&n_skip_written, "texture %#llx (fmt %#x %ux%u), made without its top %u levels, was written by " "the GPU: its levels are not the game's", (unsigned long long)e->pub.addr, e->t.d.format, e->t.d.width, e->t.d.height, e->pub.skip); } uint64_t f = vk_exec_recording_fence(); if (f > e->wr_fence) e->wr_fence = f; if (e->pub.mirrored) { e->gpu_dirty = 1; storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render if (wb_exact < 0) wb_exact = cfg_bool("NX2NAVI_WB_EXACT"); if (wb_exact && !e->snap && e->t.L.size <= SNAP_MAX && fills_no_page(e) && !(e->snap = malloc(SNAP_MAX))) rt_fatal("vk: out of memory for a texture's storage copy"); if (e->snap) memcpy(e->snap, (const void *)(uintptr_t)e->pub.addr, e->t.L.size); e->gpu_seq = ++gpu_writes; e->gpu_fence = vk_exec_recording_fence(); uint64_t fr = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (fr != e->run_frame || !e->run) { e->run = fr == e->run_frame + 1 && e->run ? (uint8_t)(e->run < WB_RUN ? e->run + 1 : WB_RUN) : 1; e->run_frame = fr; } wb_cfg_init(); if (wb_wanted(e)) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE); own_add(e); } } unlock(); } void vk_texture_release(const tex_t *t) { if (!t || !t->mem) return; TexKey k; make_key(t, &k); lock(); Tex *e = nvn_hm_get(&by_key, &k); // Staleness belongs to the registered descriptor, not the cached image: a released Tex can be destroyed before the // slot looks again (eviction, the released cap, start_at for a mirrored one). A registered texture may also be // finalized before its image is created. // Only the slots this texture object registered (its initialization serial): a slot registered with another // texture initialized at the same storage with the same parameters names that one, which is still live. Ocarina // streams terrain heightmaps (R32F 260x260) through a fixed set of storage and initializes and registers the next // chunk's texture there before it finalizes the previous one; finalizing the new registration left the slot stale, // the draws sampled the zero stand-in, and the chunk was drawn at height zero, tens of metres below the ground. int changed = 0; if (sinfo) for (int id = bucket_head[agen_idx(k.addr)]; id >= 0; id = slot_next[id]) { TexSlot *s = &tslots[id]; if (s->finalized || (uintptr_t)s->t.mem != k.addr) continue; TexKey sk; make_key(&s->t, &sk); if (memcmp(&sk, &k, sizeof k)) continue; if (s->t.serial != t->serial) { // another texture object's registration: it stays current n_fin_other++, n_fin_other_hm += terrain_heightmap(&s->t); continue; } s->finalized = 1, s->ce = NULL, changed = 1; n_fin_own++; } if (e && !e->released) { mark_released(e); changed = 0; // the image's release already invalidated the address bucket sweep_released(); } if (changed) tex_gen_bump(k.addr); unlock(); } void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc) { lock(); Tex **list = malloc(sizeof *list * (size_t)(by_key.n + 1)); int n = 0; for (uint32_t i = 0; i < by_key.cap; i++) { Tex *e = by_key.vals[i]; if (e && ((e->key.addr >= lo && e->key.addr - lo < size) || (alloc && e->bound_to == alloc))) list[n++] = e; } for (int i = 0; i < n; i++) drop_tex(list[i], DROP_RANGE); free(list); unlock(); } // ---------------------------------------------------------------- descriptor slots void vk_texture_register(int id, const tex_t *t, const view_t *v) { if (id < 0 || id >= (1 << 20)) rt_fatal("vk: texture slot %d out of range", id); if (id >= VK_POOL_TEX) { // the translator clamps pool handles to the pool's last entry: shaders cannot reach it static int said; if (!__atomic_exchange_n(&said, 1, __ATOMIC_RELAXED)) rt_log("vk: texture slot %d registered, past the %d the shaders index (VK_POOL_TEX): a shader that samples it " "reads slot %d instead (%d for a translation carried over from the translator before)", id, VK_POOL_TEX, VK_POOL_TEX - 1, VK_POOL_TEX_CARRIED - 1); } else if (id >= VK_POOL_TEX_CARRIED) { // within the pool, past what a shader of the earlier length reaches static int said; uint32_t n; if (!__atomic_load_n(&said, __ATOMIC_RELAXED) && (n = gfx_shader_pool_short_seen()) && !__atomic_exchange_n(&said, 1, __ATOMIC_RELAXED)) rt_log("vk: texture slot %d registered while %u shaders in use index the texture pool with the earlier " "length (%d, gfx_shader.c pool_short_note): they read slot %d in its place", id, n, VK_POOL_TEX_CARRIED, VK_POOL_TEX_CARRIED - 1); } lock(); if (id >= ntslots) { int n = ntslots ? ntslots : 1024; while (n <= id) n *= 2; TexSlot *s = realloc(tslots, sizeof *s * (size_t)n); if (!s) rt_fatal("vk: out of memory for texture slots"); memset(s + ntslots, 0, sizeof *s * (size_t)(n - ntslots)); tslots = s, ntslots = n; } if (!sinfo) { uint64_t *p = calloc(1 << 20, sizeof *p); slot_next = malloc(sizeof *slot_next << 20), slot_prev = malloc(sizeof *slot_prev << 20); slot_grp = malloc(sizeof *slot_grp << 20); slot_bucket = calloc(1 << 20, sizeof *slot_bucket); if (!p || !slot_next || !slot_prev || !slot_grp || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens"); for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1, slot_grp[i] = -1; for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1, bucket_ghead[i] = -1; __atomic_store_n(&sinfo, p, __ATOMIC_RELEASE); } TexSlot *s = &tslots[id]; // The same texture and view registered again (Ocarina rewrites much of its descriptor pool every frame): the slot // keeps its serial and resolved entry, so the texture pool refresh (vk_exec_draw.c pool_sync_entries) does not // resolve it again. Only while that entry is live: after a release, a registration is what makes a slot current // again (slot_resolve's stale rule), so it goes the full way. A slot held out of the pool (cheld) keeps that too: // the same descriptor names the same held texture. Both only while the resolution is current (cgen): a destroyed // entry bumped the generation and its record may be freed, so ce is not read past that. int current = s->used && !s->finalized && s->cgen == agen[sinfo[id] & 0xffff]; if (current && ((s->ce && !s->ce->released) || s->cheld) && !memcmp(&s->t, t, sizeof *t) && s->has_view == (v != NULL) && (!v || !memcmp(&s->v, v, sizeof *v))) { n_same_regs++; unlock(); return; } if (flight_on && tex_mirrored(t)) flight_tex(FLIGHT_TEX_REG, (uint64_t)(uintptr_t)t->mem, (uint32_t)t->d.format, t->d.width, t->d.height, 0, FLIGHT_TF_MIRRORED, !s->used ? 0 : s->finalized ? 3 : (uintptr_t)s->t.mem == (uintptr_t)t->mem ? 1 : 2, (uint32_t)id); s->t = *t; if (!s->used && id < VK_POOL_TEX) pool_slots++; s->used = 1; s->finalized = 0; s->ce = NULL; s->cheld = 0; s->fresh = 1; slots_gen++; if (!++reg_serial) reg_serial = 1; __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); dirty_push((uint32_t)id); s->has_view = v != NULL; if (v) s->v = *v; else memset(&s->v, 0, sizeof s->v); s->khash = 0; if (!tex_mirrored(t)) { // FNV-1a over the key (make_key zeroes its padding) and the view TexKey k; make_key(t, &k); uint64_t h = 0xcbf29ce484222325ull; const uint8_t *b = (const uint8_t *)&k; for (size_t i = 0; i < sizeof k; i++) h = (h ^ b[i]) * 0x100000001b3ull; b = (const uint8_t *)&s->v; for (size_t i = 0; i < sizeof s->v; i++) h = (h ^ b[i]) * 0x100000001b3ull; s->khash = (h ^ s->has_view) | 1; } slot_place(id, agen_idx((uintptr_t)t->mem), s->khash); unlock(); } // The descriptor hash of a GPU-written slot: two slots with equal hashes, the same address generation and no entry // created, destroyed, released or taken back between their lookups (vk_texture_entry_gen) resolve alike. 0 for an // unregistered or CPU-written slot (its resolution depends on when it was registered: slot_resolve's stale rule). uint64_t vk_texture_slot_khash_locked(int id) { return id >= 0 && id < ntslots && tslots[id].used ? tslots[id].khash : 0; } uint32_t vk_texture_entry_gen(void) { return tex_gen; } // The entry slot s resolves to, and the view and has_view a caller gets with it (the slot's, with the descriptor's // swizzle when the entry is another descriptor at the same storage). NULL when the slot names no texture. // Entry creation budget of slot lookups (vk_texture_create_budget_locked): -1 none. The bindless pools look up every // registered slot, and most of a pool is stale descriptors (storage freed or reused); with a budget a lookup creates at // most that many entries, only for mapped storage, and notes the slots it left for later (create_deferred). static int create_budget = -1, create_deferred; // A slot whose stand-in a shader sampled is on screen as that stand-in: its texture is made by the next lookup even // with the budget at 0 (the frame's creation time spent, vk_exec_draw.c pool_create_ns), WANT_PER_FRAME of them a // present. Left to the budget, it stayed dark for as long as the refresh had no time over, about a second in a town. #define WANT_PER_FRAME 32 static int want_take(void) { static uint64_t frame = ~0ull; static int left; uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (f != frame) frame = f, left = WANT_PER_FRAME; return left > 0 ? left-- : 0; } void vk_texture_create_budget_locked(int n) { create_budget = n, create_deferred = 0; } // NX2NAVI_POOL_SHOWN_FIRST (default 1; 0: such a slot waits for the budget like any other): the slot being looked up // shows a texture in the pool's set now (vk_exec_draw.c pool_fill), or the game has registered it since its last // lookup that found a texture (TexSlot.fresh). When its own texture is not made, the game has registered the slot // with another texture and draws with it next: it is made at this lookup, beside the count budget and the frame's // creation time, SHOWN_PER_FRAME of them a present. A slot that showed nothing when it was registered counts too: the // game takes slots whose texture it finalized some time ago for the next ones, and one of those was an effect's // texture with a transparent ground: drawn with the stand-in it was a flat grey patch for one frame at each of two // rounds of registrations four frames apart, in a spot where the effect is not visible otherwise. Ocarina // streams its textures by detail level and registers about 32 slots with their next level every four or five frames // while the player moves (2,300 to 3,800 registrations in 600 frames of two recordings). A refresh made 16 of them and // wrote the stand-in into the set for the others until its next round, and the set is read as it is when its // submission goes to the queue: an object whose slot was among the others was drawn grey for a frame (three lamp // posts, the windows of a house front in the recordings). An area load registers thousands of slots that showed the // area before; past the per-present count they wait for the budget as before. #define SHOWN_PER_FRAME 64 static int pool_shown; void vk_texture_pool_shown_locked(int shown) { pool_shown = shown; } static int shown_take(const TexSlot *s) { static int on = -1; static uint64_t frame = ~0ull; static int left; if (on < 0) on = cfg_bool("NX2NAVI_POOL_SHOWN_FIRST"); if (!on || !(pool_shown || s->fresh)) return 0; uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (f != frame) frame = f, left = SHOWN_PER_FRAME; return left > 0 ? left-- : 0; } // What the pool's lookups have spent making entries (entry_of under the budget) and uploading them the first time, // since the last call: the time vk_exec_draw.c pool_create_ns limits in a frame. Texture lock held where it is added. static uint64_t pool_make_ns; uint64_t vk_texture_pool_make_ns_take(void) { return __atomic_exchange_n(&pool_make_ns, 0, __ATOMIC_RELAXED); } static void pool_made(uint64_t t0) { __atomic_fetch_add(&pool_make_ns, kern_now_ns() - t0, __ATOMIC_RELAXED); } int vk_texture_create_deferred_locked(void) { int d = create_deferred; create_deferred = 0; return d; } // NX2NAVI_TEX_STALE_KEPT (default 1; 0: the stand-in, as before): a finalized slot of a CPU-written texture resolves to // its own entry while that entry holds the texture's content: uploaded or drawn by the GPU, and neither superseded by // another descriptor at its address nor overwritten. The game finalizes texture objects on its own thread ahead of the // replay, which still draws the frame the game recorded before that; on the console the memory still holds the texels // those draws read. Kakariko moves about 30 of its textures to new addresses every four or five frames: each move // finalized the old descriptors, the replay's draws of the frame before then met finalized slots, and their stand-ins // took the village's haze out of a frame a few times a minute (thousands of stand-in lookups in 60 frames). The image // is sampled as it is: the lookup neither takes the entry back nor reads its storage (vk_texture_by_id_locked). static int stale_kept(const Tex *own) { static int on = -1; if (on < 0) on = rt_knob("TEX_STALE_KEPT", "1") == 1; return on && own && !own->superseded && !own->overwritten && (own->uploaded || own->pub.gpu_owned); } // NX2NAVI_POOL_DATA_FIRST (default 1; 0: it waits for the budget like any other): a slot of a texture that is not // block-compressed and holds at most DATA_FIRST_BYTES is made at its lookup too, beside the count budget and the // frame's creation time, DATA_PER_FRAME of them a present. The budget exists for the pictures the game streams, ASTC // that has to be decoded; a plain-format texture is as often the game's data (a terrain height map, a lookup table, // a mask), and a draw that reads the stand-in in its place for a frame puts geometry or lighting in the wrong place // instead of showing a surface a little late. It costs an upload and no decode. #define DATA_PER_FRAME 24 #define DATA_FIRST_BYTES (4u << 20) static int data_take(const TexSlot *s) { static int on = -1; static uint64_t frame = ~0ull; static int left; if (on < 0) on = cfg_bool("NX2NAVI_POOL_DATA_FIRST"); if (!on || s->t.L.size > DATA_FIRST_BYTES) return 0; int bw = 1, bh = 1; if (!nvn_fmt_bpb(s->t.d.format, &bw, &bh) || bw > 1 || bh > 1) return 0; // unknown or block-compressed uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if (f != frame) frame = f, left = DATA_PER_FRAME; return left > 0 ? left-- : 0; } static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { *view = s->v; *has_view = s->has_view; TexKey k; make_key(&s->t, &k); Tex *e = nvn_hm_get(&by_key, &k); int stale = 0; if (!e || e->released || s->finalized) { // a finalized slot follows live GPU content, not newly reused CPU bytes // Prefer the live entry that differs from the slot descriptor only in its swizzle: transient render targets // share storage, and the newest one at the address is usually another size. The map screen samples its // 2250x1080 map target through an (R,G,B,1) descriptor and followed a 960x60 target at the same address. Tex *own = e, *any = NULL; e = NULL; // A slot whose content the CPU writes (tex_mirrored) follows another image only for content its own bytes do // not hold, the GPU's: Ocarina rasterizes each UI caption on the CPU into a small r8 glyph atlas and reuses the // memory of a freed one for the next, of another width. Following the older 256x39 atlas at the address, the // replay uploaded the new 128-wide bytes with a 256-texel row, and the Basic Controls screen's ZL caption // ("Target") sampled a mix of its glyphs and stale ones. Without such an image the slot gets its own below // (entry_of), uploaded from its storage with its own geometry. int cpu_slot = tex_mirrored(&s->t); for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) { if (p->released) continue; // Only an entry of the slot's target: Ocarina's volumetric fog registers a 3D two-channel volume (32x24x64) // over the storage of the bloom pass's 480x270 2D target; followed, the fog computes and the fog pass got // that 2D image, no 3D view can be made of it (vk_exec_draw.c get_view) and they read and wrote stand-ins. // With the target kept they get the volume itself. if (p->key.d.target != k.d.target) continue; if (same_but_swizzle(&p->key, &k) && (!cpu_slot || !s->finalized || alias_has_gpu_content(p))) { e = p; break; } if (!any && (!cpu_slot || alias_has_gpu_content(p)) && follow_size_ok(p, &k)) any = p; } // A finalized CPU descriptor must not recreate its image from freed storage. The slot keeps this state even // after its released image was evicted; registering the descriptor again makes it current. While its own // entry still holds the texture (stale_kept), the slot samples that image as it is. if (!e && any) e = any; else if (!e && !cpu_slot) e = own && own->superseded ? NULL : own; // a superseded one: entry_of takes it back else if (!e && cpu_slot && s->finalized && stale_kept(own)) e = own, n_stale_kept++; else if (!e && cpu_slot && s->finalized) stale = 1, n_stale++, n_stale_hm += terrain_heightmap(&s->t); } resolve_stale = stale; if (!e && s->t.mem && !stale) { // evicted by disuse: the pool refresh keeps the stand-in, without spending the creation budget on it if (pool_ctx && holds.n && nvn_hm_get(&holds, &k)) resolve_held = 1; else if (create_budget < 0) e = entry_of(&s->t); // Unmapped storage first: such a slot has nothing to make, now or later. Asked after the budget, every stale // slot a refresh met with the budget at 0 was noted as left: in a busy frame (its creation time spent) that // was about a thousand slots at once, which vk_exec_draw.c pool_sync_entries took for an area load, several // times a second in some places, and each time lifted the count budget and the frame's creation time. else if (!mem_readable((uint64_t)(uintptr_t)s->t.mem, s->t.L.size)) e = NULL; else if (!create_budget && !(s->cwant && want_take()) && !shown_take(s) && !data_take(s)) create_deferred = 1; else { uint64_t m0 = kern_now_ns(); e = entry_of(&s->t); pool_made(m0); if (create_budget > 0) create_budget--; } if (e) s->cwant = 0, s->fresh = 0; } if (e && e->retiring) { // retired by disuse: the pool refresh gets the stand-in, any other lookup takes it back if (pool_ctx) e = NULL, resolve_held = 1; else retire_cancel(e); } // A followed texture is another descriptor at the same storage: its image holds the content, but sampling takes the // slot descriptor's swizzle. The UI renders text into RGBA8 and samples it through a (A,A,A,A) descriptor; with the // render target's RGBA swizzle the text blur saw R = 1 everywhere and drew solid boxes behind tips and labels. if (e && !(s->has_view && s->v.set_swizzle)) { const nvn_texdesc *d = &s->t.d; uint8_t sw[4] = {(uint8_t)d->swizzle_r, (uint8_t)d->swizzle_g, (uint8_t)d->swizzle_b, (uint8_t)d->swizzle_a}; if (!(sw[0] | sw[1] | sw[2] | sw[3])) // descriptors built without nvnTextureBuilderSetDefaults sw[0] = NVN_TEXTURE_SWIZZLE_R, sw[1] = NVN_TEXTURE_SWIZZLE_G, sw[2] = NVN_TEXTURE_SWIZZLE_B, sw[3] = NVN_TEXTURE_SWIZZLE_A; if (memcmp(sw, e->pub.swz, 4)) { if (!s->has_view) memset(view, 0, sizeof *view); memcpy(view->swz, sw, 4); view->set_swizzle = 1; *has_view = 1; } } return e; } uint32_t vk_texture_slot_why_locked(int id, uint64_t *addr, uint32_t *format, uint32_t *w, uint32_t *h) { *addr = 0, *format = *w = *h = 0; if (id < 0 || id >= ntslots || !tslots[id].used) return 0; const TexSlot *s = &tslots[id]; TexKey k; make_key(&s->t, &k); const Tex *own = nvn_hm_get(&by_key, &k); *addr = k.addr, *format = (uint32_t)s->t.d.format, *w = s->t.d.width, *h = s->t.d.height; const Tex *ce = s->ce && s->cgen == agen[sinfo[id] & 0xffff] ? s->ce : NULL; // an older one may be gone return (s->finalized ? 1u : 0) | (own ? 2u : 0) | (own && own->released ? 4u : 0) | (own && own->superseded ? 8u : 0) | (own && own->overwritten ? 16u : 0) | (own && own->uploaded ? 32u : 0) | (s->t.mem && mem_readable((uint64_t)(uintptr_t)s->t.mem, s->t.L.size) ? 64u : 0) | (holds.n && nvn_hm_get(&holds, &k) ? 128u : 0) | (tex_mirrored(&s->t) ? 256u : 0) | (own && own->pub.gpu_owned ? 512u : 0) | (own && own->retiring ? 1024u : 0) | // what the slot's last lookup kept (vk_texture_by_id_locked): an entry, and whether its image is a volume (ce ? 2048u : 0) | (ce && ce->pub.type == VK_IMAGE_TYPE_3D ? 4096u : 0) | (ce && ce != own ? 8192u : 0); } void vk_texture_lock(void) { lock(); } void vk_texture_unlock(void) { unlock(); } uint32_t vk_texture_slots_gen(void) { return slots_gen; } uint64_t vk_texture_same_regs(void) { return __atomic_load_n(&n_same_regs, __ATOMIC_RELAXED); } // Diagnosis for the texture pool log: address-generation bumps since the last call, the slots of the fullest buckets // and the bumps those buckets took. Texture lock held by none: takes it. void vk_texture_bucket_stats(char *out, size_t n) { lock(); static uint32_t occ[AGEN_N]; memset(occ, 0, sizeof occ); for (int i = 0; i < ntslots && sinfo; i++) if (tslots[i].used) occ[sinfo[i] & 0xffff]++; int top[4] = {-1, -1, -1, -1}; for (int b = 0; b < AGEN_N; b++) for (int k = 0; k < 4; k++) if (top[k] < 0 || occ[b] > occ[top[k]]) { for (int j = 3; j > k; j--) top[j] = top[j - 1]; top[k] = b; break; } uint64_t hot = 0; // bumps weighted by the slots they invalidate for (int b = 0; b < AGEN_N; b++) hot += (uint64_t)bump_count[b] * occ[b]; snprintf(out, n, "%llu new generations (destroy %llu, create %llu, take-back %llu, release %llu; %llu releases and " "take-backs went back to a generation their state had), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u", (unsigned long long)n_bumps, (unsigned long long)n_site[0], (unsigned long long)n_site[1], (unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone, (unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]], bump_count[top[0]], bump_count[top[1]], bump_count[top[2]], bump_count[top[3]]); size_t used = strlen(out); if (used < n) snprintf(out + used, n - used, "; pool freshness check: %llu entries checked, %llu passed by their regions (missed: " "%llu not uploaded, %llu a region without a snapshot yet, %llu untracked, %llu written since, %llu " "nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP; %llu released entries passed by)", (unsigned long long)n_pool_checks, (unsigned long long)n_pool_skips, (unsigned long long)n_pool_miss[0], (unsigned long long)n_pool_miss[1], (unsigned long long)n_pool_miss[2], (unsigned long long)n_pool_miss[3], (unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q, (unsigned long long)n_pool_rel); used = strlen(out); if (used < n && big_hash_n + big_same_n + big_diff_n) // NX2NAVI_TEX_SAME_BIG snprintf(out + used, n - used, "; textures over 64 KiB: %llu uploads hashed (%.1f MiB), after a write to their " "pages %llu left as they were (%.1f MiB), %llu uploaded again", (unsigned long long)big_hash_n, big_hash_bytes / 1048576.0, (unsigned long long)big_same_n, big_same_bytes / 1048576.0, (unsigned long long)big_diff_n); big_hash_n = big_hash_bytes = big_same_n = big_same_bytes = big_diff_n = 0; used = strlen(out); if (used < n) snprintf(out + used, n - used, "; %d slot groups holding %llu slots", ngrp_live, (unsigned long long)n_grouped); int held = 0, gown = 0; for (int i = 0; i < npref; i++) if (pref[i].refs && pref[i].e) held++, gown += pref[i].gown; used = strlen(out); if (used < n) snprintf(out + used, n - used, "; pool slots hold %d entries, %d of them GPU-owned (stamped in each submission " "checked by list)", held, gown); n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = n_pool_rel = 0; memset(n_pool_miss, 0, sizeof n_pool_miss); memset(n_site, 0, sizeof n_site); memset(bump_count, 0, sizeof bump_count); unlock(); } uint64_t vk_texture_slot_token(int id) { const uint64_t *si = __atomic_load_n(&sinfo, __ATOMIC_ACQUIRE); if (!si || id < 0 || id >= (1 << 20)) return 0; uint64_t v = __atomic_load_n(&si[id], __ATOMIC_ACQUIRE); return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0; } uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); } int vk_texture_slot_count_locked(void) { return ntslots; } int vk_sampler_slot_count_locked(void) { return nsslots; } // e was read while marked clobbered (mark_clobbered). It and the descriptors that differ from it only in swizzle (the // same texels: the game renders through one and samples through another) are dropped, and come back as one image in // memory of its own (create_tex, entry_of). A live descriptor of another format or geometry over its storage reads // those bytes through the pool memory, which an image of its own would no longer update: e then stays in pool memory // (the clobbered read is what it was). 1 when e was dropped. static int unalias(Tex *e) { e->clobbered = 0; Tex *sib[16]; int n = 0; for (Tex *p = nvn_hm_get(&by_addr, &e->key.addr); p; p = p->next_at) { if (p == e || p->released) continue; if (!same_but_swizzle(&p->key, &e->key) || n == 16) { n_unalias_kept++; return 0; } sib[n++] = p; } if (n_unaliased++ < 32) rt_log("vk: texture %#llx (%ux%u nvn fmt %#x) was read after another image drew over its memory: it gets memory " "of its own", (unsigned long long)e->pub.addr, e->pub.width, e->pub.height, e->key.d.format); n_unaliased_period++; nvn_hm_put(&private_keys, &e->key, (void *)1); for (int i = 0; i < n; i++) destroy_tex(sib[i]); destroy_tex(e); // bumps the address generation: every slot naming it or a sibling resolves again return 1; } const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { lock(); const VkTexImage *r = vk_texture_by_id_locked(id, view, has_view); unlock(); return r; } const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { if (id < 0 || id >= ntslots || !tslots[id].used) return NULL; TexSlot *s = &tslots[id]; again:; uint32_t g = agen[sinfo[id] & 0xffff]; // resolved again: an entry at its address was created, dropped or released, or the slot registered (s->ce // cleared); an empty result is not kept, as image creation may succeed later, except a held one for the pool // refresh (cheld): the pool walks every slot each submission, and the hold ends with a generation bump if ((!s->ce || s->cgen != g) && !(s->cheld && s->cgen == g && pool_ctx)) { view_t v; int hv; resolve_held = 0; in_lookup = 1; s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it in_lookup = 0; s->cview = v, s->chas_view = (uint8_t)hv; s->cgen = g; s->cepoch = 0; s->cheld = (uint8_t)(!s->ce && resolve_held); } Tex *e = s->ce; // a draw binding: the texture is in use, whether or not the pool's refresh checked it first in this submission // (by list that check is no lookup, vk_texture_pool_fresh_locked, so the stamp of the use is left here too) if (e && !pool_ctx) touch(e); if (e && e->clobbered && unalias(e)) { // read after another image overwrote its pool memory: own memory from now on s->ce = NULL; goto again; } // Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a // replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not // mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A // held slot counts as checked for look_round: the textures evicted by disuse are registered slots all the same. if ((e || s->cheld) && s->cepoch != submit_epoch) { round_checks++; // a released entry here is a finalized slot's own (stale_kept): the storage may be the game's to free or reuse if (e && !e->released && (!e->uploaded || e->checked != submit_epoch)) { uint64_t m0 = pool_ctx && !e->uploaded ? kern_now_ns() : 0; // the pool's lookup: as in its list check make_fresh_snap(e); if (m0) pool_made(m0); e->checked = submit_epoch; touch(e); } s->cepoch = submit_epoch; } if (view) *view = has_view ? s->cview : s->v; if (has_view) *has_view = view ? s->chas_view : s->has_view; return e ? &e->pub : NULL; } // Texture detail: the texture slot id resolves to was made without its top levels, and a shader binding reads its // texels by position or asks for its size (vk_exec_draw.c tex_info_full, GfxShaderRes.exact). Its key joins exact, the // entry goes and the lookup makes it again whole, uploaded from its storage as a texture seen for the first time. A // lookup that is out of its creation budget leaves the stand-in for now, as for any new texture. Texture lock held. const VkTexImage *vk_texture_exact_by_id_locked(int id, view_t *view, int *has_view) { if (id < 0 || id >= ntslots || !tslots[id].used) return NULL; TexSlot *s = &tslots[id]; Tex *e = s->ce && s->cgen == agen[sinfo[id] & 0xffff] ? s->ce : NULL; // what the caller's lookup just resolved if (e && e->pub.skip) { if (exact.n >= EXACT_MAX) nvn_hm_clear(&exact, NULL); // never seen; they would be found again one by one nvn_hm_put(&exact, &e->key, (void *)(uintptr_t)1); n_exact++; static volatile int32_t n_logged; log_limited(&n_logged, "texture detail: texture %#llx (fmt %#x %ux%u, %u levels) is read by texel position or its " "size is asked for: made whole again (%llu so far)", (unsigned long long)e->pub.addr, e->t.d.format, e->t.d.width, e->t.d.height, e->pub.levels + e->pub.skip, (unsigned long long)n_exact); drop_tex(e, DROP_EXACT); } return vk_texture_by_id_locked(id, view, has_view); } // vk_dump: the swizzle selectors of the descriptor registered in slot id; 0 when the slot is empty. int vk_texture_slot_swz(int id, uint8_t swz[4]) { lock(); int ok = id >= 0 && id < ntslots && tslots[id].used; if (ok) { const nvn_texdesc *d = &tslots[id].t.d; swz[0] = (uint8_t)d->swizzle_r, swz[1] = (uint8_t)d->swizzle_g, swz[2] = (uint8_t)d->swizzle_b; swz[3] = (uint8_t)d->swizzle_a; } unlock(); return ok; } uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); } void vk_texture_new_submit(void) { lock(); // the last one walked the pool, or checked its entries by list if (pool_by_list || (round_checks && (uint64_t)round_checks * 2 >= pool_slots)) look_round++; round_checks = 0, pool_by_list = 0; if (!++submit_epoch) submit_epoch = 1; rate_tick(); sweep_pressure(); if (compact_on()) compact_step(); static uint64_t log_at; uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) / 60; if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) { log_at = f; rt_log("perf: frame %llu: texture aliases: content copied %llu (%llu from a released texture), filled from storage " "%llu, left uninitialised %llu; taken back with their own content %llu; read after an overwrite: given own " "memory %llu, kept in pool memory %llu", (unsigned long long)f * 60, (unsigned long long)n_alias_copied, (unsigned long long)n_alias_released, (unsigned long long)n_alias_upload, (unsigned long long)n_alias_uninit, (unsigned long long)n_alias_kept, (unsigned long long)n_unaliased_period, (unsigned long long)n_unalias_kept); n_alias_copied = n_alias_upload = n_alias_uninit = n_alias_kept = n_alias_released = 0; if (n_dinit_inline || n_dinit_upload) rt_log("perf: frame %llu: new depth images started at the replay's position %llu (%llu ended an open pass), " "in the upload batch %llu", (unsigned long long)f * 60, (unsigned long long)n_dinit_inline, (unsigned long long)n_dinit_inpass, (unsigned long long)n_dinit_upload); n_dinit_inline = n_dinit_inpass = n_dinit_upload = 0; if (n_disuse_shared || n_disuse_diverted) rt_log("perf: frame %llu: unsampled textures kept because another descriptor shares their address %llu; " "holds dropped by a mark that another entry at the address had taken %llu", (unsigned long long)f * 60, (unsigned long long)n_disuse_shared, (unsigned long long)n_disuse_diverted); n_disuse_shared = n_disuse_diverted = 0; n_unaliased_period = n_unalias_kept = 0; rt_log("perf: frame %llu: texture re-uploads %llu (%.1f MiB, %.2f ms), after WriteTexels %llu (%.1f MiB, %.2f ms); " "skipped with unchanged bytes %llu (%.1f MiB)", (unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6, (unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6, (unsigned long long)ru_same_n, ru_same_bytes / 1048576.0); ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = ru_same_n = ru_same_bytes = 0; uint64_t ck, calls, untr; vk_mem_snap_stats(&ck, &calls, &untr); rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu", (unsigned long long)f * 60, (unsigned long long)ck, (unsigned long long)n_checks, (unsigned long long)calls, (unsigned long long)untr); n_checks = 0; rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, " "under VRAM pressure %llu, under commit pressure %llu (made again soon after %llu, checks pausing those drops %llu; " "%llu checks went on to live textures with the released ones still above their bound); idle live textures evicted %llu (%.1f MiB; " "%llu walks found none, %.2f ms); unsampled ones evicted %llu (%.1f MiB), retiring %d (%.1f MiB), freed %llu, taken back while retiring " "%llu, back %llu, held %u (marks read %u); images reused %llu, made %llu (released ones of their " "parameters passed by: %llu released too lately, %llu drawn with or sampled lately, %llu named by a " "slot), pool bindings not tried %llu, " "superseded taken back %llu; render targets overwritten at another address %llu, dropped unused %llu " "(%llu of them early, with the card past its pressure share)", (unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops, (unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_commit_back, (unsigned long long)n_commit_held, (unsigned long long)n_commit_past, (unsigned long long)n_evict, n_evict_bytes / 1048576.0, (unsigned long long)n_evict_empty, ns_evict_empty / 1e6, (unsigned long long)n_disuse, n_disuse_bytes / 1048576.0, nretiring, retiring_bytes / 1048576.0, (unsigned long long)n_retire_freed, (unsigned long long)n_retire_cancel, (unsigned long long)n_disuse_back, holds.n, marks_reads, (unsigned long long)n_reused, (unsigned long long)n_reuse_new, (unsigned long long)n_reuse_young, (unsigned long long)n_reuse_used, (unsigned long long)n_reuse_slot, (unsigned long long)n_bind_skips, (unsigned long long)n_restarts, (unsigned long long)n_overwritten, (unsigned long long)n_over_drops, (unsigned long long)n_over_full); n_cap_drops = n_pressure_drops = n_commit_drops = n_commit_past = n_evict = n_evict_bytes = 0; n_evict_empty = ns_evict_empty = 0; n_commit_back = n_commit_held = 0; n_disuse = n_disuse_bytes = n_disuse_back = n_retire_freed = n_retire_cancel = 0; n_reused = n_reuse_new = n_bind_skips = n_restarts = n_overwritten = n_over_drops = 0; n_reuse_young = n_reuse_used = n_reuse_slot = n_over_full = 0; rt_log("perf: frame %llu: texture origins: released entries dropped past RELEASED_MAX %llu, past RELEASED_HARD %llu " "(overwritten %llu, intact past their gap %llu, intact %llu, made by the GPU %llu, all sampled lately %llu), " "replaced by another " "descriptor at their address %llu; entries the CPU does not write made without content at an address it " "can see: on their own %llu, over a released entry %llu, over a live one %llu; marked slots that met an " "entry without content %llu, that show no entry %llu; GPU-drawn entries written back before their drop %llu" "; hard-count batches %llu, %.1f ms in all, the longest %.2f ms", (unsigned long long)f * 60, (unsigned long long)n_drop_by[DROP_MAX], (unsigned long long)n_drop_by[DROP_HARD], (unsigned long long)n_hard_by[TRL_HC_OVER], (unsigned long long)n_hard_by[TRL_HC_INTACT_OLD], (unsigned long long)n_hard_by[TRL_HC_INTACT], (unsigned long long)n_hard_by[TRL_HC_GPU], (unsigned long long)n_hard_by[TRL_HC_LATELY], (unsigned long long)n_drop_by[DROP_REPLACED], (unsigned long long)n_cpu_empty[0], (unsigned long long)n_cpu_empty[1], (unsigned long long)n_cpu_empty[2], (unsigned long long)n_mark_empty, (unsigned long long)n_mark_none, (unsigned long long)n_drop_wb, (unsigned long long)n_hard_calls, ns_hard / 1e6, ns_hard_max / 1e6); memset(n_drop_by, 0, sizeof n_drop_by), memset(n_cpu_empty, 0, sizeof n_cpu_empty), memset(n_hard_by, 0, sizeof n_hard_by); n_mark_empty = n_mark_none = n_drop_wb = 0; n_hard_calls = ns_hard = ns_hard_max = 0; rt_log("perf: frame %llu: texture finalizes: slots marked %llu; slots of another live texture with the same " "parameters at the storage, left current %llu (terrain heightmaps %llu); lookups the stale rule gave the " "stand-in %llu (terrain heightmaps %llu), their own kept image %llu (NX2NAVI_TEX_STALE_KEPT)", (unsigned long long)f * 60, (unsigned long long)n_fin_own, (unsigned long long)n_fin_other, (unsigned long long)n_fin_other_hm, (unsigned long long)n_stale, (unsigned long long)n_stale_hm, (unsigned long long)n_stale_kept); n_fin_own = n_fin_other = n_fin_other_hm = n_stale = n_stale_hm = n_stale_kept = 0; if (split_mode > 0) rt_log("perf: frame %llu: split fill: entries the CPU does not write started with their storage's bytes: " "copied from the released entry they replaced %llu, filled from storage %llu (%.1f MiB, %.2f ms), " "filled again after a write %llu; left empty: image taken from a released entry %llu, storage over " "%llu MiB %llu", (unsigned long long)f * 60, (unsigned long long)n_split_copy, (unsigned long long)n_split_fill, n_split_bytes / 1048576.0, n_split_ns / 1e6, (unsigned long long)n_split_refill, (unsigned long long)n_split_reused, (unsigned long long)(SPLIT_FILL_MAX >> 20), (unsigned long long)n_split_big); n_split_copy = n_split_fill = n_split_bytes = n_split_ns = n_split_refill = n_split_reused = n_split_big = 0; } unlock(); } // ---------------------------------------------------------------- texel reads and writes typedef struct { unsigned level, layer0; int nz, is3d, bw, bh, bpb; size_t wb, hb, row, img, x0, y0; } Region; static int region_of(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img, Region *g) { g->bpb = nvn_fmt_bpb(t->d.format, &g->bw, &g->bh); if (!g->bpb || r->width <= 0 || r->height <= 0 || r->depth < 0 || r->xoffset < 0 || r->yoffset < 0 || r->zoffset < 0 || r->xoffset % g->bw || r->yoffset % g->bh) return 0; g->level = v && v->set_levels ? v->base_level : 0; g->layer0 = v && v->set_layers ? v->min_layer : 0; g->nz = r->depth ? r->depth : 1; g->is3d = t->d.target == NVN_TEXTURE_TARGET_3D; g->wb = (size_t)((r->width + g->bw - 1) / g->bw) * (size_t)g->bpb; g->hb = (size_t)(r->height + g->bh - 1) / (size_t)g->bh; g->row = row ? row : g->wb; g->img = img ? img : g->row * g->hb; g->x0 = (size_t)(r->xoffset / g->bw) * (size_t)g->bpb; g->y0 = (size_t)(r->yoffset / g->bh); return 1; } // Pool bytes <-> caller rows of the region alone (nvn_texels_box), not through a // linear copy of the whole level: for a small region of a large texture that cost tens of milliseconds while the guest // thread held its virtual core and the caller the texture lock (monsters2 replay, frames 765-790). static int storage_texels(const tex_t *t, const Region *g, const NVNcopyRegion *r, uint8_t *mem, int write) { for (int z = 0; z < g->nz; z++) { unsigned layer = g->is3d ? 0 : g->layer0 + (unsigned)r->zoffset + (unsigned)z; uint32_t slice = g->is3d ? (uint32_t)(r->zoffset + z) : 0; if (nvn_texels_box(&t->d, g->level, layer, (uint32_t)g->x0, (uint32_t)g->y0, slice, (uint32_t)g->wb, (uint32_t)g->hb, t->mem, mem + (size_t)z * g->img, g->row, !write)) return 0; } return 1; } static int read_aspect(const Tex *e, VkImageAspectFlags *a) { if (!e->f.depth && !e->f.stencil) { *a = VK_IMAGE_ASPECT_COLOR_BIT; return 1; } switch (e->t.d.format) { // packed depth-stencil storage has no single-aspect copy case 0x31: *a = VK_IMAGE_ASPECT_STENCIL_BIT; return 1; case 0x32: case 0x34: *a = VK_IMAGE_ASPECT_DEPTH_BIT; return 1; default: return 0; } } // An image region read through a staging buffer. texels_read_record records the copy into the open upload batch; // texels_read_take copies the rows out once that batch is done (vk_upload_flush(1)) and frees the buffer. Several // records before one flush cost one GPU round trip instead of one each. typedef struct { VkBuffer sb; VmaAllocation sa; const uint8_t *stg; size_t slice, ib; // ib: bytes per texel of an expanded RGB image, 0 when the image holds the storage layout uint32_t w; } TexRead; static int texels_read_record(Tex *e, const Region *g, const NVNcopyRegion *r, TexRead *t) { static volatile int32_t n_astc, n_ds; VkImageAspectFlags aspect; if (e->f.astc_bw) { log_limited(&n_astc, "ReadTexels of ASTC texture %#llx: the image holds decoded RGBA8; zeros returned", (unsigned long long)e->pub.addr); return 0; } if (!read_aspect(e, &aspect)) { log_limited(&n_ds, "texels of packed depth-stencil format %#x are not copied", e->t.d.format); return 0; } if (g->level < e->pub.skip || g->level - e->pub.skip >= e->pub.levels) return 0; // a level the image lacks uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height, il = g->level - e->pub.skip; t->w = w; t->ib = e->f.expand_rgb ? (size_t)g->bpb / 3 * 4 : 0; t->slice = t->ib ? (size_t)w * h * t->ib : g->wb * g->hb; t->stg = vk_staging(t->slice * (size_t)g->nz, &t->sb, &t->sa); VkBufferImageCopy *rg = calloc((size_t)g->nz, sizeof *rg); if (!rg) rt_fatal("vk: out of memory for a texture read"); for (int z = 0; z < g->nz; z++) rg[z] = (VkBufferImageCopy){.bufferOffset = t->slice * (size_t)z, .imageSubresource = {aspect, il, g->is3d ? 0 : g->layer0 + (uint32_t)r->zoffset + (uint32_t)z, 1}, .imageOffset = {r->xoffset, r->yoffset, g->is3d ? r->zoffset + z : 0}, .imageExtent = {w, h, 1}}; VkCommandBuffer cb = vk_upload_begin(); barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); vkCmdCopyImageToBuffer(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, t->sb, (uint32_t)g->nz, rg); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT, VK_ACCESS_2_HOST_READ_BIT); vk_upload_end(); free(rg); return 1; } static void texels_read_take(const Tex *e, const Region *g, const TexRead *t, uint8_t *mem) { VkCtx *c = vk_ctx(); vmaInvalidateAllocation(c->vma, t->sa, 0, VK_WHOLE_SIZE); for (int z = 0; z < g->nz; z++) { const uint8_t *s = t->stg + t->slice * (size_t)z; uint8_t *m = mem + (size_t)z * g->img; for (size_t y = 0; y < g->hb; y++) { if (t->ib) shrink_rgb(e->t.d.format, s + y * t->w * t->ib, m + y * g->row, t->w); else memcpy(m + y * g->row, s + y * g->wb, g->wb); } } vmaDestroyBuffer(c->vma, t->sb, t->sa); } // Image <-> caller rows through a staging buffer; the GPU copy is waited for. static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t *mem, int write) { static volatile int32_t n_astc, n_ds; VkCtx *c = vk_ctx(); VkImageAspectFlags aspect; if (!write) { TexRead t; if (!texels_read_record(e, g, r, &t)) return 0; vk_upload_flush(1); texels_read_take(e, g, &t, mem); return 1; } if (!read_aspect(e, &aspect)) { log_limited(&n_ds, "texels of packed depth-stencil format %#x are not copied", e->t.d.format); return 0; } // Texture detail: a level the image was made without takes no texels (the storage holds them, vk_write_texels); the // others are the image's level g->level - skip, the rectangle unchanged. if (g->level < e->pub.skip) return 1; uint32_t il = g->level - e->pub.skip; if (il >= e->pub.levels) return 0; e->had |= HAD_TEXELS, e->sfill = 0; // its content is no longer the storage's (split fill) uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height; int rgb = e->f.expand_rgb, astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu(); size_t texels = (size_t)w * h, ib = agpu ? 0 : astc ? 4 : rgb ? (size_t)g->bpb / 3 * 4 : 0; size_t slice = ib ? texels * ib : g->wb * g->hb; // agpu: the raw blocks, decoded by vk_astc_record VkBuffer sb; VmaAllocation sa; uint8_t *stg = vk_staging(slice * (size_t)g->nz, &sb, &sa); VkBufferImageCopy *rg = calloc((size_t)g->nz, sizeof *rg); for (int z = 0; z < g->nz; z++) { uint8_t *s = stg + slice * (size_t)z; const uint8_t *m = mem + (size_t)z * g->img; if (agpu) { for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb); } else if (astc) { uint8_t *tight = tmp_alloc(g->wb * g->hb); for (size_t y = 0; y < g->hb; y++) memcpy(tight + y * g->wb, m + y * g->row, g->wb); uint64_t a0 = perf_begin(); astc_decode_ldr_mt(tight, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, s); perf_end(PERF_ASTC, a0); tmp_free(tight, g->wb * g->hb); } else if (rgb) { for (size_t y = 0; y < g->hb; y++) expand_rgb(e->t.d.format, m + y * g->row, s + y * w * ib, w); } else { for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb); } rg[z] = (VkBufferImageCopy){.bufferOffset = slice * (size_t)z, .imageSubresource = {aspect, il, g->is3d ? 0 : g->layer0 + (uint32_t)r->zoffset + (uint32_t)z, 1}, .imageOffset = {r->xoffset, r->yoffset, g->is3d ? r->zoffset + z : 0}, .imageExtent = {w, h, 1}}; } VkCommandBuffer cb = vk_upload_begin(); vmaFlushAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); int done = 0; // A BC3 image takes whole 4x4 blocks, or a part that ends on the mip's edge; a region of a 5x5 or 6x6 footprint // may end inside a block (vk_astc_record checks the start corner). uint32_t mw = e->t.d.width >> g->level, mh = e->t.d.height >> g->level; int bc_cut = e->f.astc_bc && (((w & 3) && (uint32_t)r->xoffset + w != (mw ? mw : 1)) || ((h & 3) && (uint32_t)r->yoffset + h != (mh ? mh : 1))); if (agpu && !bc_cut) { // bc_cut: nothing recorded, logged below as not decoded VkAstcRegion *ar = calloc((size_t)g->nz, sizeof *ar); for (int z = 0; z < g->nz; z++) ar[z] = (VkAstcRegion){.src = slice * (size_t)z, .w = w, .h = h, .slices = 1, .level = rg[z].imageSubresource.mipLevel, .layer = rg[z].imageSubresource.baseArrayLayer, .offset = rg[z].imageOffset}; uint64_t a0 = perf_begin(); done = vk_astc_record(cb, sb, slice * (size_t)g->nz, ar, g->nz, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc, e->pub.image, aspect, VK_NULL_HANDLE); perf_end(PERF_ASTC, a0); free(ar); } if (!agpu) vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)g->nz, rg); else if (done <= 0) log_limited(&n_astc, "WriteTexels of ASTC texture %#llx not decoded on the GPU: left as it was", (unsigned long long)e->pub.addr); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_upload_end(); vk_upload_flush(1); free(rg); vmaDestroyBuffer(c->vma, sb, sa); return 1; } static void zero_rows(const Region *g, uint8_t *dst) { for (int z = 0; z < g->nz; z++) for (size_t y = 0; y < g->hb; y++) memset(dst + (size_t)z * g->img + y * g->row, 0, g->wb); } int vk_texture_read_image(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) { Region g; if (!t || !region_of(t, v, r, row, img, &g)) return 0; lock(); Tex *e = entry_of(t); int ok = 0; if (e) { make_fresh(e); ok = image_texels(e, &g, r, dst, 0); } unlock(); if (!ok) zero_rows(&g, dst); return ok; } // ReadTexels: the pool bytes are the content of a mirrored texture the GPU has not rendered into; every other texture // is read back from its image. void vk_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) { static volatile int32_t n_bad; Region g; if (!region_of(t, v, r, row, img, &g)) { log_limited(&n_bad, "ReadTexels: invalid region %d,%d,%d %dx%dx%d (format %#x)", r->xoffset, r->yoffset, r->zoffset, r->width, r->height, r->depth, t->d.format); return; } lock(); Tex *e = entry_of(t); int ok = 0; if (e && e->pub.mirrored && !e->pub.gpu_owned) ok = storage_texels(t, &g, r, dst, 0); else if (e) { make_fresh(e); ok = image_texels(e, &g, r, dst, 0); } unlock(); if (!ok) zero_rows(&g, dst); } // WriteTexels: a mirrored texture gets the texels in its pool bytes (and in its image once the GPU owns it); every // other texture in its image. void vk_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row, uint64_t img) { static volatile int32_t n_bad; Region g; if (!region_of(t, v, r, row, img, &g) || (v && v->set_format && v->format != t->d.format)) { log_limited(&n_bad, "WriteTexels: invalid region %d,%d,%d %dx%dx%d or view format (format %#x)", r->xoffset, r->yoffset, r->zoffset, r->width, r->height, r->depth, t->d.format); return; } lock(); Tex *e = entry_of(t); if (e && e->pub.mirrored) { if (!storage_texels(t, &g, r, (uint8_t *)src, 1)) log_limited(&n_bad, "WriteTexels: region outside level %u of texture %#llx", g.level, (unsigned long long)e->pub.addr); vk_guest_written((uintptr_t)t->mem, t->L.size); if (e->pub.gpu_owned) image_texels(e, &g, r, (uint8_t *)src, 1); else if (e->uploaded) e->wt_n++; } else if (e) { image_texels(e, &g, r, (uint8_t *)src, 1); } unlock(); } // ---------------------------------------------------------------- CPU reads of GPU-written storage // A mirrored texture the GPU rendered into holds its texels in its image only; its storage keeps what the CPU last put // there. The game may still read that storage with the CPU: NX2navi moves its generated grass maps between pools with // memcpy once the GPU is done, and the copy then carried zeros. Such a read writes the image back first. // // Every level and layer of every texture in v is copied to staging in one upload batch with one wait: one GPU round // trip per level and layer took 0.1 ms each, and a 260x260 grass map has 9 levels. The storage is then written in the // order of v, so where two textures share storage the later one wins. typedef struct { Tex *e; Region g; NVNcopyRegion r; TexRead t; } WbPart; static void write_back_n(Tex **v, int n) { int np = 0, cap = 0; WbPart *p = NULL; for (int i = 0; i < n; i++) { Tex *e = v[i]; const nvn_texdesc *d = &e->t.d; int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D; uint32_t levels = linear ? 1 : e->pub.levels + e->pub.skip, layers = linear || is3d ? 1 : e->pub.layers; for (uint32_t lv = linear ? 0 : e->pub.skip; lv < levels; lv++) // the game's levels the image holds for (uint32_t ly = 0; ly < layers; ly++) { if (np == cap) { cap = cap ? cap * 2 : 16; if (!(p = realloc(p, sizeof *p * (size_t)cap))) rt_fatal("vk: out of memory for a texture write-back"); } WbPart *q = &p[np]; view_t vw = {.set_levels = 1, .set_layers = 1, .base_level = (uint8_t)lv, .min_layer = (uint16_t)ly}; q->e = e; q->r = (NVNcopyRegion){.width = (int)mip(d->width, lv), .height = d->target == 0 || d->target == 3 ? 1 : (int)mip(d->height, lv), .depth = is3d ? (int)mip(d->depth, lv) : 1}; if (region_of(&e->t, &vw, &q->r, 0, 0, &q->g) && texels_read_record(e, &q->g, &q->r, &q->t)) np++; } } if (np) vk_upload_flush(1); for (int i = 0; i < np; i++) { WbPart *q = &p[i]; uint8_t *buf = tmp_alloc(q->g.img * (size_t)q->g.nz); if (!buf) rt_fatal("vk: out of memory for a texture write-back"); texels_read_take(q->e, &q->g, &q->t, buf); storage_texels(&q->e->t, &q->g, &q->r, buf, 1); tmp_free(buf, q->g.img * (size_t)q->g.nz); } free(p); for (int i = 0; i < n; i++) v[i]->gpu_dirty = 0; } // Mirrored textures overlapping [addr, addr+size) that the GPU wrote since their last write-back. static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) { int n = 0; for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) { Tex *e = owned[i]; if (__atomic_load_n(&e->gpu_dirty, __ATOMIC_RELAXED) && e->pub.addr + e->t.L.size > addr) { if (!hit) return 1; hit[n++] = e; } } return n; } const VkTexImage *vk_texture_at_locked(const VkTexImage *img, VkFormat format) { for (Tex *p = nvn_hm_get(&by_addr, &((Tex *)img)->key.addr); p; p = p->next_at) if (!p->released && p->pub.format == format && p->pub.width == img->width && p->pub.height == img->height) return &p->pub; return NULL; } // A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c // tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels. void vk_texture_storage_bound_locked(const VkTexImage *img) { Tex *e = (Tex *)img; // pub is the first member uint64_t f = vk_exec_recording_fence(); if (f > e->wr_fence) e->wr_fence = f; } uint64_t vk_texture_last_write(const tex_t *t) { lock(); Tex *e = entry_of(t); uint64_t f = e ? e->wr_fence : 0; unlock(); return f; } size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) { Region g; if (!region_of(t, v, r, row, img, &g)) return 0; return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb; } int vk_texture_cpu_stale(uint64_t addr, uint64_t size) { if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0; plat_mutex_lock_shared(&owned_lock); int dirty = owned_dirty(addr, size, NULL); plat_mutex_unlock_shared(&owned_lock); return dirty; } // The newest submission recorded to write a GPU-written mirrored texture overlapping [addr, addr+size); 0: none. uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size) { if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0; lock(); uint64_t f = 0; for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) { const Tex *e = owned[i]; if (!e->gpu_dirty || e->pub.addr + e->t.L.size <= addr) continue; if (e->gpu_fence > f) f = e->gpu_fence; if (e->wr_fence > f) f = e->wr_fence; } unlock(); return f; } // The image is the newest content only while nothing wrote the storage since the GPU render: NX2navi finalizes grass maps, // reuses their storage for new data and memcpys that data on, and writing the old image back first handed the copy // a stale map or stale resource bytes (wrong textures in the distance). Every hit is checked before any is written // back, since a write-back is itself a CPU write to storage another hit may share; the hits then go back oldest render // first, so where two descriptors share storage the newer render wins. void vk_texture_write_back(uint64_t addr, uint64_t size) { lock(); Tex **hit = malloc(sizeof *hit * (size_t)(nowned + 1)); if (!hit) rt_fatal("vk: out of memory for a texture write-back"); int n = owned_dirty(addr, size, hit), k = 0; // owned[] cannot change while tex_lock is held for (int i = 0; i < n; i++) { uint64_t now; if (storage_rewritten(hit[i], hit[i]->gpu_gen, &now) == 1) hit[i]->gpu_dirty = 0; else hit[k++] = hit[i]; } for (int i = 1; i < k; i++) for (int j = i; j > 0 && hit[j - 1]->gpu_seq > hit[j]->gpu_seq; j--) { Tex *x = hit[j]; hit[j] = hit[j - 1], hit[j - 1] = x; } write_back_n(hit, k); for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB); for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP); free(hit); unlock(); } // NX2navi's grass system renders a 128x128 RGBA8 map of cut and trampled grass into a CPU-cached pool every frame and its // CPU code reads the pool bytes without nvnMemoryPoolInvalidateMappedRange (cut reactions: blades, insects, the cut // itself). On the Switch the bytes are there once the frame's fence has passed. So when the // CPU waits for a fence, mirrored targets rendered by work up to that fence go back to their storage (wb_wanted: // targets over wb_limit only when not drawn every frame), unless the CPU rewrote it since. Targets drawn every frame // still go back on an invalidate, memcpy or ReadTexels. int vk_texture_small_dirty(void) { return __atomic_load_n(&small_pending, __ATOMIC_ACQUIRE); } // NX2NAVI_TRACE=gfx: a dirty target the fence write-back leaves alone because it is over the limit and drawn every frame // (what a CPU read after the wait would miss), or sends back although it is over the default limit. One line per // target address and outcome; the table is small and a full one stops logging. static void wb_note(const Tex *e, int skipped) { static uint64_t seen[2][64]; static int n_seen[2]; for (int i = 0; i < n_seen[skipped]; i++) if (seen[skipped][i] == e->pub.addr) return; if (n_seen[skipped] == 64) return; seen[skipped][n_seen[skipped]++] = e->pub.addr; const pool_t *p = e->t.pool; unsigned fl = p ? p->flags : 0; rt_log("vk: fence write-back %s %#llx %ux%u fmt %#x size %llu pool %s (limit %llu)", skipped ? "skipped-drawn-every-frame" : "over-default", (unsigned long long)e->pub.addr, e->t.d.width, e->t.d.height, e->t.d.format, (unsigned long long)e->t.L.size, !p ? "none" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_CACHED) ? "cpu-cached" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED) ? "cpu-uncached" : "other", (unsigned long long)wb_limit); } void vk_texture_write_back_small(uint64_t fence) { lock(); wb_cfg_init(); __atomic_store_n(&small_pending, 0, __ATOMIC_RELAXED); plat_mutex_lock_shared(&owned_lock); int n = 0, later = 0; for (int i = 0; i < nowned; i++) n += owned[i]->gpu_dirty && wb_wanted(owned[i]); Tex **hit = n ? malloc(sizeof *hit * (size_t)n) : NULL; if (n && !hit) rt_fatal("vk: out of memory for a texture write-back"); n = 0; for (int i = 0; i < nowned; i++) { Tex *e = owned[i]; if (!e->gpu_dirty) continue; if (!wb_wanted(e)) { if (wb_trace && e->gpu_fence <= fence) { wb_note(e, 1); perf_count(PERF_CNT_WB_SKIP); } continue; } if (e->gpu_fence > fence) later = 1; else { if (wb_trace && e->t.L.size > WB_SMALL_DEFAULT) wb_note(e, 0); hit[n++] = e; } } plat_mutex_unlock_shared(&owned_lock); if (later) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE); int k = 0; for (int i = 0; i < n; i++) { uint64_t now; if (storage_rewritten(hit[i], hit[i]->gpu_gen, &now) == 1) hit[i]->gpu_dirty = 0; else hit[k++] = hit[i]; } for (int i = 1; i < k; i++) for (int j = i; j > 0 && hit[j - 1]->gpu_seq > hit[j]->gpu_seq; j--) { Tex *x = hit[j]; hit[j] = hit[j - 1], hit[j - 1] = x; } write_back_n(hit, k); for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB); for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP); free(hit); unlock(); } // ---------------------------------------------------------------- samplers (LAS textures.c) static VkCompareOp compare_op(uint32_t f) { switch (f) { case NVN_COMPARE_FUNC_NEVER: return VK_COMPARE_OP_NEVER; case NVN_COMPARE_FUNC_LESS: return VK_COMPARE_OP_LESS; case NVN_COMPARE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL; case NVN_COMPARE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL; case NVN_COMPARE_FUNC_GREATER: return VK_COMPARE_OP_GREATER; case NVN_COMPARE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL; case NVN_COMPARE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL; default: return VK_COMPARE_OP_ALWAYS; } } static struct { int init, mirror_clamp_edge, minmax, border_no_format; float aniso; } sf; // aniso: NX2NAVI_ANISO, 0 = the game's static float aniso_setting(void) { const char *an = cfg_value("NX2NAVI_ANISO"); return strcmp(an, "game") ? (float)atoi(an) : 0; } static void aniso_log(void) { if (sf.aniso > 1) rt_log("vk: anisotropic filtering: %gx", sf.aniso); else rt_log("vk: anisotropic filtering: the game's"); } // Caller holds the lock. static void sampler_features(VkCtx *c) { if (sf.init) return; VkPhysicalDeviceCustomBorderColorFeaturesEXT cbf = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT}; VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, .pNext = c->has_custom_border_color ? &cbf : NULL}; VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12}; vkGetPhysicalDeviceFeatures2(c->phys, &f2); sf.mirror_clamp_edge = f12.samplerMirrorClampToEdge; sf.minmax = f12.samplerFilterMinmax; sf.border_no_format = c->has_custom_border_color && cbf.customBorderColors && cbf.customBorderColorWithoutFormat; sf.aniso = aniso_setting(); aniso_log(); sf.init = 1; } // NX2NAVI_ANISO raises the level for filtered, mipmapped colour samplers that repeat on U and V: the surface textures, // which tile, so ground and walls seen at a shallow angle stay sharp into the distance. Depth-compare (shadow) // samplers, nearest or unmipmapped ones (data maps, LUTs) and clamped ones keep the game's level: clamped samplers // read screen-sized buffers, whose coordinates can stretch steeply at an object's edge, where anisotropic taps reach // across it and leave a speckled fringe (the inventory's character preview). The game's own wrap modes decide: a // mirror-clamp one maps to MIRRORED_REPEAT where the device lacks mirror-clamp-to-edge (make_sampler). Caller holds the // lock, after sampler_features. static int aniso_raised(const samp_par *d) { int tiled = (d->ws == NVN_WRAP_MODE_REPEAT || d->ws == NVN_WRAP_MODE_MIRRORED_REPEAT) && (d->wt == NVN_WRAP_MODE_REPEAT || d->wt == NVN_WRAP_MODE_MIRRORED_REPEAT); return sf.aniso > d->aniso && tiled && !d->cmp_mode && d->magf == NVN_MAG_FILTER_LINEAR && (d->minf == NVN_MIN_FILTER_LINEAR_MIPMAP_NEAREST || d->minf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR); } static VkSampler make_sampler(VkCtx *c, const samp_par *d) { sampler_features(c); VkSamplerCreateInfo si = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; si.magFilter = d->magf == NVN_MAG_FILTER_LINEAR ? VK_FILTER_LINEAR : VK_FILTER_NEAREST; uint32_t mf = d->minf; si.minFilter = mf == 1 || mf == 3 || mf == 5 ? VK_FILTER_LINEAR : VK_FILTER_NEAREST; si.mipmapMode = mf == 4 || mf == 5 ? VK_SAMPLER_MIPMAP_MODE_LINEAR : VK_SAMPLER_MIPMAP_MODE_NEAREST; VkSamplerAddressMode modes[3]; const uint8_t wrap[3] = {d->ws, d->wt, d->wr}; int border = 0; for (int i = 0; i < 3; i++) { switch (wrap[i]) { case NVN_WRAP_MODE_CLAMP: case NVN_WRAP_MODE_CLAMP_TO_EDGE: modes[i] = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; break; case NVN_WRAP_MODE_MIRROR_CLAMP: case NVN_WRAP_MODE_MIRROR_CLAMP_TO_EDGE: modes[i] = sf.mirror_clamp_edge ? VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE : VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT; break; case NVN_WRAP_MODE_MIRROR_CLAMP_TO_BORDER: case NVN_WRAP_MODE_CLAMP_TO_BORDER: modes[i] = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; border = 1; break; case NVN_WRAP_MODE_MIRRORED_REPEAT: modes[i] = VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT; break; default: modes[i] = VK_SAMPLER_ADDRESS_MODE_REPEAT; } } si.addressModeU = modes[0], si.addressModeV = modes[1], si.addressModeW = modes[2]; si.minLod = d->lod_min; si.maxLod = d->lod_max > 1000 ? VK_LOD_CLAMP_NONE : d->lod_max; if (mf <= NVN_MIN_FILTER_LINEAR) si.maxLod = si.minLod + 0.25f; // not mipmapped: stay on the base level if (si.maxLod < si.minLod) si.maxLod = si.minLod; float lim = c->props.limits.maxSamplerLodBias; si.mipLodBias = fmaxf(-lim, fminf(lim, d->lod_bias)); // d->aniso holds NX2NAVI_ANISO's level where it applies (sampler_for, aniso_raised) if (d->aniso > 1) { si.anisotropyEnable = VK_TRUE; si.maxAnisotropy = fminf(d->aniso, c->props.limits.maxSamplerAnisotropy); } if (d->cmp_mode) { si.compareEnable = VK_TRUE; si.compareOp = compare_op(d->cmp_func); // A depth-compare sampler set to nearest returns 0 or 1 per tap, and the game's shaders smooth shadow edges by // a few taps spaced for its own shadow map texels. With NX2NAVI_SHADOW_SCALE those taps cover a fraction of the // area and the edges turn hard and stepped; linear filtering makes every tap a 2x2 PCF instead. NX2NAVI_SHADOW_PCF // does the same at the game's own map size (shadow_pcf). compare_sampler_note(d); if (shadow_scale() > 1 || shadow_pcf()) si.magFilter = si.minFilter = VK_FILTER_LINEAR; } VkSamplerCustomBorderColorCreateInfoEXT cb = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT}; VkSamplerReductionModeCreateInfo rm = {.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO}; const void **tail = &si.pNext; if (border) { const float *b = d->border; if (b[0] == 0 && b[1] == 0 && b[2] == 0 && b[3] == 0) si.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; else if (b[0] == 0 && b[1] == 0 && b[2] == 0 && b[3] == 1) si.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK; else if (b[0] == 1 && b[1] == 1 && b[2] == 1 && b[3] == 1) si.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE; else if (sf.border_no_format) { memcpy(cb.customBorderColor.float32, b, 16); si.borderColor = VK_BORDER_COLOR_FLOAT_CUSTOM_EXT; *tail = &cb; tail = &cb.pNext; } else si.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; } if (d->reduction && sf.minmax && !si.compareEnable) { rm.reductionMode = d->reduction == NVN_SAMPLER_REDUCTION_MIN ? VK_SAMPLER_REDUCTION_MODE_MIN : VK_SAMPLER_REDUCTION_MODE_MAX; *tail = &rm; } VkSampler s; VkResult r = vkCreateSampler(c->device, &si, NULL, &s); if (r != VK_SUCCESS) rt_fatal("vk: vkCreateSampler failed (%d)", (int)r); return s; } void vk_sampler_register(int id, const samp_par *s) { if (id < 0 || id >= (1 << 12)) rt_fatal("vk: sampler slot %d out of range", id); lock(); if (id >= nsslots) { int n = nsslots ? nsslots : 256; while (n <= id) n *= 2; SampSlot *p = realloc(sslots, sizeof *p * (size_t)n); if (!p) rt_fatal("vk: out of memory for sampler slots"); memset(p + nsslots, 0, sizeof *p * (size_t)(n - nsslots)); sslots = p, nsslots = n; } sslots[id].p = *s; sslots[id].s = sslots[id].sp = sslots[id].sc = sslots[id].sd[0] = sslots[id].sd[1] = VK_NULL_HANDLE; sslots[id].used = 1; samp_gen++; unlock(); } // DLSS upscaling (vk_sampler_dlss_bias): the scene's material textures, the filtered mipmapped ones that repeat, are // sampled for the output's resolution; clamped ones (post effects, LUTs, the UI) keep the game's bias. static float dlss_bias; static int dlss_biased(const samp_par *k) { int rep_s = k->ws == NVN_WRAP_MODE_REPEAT || k->ws == NVN_WRAP_MODE_MIRRORED_REPEAT; int rep_t = k->wt == NVN_WRAP_MODE_REPEAT || k->wt == NVN_WRAP_MODE_MIRRORED_REPEAT; return !k->cmp_mode && k->minf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR && (rep_s || rep_t); } void vk_sampler_dlss_bias(float b) { lock(); if (b != dlss_bias) { rt_log("vk: texture LOD bias %.2f on the scene's tiled textures (the upscaler's render scale; was %.2f)", b, dlss_bias); dlss_bias = b; for (int i = 0; i < nsslots; i++) // looked up again (the key moves) sslots[i].s = sslots[i].sp = sslots[i].sc = sslots[i].sd[0] = sslots[i].sd[1] = VK_NULL_HANDLE; samp_gen++; } unlock(); } // NX2NAVI_ANISO, read again at each present after a settings change (vk_backend.c present_image, beside the DLSS bias of // vk_dlaa_frame): the raised level is part of the sampler key (sampler_for), so on a change the slots look their // samplers up again and the descriptor caches follow samp_gen, as for vk_sampler_dlss_bias. The old level's samplers // stay until shutdown like every sampler: one set per level at most. Before the first sampler there is nothing to do: // sampler_features reads the setting then. void vk_sampler_aniso_live(void) { static unsigned seen = ~0u; unsigned v = cfg_version(); if (v == seen) return; seen = v; lock(); float a = aniso_setting(); if (sf.init && a != sf.aniso) { sf.aniso = a; aniso_log(); for (int i = 0; i < nsslots; i++) // looked up again (the key moves) sslots[i].s = sslots[i].sp = sslots[i].sc = sslots[i].sd[0] = sslots[i].sd[1] = VK_NULL_HANDLE; samp_gen++; } unlock(); } // Caller holds the lock. static VkSampler sampler_for(VkCtx *c, const samp_par *s) { sampler_features(c); samp_par k; memset(&k, 0, sizeof k); k = *s; if (dlss_bias != 0 && dlss_biased(&k)) k.lod_bias += dlss_bias; // part of the key: samplers live until shutdown if (aniso_raised(&k)) k.aniso = sf.aniso; // likewise VkSampler v = (VkSampler)nvn_hm_get(&samplers, &k); if (!v) { v = make_sampler(c, &k); nvn_hm_put(&samplers, &k, (void *)v); } return v; } VkSampler vk_sampler_for(const samp_par *s) { VkCtx *c = vk_ctx(); if (!c || !s) return VK_NULL_HANDLE; lock(); VkSampler v = sampler_for(c, s); unlock(); return v; } // Samplers live until shutdown, so a slot keeps the one its parameters map to until it is registered again. VkSampler vk_sampler_by_id_locked(int id) { VkCtx *c = vk_ctx(); if (!c) return VK_NULL_HANDLE; VkSampler v = VK_NULL_HANDLE; if (id >= 0 && id < nsslots && sslots[id].used) { SampSlot *s = &sslots[id]; if (!s->s) s->s = sampler_for(c, &s->p); v = s->s; } return v; } VkSampler vk_sampler_point_by_id_locked(int id) { VkCtx *c = vk_ctx(); if (!c || id < 0 || id >= nsslots || !sslots[id].used) return VK_NULL_HANDLE; SampSlot *s = &sslots[id]; if (!s->sp) { samp_par k = s->p; k.magf = NVN_MAG_FILTER_NEAREST; k.minf = k.minf == NVN_MIN_FILTER_LINEAR ? NVN_MIN_FILTER_NEAREST : k.minf > NVN_MIN_FILTER_LINEAR ? NVN_MIN_FILTER_NEAREST_MIPMAP_NEAREST : k.minf; k.aniso = 1; s->sp = sampler_for(c, &k); } return s->sp; } // A depth-compare read through a sampler without a compare is undefined in Vulkan. NVIDIA's driver compares anyway; AMD's // returns the raw depth, so the game's shadow tests (which sample this way) come out wrong there. Such a read gets the // slot's sampler with the compare on: the game's function when it set a valid one, LEQUAL otherwise, as NVIDIA's driver // compares. NX2NAVI_SHADOW_COMPARE=0 binds the game's sampler as it is. VkSampler vk_sampler_compare_by_id_locked(int id) { VkCtx *c = vk_ctx(); if (!c || id < 0 || id >= nsslots || !sslots[id].used) return VK_NULL_HANDLE; SampSlot *s = &sslots[id]; static int on = -1; if (on < 0) on = cfg_int("NX2NAVI_SHADOW_COMPARE") != 0; if (s->p.cmp_mode || !on) return vk_sampler_by_id_locked(id); if (!s->sc) { samp_par k = s->p; k.cmp_mode = 1; if (k.cmp_func < NVN_COMPARE_FUNC_NEVER || k.cmp_func > NVN_COMPARE_FUNC_ALWAYS) k.cmp_func = NVN_COMPARE_FUNC_LEQUAL; static int told; if (told++ < 8) rt_log("vk: a depth-compare read through sampler slot %d without a compare (function %u): compared with %u", id, (unsigned)s->p.cmp_func, (unsigned)k.cmp_func); s->sc = sampler_for(c, &k); } return s->sc; } // Texture detail: a texture made without its top `skip` levels has the game's level i + skip at its level i, and the // LOD the device works out for it is the game's less skip, so the level a sample lands on is the game's wherever that // level is kept. A sampler's LOD clamp counts the game's levels and has to move down with them: a clamp to levels 0-2 // left alone would let the image's level 2 through, the game's level 4. The bias is relative and stays. A clamp that is // open (from 0, up to more levels than a texture has) is the same sampler. Only draws that bind texture and sampler // as one resource come here (vk_exec_draw.c tex_info_full); the sampler pool's entries serve every texture and keep // the game's clamp. VkSampler vk_sampler_detail_by_id_locked(int id, uint32_t skip) { VkCtx *c = vk_ctx(); if (!skip || skip > 2) return vk_sampler_by_id_locked(id); if (!c || id < 0 || id >= nsslots || !sslots[id].used) return VK_NULL_HANDLE; SampSlot *s = &sslots[id]; if (!s->sd[skip - 1]) { samp_par k = s->p; if (k.lod_min > 0) k.lod_min = fmaxf(k.lod_min - (float)skip, 0); if (k.lod_max < 32) k.lod_max = fmaxf(k.lod_max - (float)skip, 0); // 32 and more: above any texture's last level if (k.lod_min == s->p.lod_min && k.lod_max == s->p.lod_max) { s->sd[skip - 1] = vk_sampler_by_id_locked(id); } else { static int told; // a clamp that leaves levels out; one up to a texture's last level changes nothing if ((s->p.lod_min > 0 || s->p.lod_max < 8) && told++ < 8) rt_log("vk: texture detail: sampler slot %d clamps the LOD to %g..%g; %g..%g for a texture made without %u " "levels", id, s->p.lod_min, s->p.lod_max, k.lod_min, k.lod_max, skip); s->sd[skip - 1] = sampler_for(c, &k); } } return s->sd[skip - 1]; } VkSampler vk_sampler_by_id(int id) { lock(); VkSampler v = vk_sampler_by_id_locked(id); unlock(); return v; } int vk_sampler_params(int id, samp_par *out) { lock(); int ok = id >= 0 && id < nsslots && sslots[id].used; if (ok) *out = sslots[id].p; unlock(); return ok; } // ---------------------------------------------------------------- compaction (vk_tex_compact.h) // The block table: which of VMA's blocks hold textures, and what else is known to lie in them (the caller's rows: the // VRAM mirrors, vk_memory.c mirror_rows). VMA does not say what a block holds, so the table is built from the entries' // own allocations (Tex.blk_mem, own_memory); memory of another kind in a block is not in it. // // What may be done with a texture's allocation (compact_what): // - drop: a released entry the size cap would drop too (drop_stale's rule: it comes back intact, was released // stale_after presents ago, and no lookup reaches it); // - move: an entry whose image holds what its storage holds and nothing else (comes_back_intact: the CPU provides it, // the GPU never wrote it), uploaded, with no upload waiting, that no command ever wrote (wr_fence) and no shader // sampled for COMPACT_IDLE presents (samp_frame, by the pool's marks: without them nothing moves), single-sampled, // not depth or stencil, and not larger than `most`; // - stay: everything else, and every row of the caller. // The table is of one memory type, the device-local one whose blocks hold the most texture memory: a move lands in the // type it came from, and the allocator's free bytes are counted per heap. #define COMPACT_SPARSE_PCT 50 #define COMPACT_IDLE PRESENTS(600) static int compact_held(uint64_t mem); typedef struct { TcpBlock *b; int nb, cb; TcpAlloc *a; Tex **ae; // per allocation: its entry; NULL for a row that is not a texture's int na, ca; nvn_hmap by_mem; // device memory handle -> 1 + the block's index in b uint32_t type; // the memory type the table is of uint64_t tex_bytes, other_bytes; int marks; // the pool's marks tell which textures are sampled (compact_marks) } CompTable; static int compact_marks(void) { return marks_tell(); } static int compact_what(const Tex *e, uint64_t now, int marks, uint64_t most) { if (!e->pub.image || e->pub.gone || e->retiring || !comes_back_intact(e)) return TCP_STAY; if (e->released && now - e->rel_frame >= stale_after(e) && idle(e, now)) return TCP_DROP; if (!marks || !e->uploaded || e->up_retry || e->wr_fence || e->clobbered || e->pub.samples > 1 || e->f.depth || e->f.stencil || e->bytes > most) return TCP_STAY; return now - e->samp_frame >= COMPACT_IDLE ? TCP_MOVE : TCP_STAY; } // What keeps an allocation compact_what leaves where it is, for the holes line (vk_texture_holes): a VRAM mirror (a row // of the caller's, no entry), a texture whose content only the GPU made, one a shader sampled within COMPACT_IDLE // presents, or one that cannot move for another reason (compact_what's list). A sparse block that one kind alone // keeps is what a change to that kind's placement or rule would give back. enum { KEEP_MIRROR = 1, KEEP_GPU = 2, KEEP_USED = 4, KEEP_OTHER = 8 }; static int stay_why(const Tex *e, int marks, uint64_t most) { if (!e) return KEEP_MIRROR; if (!e->pub.image || e->pub.gone || e->retiring) return KEEP_OTHER; if (!comes_back_intact(e)) return KEEP_GPU; if (!marks || !e->uploaded || e->up_retry || e->wr_fence || e->clobbered || e->pub.samples > 1 || e->f.depth || e->f.stencil || e->bytes > most) return KEEP_OTHER; return KEEP_USED; } static void table_add(CompTable *t, uint64_t mem, uint64_t size, uint64_t bytes, int what, Tex *e) { void *v = nvn_hm_get(&t->by_mem, &mem); uint32_t bi = v ? (uint32_t)((uintptr_t)v - 1) : (uint32_t)t->nb; if (!v) { if (t->nb == t->cb) { t->cb = t->cb ? t->cb * 2 : 256; if (!(t->b = realloc(t->b, sizeof *t->b * (size_t)t->cb))) rt_fatal("vk: out of memory for the block table"); } t->b[t->nb] = (TcpBlock){.id = mem, .size = size, .held = (uint8_t)compact_held(mem)}; nvn_hm_put(&t->by_mem, &mem, (void *)(uintptr_t)(++t->nb)); } if (t->na == t->ca) { t->ca = t->ca ? t->ca * 2 : 4096; t->a = realloc(t->a, sizeof *t->a * (size_t)t->ca), t->ae = realloc(t->ae, sizeof *t->ae * (size_t)t->ca); if (!t->a || !t->ae) rt_fatal("vk: out of memory for the block table"); } t->a[t->na] = (TcpAlloc){.block = bi, .what = (uint8_t)what, .bytes = bytes}; t->ae[t->na++] = e; } static void table_free(CompTable *t) { free(t->b), free(t->a), free(t->ae), free(t->by_mem.keys), free(t->by_mem.vals); memset(t, 0, sizeof *t); } // tex_lock held. other[0..nother): allocations of another kind the caller knows, which stay. static void compact_table(CompTable *t, const VkBlockRow *other, int nother, uint64_t most) { VkCtx *c = vk_ctx(); memset(t, 0, sizeof *t); nvn_hm_init(&t->by_mem, 8); uint64_t by_type[VK_MAX_MEMORY_TYPES] = {0}, best = 0, now = frames_now(); for (uint32_t i = 0; i < by_key.cap; i++) { const Tex *e = by_key.vals[i]; if (e && e->bytes && !e->blk_own && e->blk_type < VK_MAX_MEMORY_TYPES) by_type[e->blk_type] += e->bytes; } for (uint32_t ty = 0; c && ty < c->mem_props.memoryTypeCount && ty < VK_MAX_MEMORY_TYPES; ty++) if ((c->mem_props.memoryTypes[ty].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) && by_type[ty] > best) best = by_type[ty], t->type = ty; if (!best) return; t->marks = compact_marks(); for (uint32_t i = 0; i < by_key.cap; i++) { Tex *e = by_key.vals[i]; if (!e || !e->bytes || e->blk_own || e->blk_type != t->type) continue; table_add(t, e->blk_mem, e->blk_size, e->bytes, compact_what(e, now, t->marks, most), e); t->tex_bytes += e->bytes; } for (int i = 0; i < nother; i++) { if (other[i].type != t->type) continue; table_add(t, other[i].mem, other[i].block_size, other[i].bytes, TCP_STAY, NULL); t->other_bytes += other[i].bytes; } tcp_tally(t->b, t->nb, t->a, t->na); } // ---- the pass (NX2NAVI_TEX_COMPACT, default 0; read at start) // With the setting on, the holes are closed by moving textures: every COMPACT_PERIOD presents, when the free space // inside the device-local blocks is at least NX2NAVI_TEX_COMPACT_MIN_MB (one block while the video memory in use is // above the VRAM pressure threshold), the block table is built and vk_tex_compact.h chooses the blocks to empty // (vk_texture_compact_plan, at a present). Each submission then takes a few of the listed textures (compact_step): // - a released one the size cap would drop is dropped (drop_tex, DROP_COMPACT), as that rule does it; // - any other gets a new image in a hole of a block that is not being emptied (compact_move): the allocator is asked // for existing blocks only, and it fills the fullest first; the old image is copied into it on the GPU, in the // upload batch, and the entry takes the new image and allocation. Nothing is decoded or uploaded again. // Moving was chosen over releasing because releasing has nothing to take: the entries a compaction would have to // release are live ones the texture pool still names, which idle() never holds for (in four traced sessions the idle // part of 4 to 5 GiB of CPU-provided textures was 0 MiB), and the path that takes such entries away (evict_disused) is // off by default for the stand-ins it left on screen. A moved texture never shows a stand-in: until a slot is filled // again it names the old image, which holds the same texels. // // Who holds a moved texture's old image or its views, and what becomes of each: // - the entry itself (pub.image, alloc, the block fields): swapped under the texture lock; every later lookup, upload, // read and write-back takes the new image. // - slots and whatever was resolved through them (a slot's cached entry and view, the texture pool's tokens and // descriptors, the draws' descriptor memos): tex_gen_bump moves the generation of the entry's address, as for an // entry made or destroyed, so each is resolved again and gets a view of the new image. // - the texture pool's descriptor sets: the current set is rewritten by the next sync, the others when they are next // taken. The old image and its views stay until the pool has taken every set of its ring since the move // (vk_pool_takes, as retire_sweep waits) and the GPU is past the submissions recorded until then (compact_old_sweep). // - the replay's view cache, the stand-ins of 3D BC targets and the HDR path's notes, all keyed by the image handle // (vk_exec_image_gone): purged when the old image goes, not before, so a submission that holds one of its views // keeps a live view. Views of the new image are made on demand under its own handle. // - command buffers recorded or in flight, a submission held at a wait, and the submission being recorded: they // name the old image or its views, which live on with their texels unchanged; vk_defer_image then waits for the // submission being recorded as for any destroyed texture. // - the upload batch: the copy is recorded behind every upload of the old image (the batch and the queue keep // their order) and ahead of every submission recorded after the move. // - uploads in progress: none can be. A candidate is uploaded with no retry waiting, an upload records all its // parts under the texture lock, and the first upload of an ASTC texture that changes its format (astc_reformat) // is behind it. // - render targets, the lists of GPU-written textures (owned, rts, gwin), the image reuse lists, and the inputs of // DLSS, FSR and frame generation: all of these hold textures the GPU wrote, which never move (comes_back_intact, // wr_fence). // - a flat texture's 3D copy (flat3d) and a compute pass's depth twin (vk_exec_draw.c twin_update): images of their // own, made from the entry's image when used; the copy keeps its content, the twin is copied again for each use. // // The work is bounded by the frame: COMPACT_FRAME_N textures and NX2NAVI_TEX_COMPACT_FRAME_MB of them per presented // frame, none while the texture pool loads an area, while this frame already spent COMPACT_WORK_NS making or uploading // textures, or right after a frame that took a quarter longer than the frames before it (compact_slack). A texture // larger than the frame's bytes is not moved at all, so no frame copies more than that. // A chosen block the pass cannot finish is given up: one of its textures came into use, or no hole outside the chosen // blocks took it. A block emptied of its textures that the allocator does not give back holds memory of another kind // (or is the one empty block the allocator keeps per memory type); it is not chosen again for COMPACT_HELD_FRAMES. // A pass that gave nothing back doubles the wait for the next look, up to COMPACT_WAIT_MAX. #define COMPACT_PERIOD PRESENTS(600) #define COMPACT_WAIT_MAX PRESENTS(18000) #define COMPACT_PASS_FRAMES PRESENTS(3600) // a pass still not through its list after this many presents ends there #define COMPACT_SETTLE PRESENTS(120) // presents a finished pass waits for its blocks to be freed #define COMPACT_HELD_FRAMES PRESENTS(36000) #define COMPACT_ROOM_PCT 50 #define COMPACT_PASS_MAX ((uint64_t)1 << 30) // bytes one pass moves at most #define COMPACT_FRAME_N 4 #define COMPACT_WORK_NS 1000000 #define COMPACT_OLD_MAX ((uint64_t)64 << 20) // the moved textures' old images hold this much at most while they wait #define COMPACT_OLD_N 256 #define COMPACT_VICTIMS 32 #define COMPACT_HELD_N 64 typedef struct { VkImage img; VmaAllocation alloc; uint64_t bytes, takes, due, frame; } CompOld; typedef struct { uint64_t mem, size; volatile int32_t freed; uint8_t gave_up; } CompVictim; typedef struct { Tex *e; uint8_t v; } CompItem; static struct { int on; // -1: the settings are not read yet uint64_t min_bytes, frame_bytes; volatile int32_t active; // a pass runs; set and cleared under the texture lock, read without it volatile uint64_t next_look; // gfx_counters.frames from which the holes are looked at again uint64_t wait; // presents to the look after the next pass (0: COMPACT_PERIOD) CompVictim v[COMPACT_VICTIMS]; // the blocks the pass empties int nv; CompItem *item; // their textures, block by block in the plan's order int nitem, at; uint64_t frame0, done_frame; // gfx_counters.frames when the pass began, when its list was through uint64_t plan_ns, step_ns; // time the plan took (a present), and the steps (the submitting thread) uint64_t holes0; // free bytes inside the device-local blocks when it began uint64_t n_moved, moved_bytes, n_dropped, dropped_bytes, n_left; uint64_t frame, frame_n, frame_b; // this presented frame's count and bytes CompOld old[COMPACT_OLD_N]; // moved textures' old images, waiting for the pool sets and the GPU int nold; uint64_t old_bytes; struct { uint64_t mem, until; } held[COMPACT_HELD_N]; // blocks emptied and not given back unsigned held_next; volatile uint64_t last_ns, avg_ns; // the last frame's interval and a running mean of them (compact_frame_time) uint64_t passes, returned; // passes so far, and what they gave back int idle_logs, old_told; } CP = {.on = -1}; static int compact_on(void) { if (CP.on < 0) { int mb = cfg_int("NX2NAVI_TEX_COMPACT_MIN_MB"), fmb = cfg_int("NX2NAVI_TEX_COMPACT_FRAME_MB"); CP.min_bytes = (uint64_t)(mb < 64 ? 64 : mb > 65536 ? 65536 : mb) << 20; CP.frame_bytes = (uint64_t)(fmb < 1 ? 1 : fmb > 32 ? 32 : fmb) << 20; CP.on = cfg_bool("NX2NAVI_TEX_COMPACT") ? 1 : 0; if (CP.on) rt_log("vk: texture compaction on (NX2NAVI_TEX_COMPACT): idle textures move out of sparsely used blocks of " "video memory once %llu MiB or more is free inside the blocks (NX2NAVI_TEX_COMPACT_MIN_MB), at most " "%d textures and %llu MiB a frame (NX2NAVI_TEX_COMPACT_FRAME_MB)", (unsigned long long)(CP.min_bytes >> 20), COMPACT_FRAME_N, (unsigned long long)(CP.frame_bytes >> 20)); } return CP.on; } int vk_texture_compact_on(void) { return compact_on(); } static void compact_frame_time(uint64_t d) { // each present (vk_texstat_frame) uint64_t avg = __atomic_load_n(&CP.avg_ns, __ATOMIC_RELAXED); __atomic_store_n(&CP.last_ns, d, __ATOMIC_RELAXED); __atomic_store_n(&CP.avg_ns, avg ? avg + d / 16 - avg / 16 : d, __ATOMIC_RELAXED); } // Whether the frame being made has time to spare (see above). static int compact_slack(void) { if (vk_exec_pool_loading()) return 0; if (__atomic_load_n(&ts_ns[VK_TS_CREATE], __ATOMIC_RELAXED) + __atomic_load_n(&ts_ns[VK_TS_UPLOAD], __ATOMIC_RELAXED) > COMPACT_WORK_NS) return 0; uint64_t last = __atomic_load_n(&CP.last_ns, __ATOMIC_RELAXED), avg = __atomic_load_n(&CP.avg_ns, __ATOMIC_RELAXED); return last <= avg + avg / 4; } // mem is a block a pass emptied of its textures and the allocator kept (compact_finish). static int compact_held(uint64_t mem) { uint64_t now = frames_now(); for (int i = 0; i < COMPACT_HELD_N; i++) if (CP.held[i].mem == mem && CP.held[i].until > now) return 1; return 0; } // The allocator gave device memory back to the driver (vk_headroom.c on_free; any thread, no lock): a block of the // pass counts as returned. void vk_texture_block_freed(uint64_t mem) { if (!__atomic_load_n(&CP.active, __ATOMIC_ACQUIRE)) return; for (int i = 0; i < COMPACT_VICTIMS; i++) if (__atomic_load_n(&CP.v[i].mem, __ATOMIC_RELAXED) == mem) __atomic_store_n(&CP.v[i].freed, 1, __ATOMIC_RELEASE); } static int compact_victim(uint64_t mem) { for (int i = 0; i < CP.nv; i++) if (CP.v[i].mem == mem && !CP.v[i].gave_up) return 1; return 0; } // destroy_tex: e goes another way while the pass lists it. static void compact_forget(Tex *e) { for (int i = CP.at; i < CP.nitem; i++) if (CP.item[i].e == e) CP.item[i].e = NULL; e->compacting = 0; } // The old images of moved textures whose views no pool set can hold any more, and no submission that held one still // runs (see above): their views and they go, as a destroyed texture's. static void compact_old_sweep(void) { uint64_t takes = vk_pool_takes(), done = vk_sync_fence_done(), now = frames_now(); int k = 0; for (int i = 0; i < CP.nold; i++) { CompOld *o = &CP.old[i]; if (!o->due && takes - o->takes > VK_POOL_SETS) o->due = vk_sync_issued(); // The takes that count are those of syncs that left no slot behind (vk_exec_draw.c pool_sync), and with no // creation time to spare, or nothing drawn through the pool, there may be none for a long time. The old // images wait all the same: COMPACT_OLD_MAX bounds what they hold, and the pass moves nothing more. if (!o->due && !CP.old_told && now - o->frame > COMPACT_PASS_FRAMES) { CP.old_told = 1; rt_log("vk: texture compaction: %d old images of moved textures (%.1f MiB) still wait for the texture " "pool to take all its sets, %llu presents after the move (%llu takes since); the pass waits", CP.nold, CP.old_bytes / 1048576.0, (unsigned long long)(now - o->frame), (unsigned long long)(takes - o->takes)); } if (o->due && done >= o->due) { vk_exec_image_gone(o->img); vk_defer_image(o->img, o->alloc); CP.old_bytes -= o->bytes; continue; } CP.old[k++] = *o; } CP.nold = k; } // e gets a new image in a hole of a block the pass does not empty, with the old image's texels (see above). 0 when no // such hole took it; e is as it was then. static int compact_move(VkCtx *c, Tex *e) { const ImgKey *k = &e->ik; VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = k->flags, .imageType = (VkImageType)k->type, .format = (VkFormat)k->format, .extent = {k->w, k->h, k->d}, .mipLevels = k->levels, .arrayLayers = k->layers, .samples = (VkSampleCountFlagBits)k->samples, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = k->usage, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; VkImage img; if (vkCreateImage(c->device, &ci, NULL, &img) != VK_SUCCESS) return 0; VkMemoryRequirements mr; vkGetImageMemoryRequirements(c->device, img, &mr); // existing blocks of the old allocation's memory type only: a new block, or another type's, would be no gain VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT, .memoryTypeBits = 1u << e->blk_type}; VmaAllocation a = NULL; VmaAllocationInfo at; int ok = (mr.memoryTypeBits >> e->blk_type & 1) && vmaAllocateMemory(c->vma, &mr, &ai, &a, &at) == VK_SUCCESS; if (ok && compact_victim((uint64_t)(uintptr_t)at.deviceMemory)) { // only a block being emptied had the room vmaFreeMemory(c->vma, a); ok = 0; } if (ok && vmaBindImageMemory(c->vma, a, img) != VK_SUCCESS) { vmaFreeMemory(c->vma, a); ok = 0; } if (!ok) { vkDestroyImage(c->device, img, NULL); return 0; } vk_exec_image_gone(img); // views cached under the handle of a destroyed image (create_tex) VkCommandBuffer cb = vk_upload_begin(); VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT}; VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_NONE, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = img, .subresourceRange = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib}; vkCmdPipelineBarrier2(cb, &di); VkImageCopy rg[16]; for (uint32_t lv = 0; lv < e->pub.levels;) { // every level whole, all layers at once uint32_t n = 0; for (; lv < e->pub.levels && n < 16; lv++, n++) rg[n] = (VkImageCopy){.srcSubresource = {e->pub.aspect, lv, 0, e->pub.layers}, .dstSubresource = {e->pub.aspect, lv, 0, e->pub.layers}, .extent = {mip(e->pub.width, lv), mip(e->pub.height, lv), mip(e->pub.depth, lv)}}; vkCmdCopyImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, img, VK_IMAGE_LAYOUT_GENERAL, n, rg); } barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_upload_end(); CP.old[CP.nold++] = (CompOld){e->pub.image, e->alloc, e->bytes, vk_pool_takes(), 0, frames_now()}; CP.old_bytes += e->bytes; if (e->released) rel_bytes -= e->bytes; e->pub.image = img, e->alloc = a, e->reused = 0; own_memory(c, e); if (e->released) rel_bytes += e->bytes; tex_gen_bump(e->key.addr); return 1; } // The pass is over: its line, the blocks the allocator kept, and when the holes are looked at next. static void compact_finish(uint64_t now) { VkCtx *c = vk_ctx(); uint64_t holes = 0, back = 0; int n_back = 0, n_gave = 0, n_kept = 0; for (int i = 0; i < CP.nv; i++) { CompVictim *v = &CP.v[i]; if (__atomic_load_n(&v->freed, __ATOMIC_ACQUIRE)) { n_back++, back += v->size; } else if (v->gave_up) { n_gave++; } else { // emptied of its textures and still there n_kept++; CP.held[CP.held_next++ % COMPACT_HELD_N] = (typeof(CP.held[0])){v->mem, now + COMPACT_HELD_FRAMES}; } } if (c) { VmaBudget vb[VK_MAX_MEMORY_HEAPS]; vmaGetHeapBudgets(c->vma, vb); for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) if ((c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) && vb[h].statistics.blockBytes > vb[h].statistics.allocationBytes) holes += vb[h].statistics.blockBytes - vb[h].statistics.allocationBytes; } CP.passes++, CP.returned += back; CP.wait = back ? 0 : CP.wait ? (CP.wait * 2 < COMPACT_WAIT_MAX ? CP.wait * 2 : COMPACT_WAIT_MAX) : 2 * COMPACT_PERIOD; uint64_t wait = CP.wait ? CP.wait : COMPACT_PERIOD; const double M = 1048576.0; rt_log("vk: texture compaction: pass %llu over %llu frames: %llu textures moved (%.1f MiB), %llu released ones dropped " "(%.1f MiB), %llu left where they were; %d of %d blocks emptied and given back to the driver, %.0f MiB; %d " "given up (a texture came into use or found no room elsewhere), %d emptied of textures and kept by the " "allocator (its one empty block, or memory of another kind); free inside the blocks %.0f MiB before, %.0f MiB " "now; %.2f ms on the submitting thread, %.2f ms for the plan; %.0f MiB given back by all passes so far; next " "look in %llu presents", (unsigned long long)CP.passes, (unsigned long long)(now - CP.frame0), (unsigned long long)CP.n_moved, CP.moved_bytes / M, (unsigned long long)CP.n_dropped, CP.dropped_bytes / M, (unsigned long long)CP.n_left, n_back, CP.nv, back / M, n_gave, n_kept, CP.holes0 / M, holes / M, CP.step_ns / 1e6, CP.plan_ns / 1e6, CP.returned / M, (unsigned long long)wait); free(CP.item); CP.item = NULL, CP.nitem = CP.at = 0; __atomic_store_n(&CP.next_look, now + wait, __ATOMIC_RELAXED); __atomic_store_n(&CP.active, 0, __ATOMIC_RELEASE); } // Each submission, texture lock held (vk_texture_new_submit): the old images that can go, then this frame's share of // the pass. static void compact_step(void) { if (CP.nold) compact_old_sweep(); if (!CP.active) return; uint64_t t0 = plat_now_ns(), now = frames_now(); if (now != CP.frame) CP.frame = now, CP.frame_n = 0, CP.frame_b = 0; if (CP.at < CP.nitem && now - CP.frame0 > COMPACT_PASS_FRAMES) { // the frames never had the time: what is left stays for (; CP.at < CP.nitem; CP.at++) { CompItem *it = &CP.item[CP.at]; if (it->e) it->e->compacting = 0, CP.v[it->v].gave_up = 1, CP.n_left++; } } if (CP.at < CP.nitem && compact_slack()) { VkCtx *c = vk_ctx(); int marks = compact_marks(); while (c && CP.at < CP.nitem && CP.frame_n < COMPACT_FRAME_N && CP.frame_b < CP.frame_bytes && CP.nold < COMPACT_OLD_N && CP.old_bytes < COMPACT_OLD_MAX) { CompItem it = CP.item[CP.at]; Tex *e = it.e; if (e && CP.frame_b && CP.frame_b + e->bytes > CP.frame_bytes) break; // the next frame's CP.at++; if (!e) continue; // destroyed since the plan e->compacting = 0; CompVictim *v = &CP.v[it.v]; if (e->blk_mem != v->mem) continue; // its image was made again since (astc_reformat): no longer in the block uint64_t bytes = e->bytes; // as it is now: sampled, written or taken back since the plan, it may no longer move int what = v->gave_up ? TCP_STAY : compact_what(e, now, marks, CP.frame_bytes); if (what == TCP_DROP) { drop_tex(e, DROP_COMPACT); CP.n_dropped++, CP.dropped_bytes += bytes, CP.frame_n++; } else if (what == TCP_MOVE && compact_move(c, e)) { CP.n_moved++, CP.moved_bytes += bytes, CP.frame_n++, CP.frame_b += bytes; } else { v->gave_up = 1, CP.n_left++; } } } CP.step_ns += plat_now_ns() - t0; if (CP.at < CP.nitem || CP.nold) return; if (!CP.done_frame) CP.done_frame = now; int waiting = 0; // blocks emptied whose memory the deferred destroys have not freed yet for (int i = 0; i < CP.nv; i++) waiting += !CP.v[i].gave_up && !__atomic_load_n(&CP.v[i].freed, __ATOMIC_ACQUIRE); if (!waiting || now - CP.done_frame >= COMPACT_SETTLE) compact_finish(now); } // At a present, no lock: whether the holes are to be looked at now (vk_memory.c vk_mem_census, which then gathers the // VRAM mirrors' rows for vk_texture_compact_plan). int vk_texture_compact_due(void) { return compact_on() && !__atomic_load_n(&CP.active, __ATOMIC_ACQUIRE) && frames_now() >= __atomic_load_n(&CP.next_look, __ATOMIC_RELAXED); } // Looks at the holes and, when emptying blocks would give enough back, starts a pass (see above). void vk_texture_compact_plan(const VkBlockRow *other, int nother) { VkCtx *c = vk_ctx(); if (!c || !compact_on()) return; uint64_t t0 = plat_now_ns(); lock(); uint64_t now = frames_now(); if (CP.active || CP.nold || now < CP.next_look) { // the last pass's old images first unlock(); return; } __atomic_store_n(&CP.next_look, now + (CP.wait ? CP.wait : COMPACT_PERIOD), __ATOMIC_RELAXED); VmaBudget vb[VK_MAX_MEMORY_HEAPS]; vmaGetHeapBudgets(c->vma, vb); uint64_t holes = 0, used = 0, budget = 0; for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) if ((c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) && vb[h].statistics.blockBytes > vb[h].statistics.allocationBytes) holes += vb[h].statistics.blockBytes - vb[h].statistics.allocationBytes; int exact; int pressed = vram_pressure > 0 && vk_mem_vram(&used, &budget, &exact) && budget && used >= budget / 100 * (uint64_t)vram_pressure; uint64_t min = pressed ? vk_vma_block_size() : CP.min_bytes; if (holes < min) { unlock(); return; } CompTable t; compact_table(&t, other, nother, CP.frame_bytes); uint32_t heap = c->mem_props.memoryTypes[t.type].heapIndex, order[COMPACT_VICTIMS]; const VmaStatistics *hs = &vb[heap < VK_MAX_MEMORY_HEAPS ? heap : 0].statistics; TcpParams p = {.free_bytes = hs->blockBytes > hs->allocationBytes ? hs->blockBytes - hs->allocationBytes : 0, .sparse_pct = COMPACT_SPARSE_PCT, .room_pct = COMPACT_ROOM_PCT, .min_gain = min, .max_move = COMPACT_PASS_MAX, .max_blocks = COMPACT_VICTIMS}; TcpPlan plan; int n = t.nb ? tcp_plan(t.b, t.nb, &p, order, COMPACT_VICTIMS, &plan) : 0; CompItem *item = n ? malloc(sizeof *item * (size_t)(plan.n_move + plan.n_drop + 1)) : NULL; if (!item) { TcpStats s; tcp_stats(t.b, t.nb, COMPACT_SPARSE_PCT, &s); CP.wait = CP.wait ? (CP.wait * 2 < COMPACT_WAIT_MAX ? CP.wait * 2 : COMPACT_WAIT_MAX) : 2 * COMPACT_PERIOD; __atomic_store_n(&CP.next_look, now + CP.wait, __ATOMIC_RELAXED); if (CP.idle_logs < 8) { CP.idle_logs++; const double M = 1048576.0; rt_log("vk: texture compaction: %.0f MiB free inside the blocks and no pass: %u of %u blocks with textures are " "at most half full, %u of them (%.0f MiB) hold nothing but textures that could move or be dropped, " "and a pass needs %.0f MiB of them with room elsewhere for what moves (%.0f MiB free in the memory " "type's heap)%s; next look in %llu presents", holes / M, s.sparse, s.blocks, s.emptiable, s.emptiable_bytes / M, min / M, p.free_bytes / M, t.marks ? "" : "; the texture pool's marks are not in use, so no texture counts as unsampled", (unsigned long long)CP.wait); } table_free(&t); unlock(); return; } CP.nv = n, CP.nitem = 0; for (int j = 0; j < n; j++) { CP.v[j] = (CompVictim){.mem = t.b[order[j]].id, .size = t.b[order[j]].size}; for (int i = 0; i < t.na; i++) { // the block's textures; n is at most COMPACT_VICTIMS if (t.a[i].block != order[j] || !t.ae[i] || t.a[i].what == TCP_STAY) continue; item[CP.nitem++] = (CompItem){t.ae[i], (uint8_t)j}; t.ae[i]->compacting = 1; } } for (int j = n; j < COMPACT_VICTIMS; j++) CP.v[j] = (CompVictim){0}; table_free(&t); CP.item = item, CP.at = 0; CP.frame0 = now, CP.done_frame = 0, CP.holes0 = holes, CP.step_ns = 0; CP.n_moved = CP.moved_bytes = CP.n_dropped = CP.dropped_bytes = CP.n_left = 0, CP.old_told = 0; CP.plan_ns = plat_now_ns() - t0; __atomic_store_n(&CP.active, 1, __ATOMIC_RELEASE); unlock(); } void vk_texture_holes(const VkBlockRow *other, int nother, VkTexHoles *h) { memset(h, 0, sizeof *h); lock(); CompTable t; compact_on(); // reads the frame's bytes, which are also the largest texture that moves compact_table(&t, other, nother, CP.frame_bytes); TcpStats s; tcp_stats(t.b, t.nb, COMPACT_SPARSE_PCT, &s); *h = (VkTexHoles){.type = t.type, .marks = t.marks, .blocks = s.blocks, .sparse = s.sparse, .emptiable = s.emptiable, .block_bytes = s.block_bytes, .tex_bytes = t.tex_bytes, .other_bytes = t.other_bytes, .move_bytes = s.move_bytes, .drop_bytes = s.drop_bytes, .sparse_bytes = s.sparse_bytes, .sparse_listed = s.sparse_listed, .emptiable_bytes = s.emptiable_bytes, .emptiable_move = s.emptiable_move, .emptiable_drop = s.emptiable_drop}; // the sparse blocks a plan could not choose, by what stays in them uint8_t *why = t.nb ? calloc((size_t)t.nb, 1) : NULL; for (int i = 0; why && i < t.na; i++) if (t.a[i].what == TCP_STAY && t.a[i].block < (uint32_t)t.nb) why[t.a[i].block] |= (uint8_t)stay_why(t.ae[i], t.marks, CP.frame_bytes); for (int i = 0; why && i < t.nb; i++) { const TcpBlock *k = &t.b[i]; if (!tcp_sparse(k, COMPACT_SPARSE_PCT) || tcp_emptiable(k, COMPACT_SPARSE_PCT) || !why[i]) continue; h->kept++, h->kept_bytes += k->size; for (int j = 0; j < VK_HOLES_KINDS; j++) { if (!(why[i] >> j & 1)) continue; h->kept_with[j]++; if (why[i] == 1u << j) h->kept_only[j]++, h->kept_only_bytes[j] += k->size; } } free(why); table_free(&t); unlock(); } // ---------------------------------------------------------------- statistics and shutdown void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) { lock(); if (images) *images = by_key.n; if (uploads) *uploads = n_uploads; if (reuploads) *reuploads = n_reuploads; unlock(); } // A walk over the entries under the texture lock, with the sizes the entries keep (Tex.bytes): about 0.1 ms with ten // thousand of them, at most once a HOLES_PERIOD (vk_memory.c owner_log) without the perf trace. void vk_texture_census(VkTexCensus *s) { memset(s, 0, sizeof *s); lock(); uint64_t now = frames_now(); for (uint32_t i = 0; i < by_key.cap; i++) { const Tex *e = by_key.vals[i]; if (!e) continue; uint64_t bytes = e->alloc ? e->bytes : 0; if (e->released) { s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes; if (now - e->rel_frame < stale_after(e)) s->bytes_released_young += bytes; } else { s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes; if (!comes_back_intact(e)) s->bytes_live_gpu += bytes; if (idle(e, now)) s->bytes_live_idle += bytes; } } unlock(); } // The texture cache's host memory for the memory report (vk_memory.c system_log): the entries with their write-back // snapshots (SNAP_MAX each), the slot tables (fixed once the first slot is registered, vk_texture_register) and the CPU's // temporaries (tmp_alloc): those in use now and the most at once since the last call, which starts the next span. // The same walk as vk_texture_census, at the same time. void vk_texture_host(VkTexHost *h) { memset(h, 0, sizeof *h); lock(); for (uint32_t i = 0; i < by_key.cap; i++) { const Tex *e = by_key.vals[i]; if (!e) continue; h->entries++; h->bytes += sizeof *e + (e->snap ? SNAP_MAX : 0); } h->bytes += (uint64_t)by_key.cap * (sizeof *by_key.vals + by_key.klen) + (uint64_t)ntslots * sizeof *tslots + (uint64_t)nsslots * sizeof *sslots; if (sinfo) h->bytes += (sizeof *sinfo + sizeof *slot_next + sizeof *slot_prev + sizeof *slot_grp + sizeof *slot_bucket) << 20; unlock(); h->tmp = (uint64_t)__atomic_load_n(&tmp_now, __ATOMIC_RELAXED); h->tmp_peak = (uint64_t)__atomic_exchange_n(&tmp_peak, (int64_t)h->tmp, __ATOMIC_RELAXED); } // ---- reuse of released textures (vk_texture_reuse_log) // How often a released entry is taken back and how long it had been released then, and how often an entry is made // again after a drop that a bound or a pressure made (the image made and its content uploaded or decoded once more). // The two tell what the bounds on the released entries cost: an entry taken back after minutes is one a smaller bound // may have dropped, and one made again soon after its drop is one the bound did drop. Counted since the last line; the // dropped keys are remembered in a table of DROPPED_N slots indexed by the key's hash, a newer drop taking the slot. // Ages are wall seconds (secs_since); 1-3 min and 3-5 min lie either side of the default hold (TRL_HOLD_S). enum { AGE_YOUNG, AGE_10S, AGE_1M, AGE_3M, AGE_5M, AGE_OLD, AGE_N }; static const char *const age_names[AGE_N] = {"under 2 s", "2-10 s", "10-60 s", "1-3 min", "3-5 min", "longer"}; static uint64_t rb_n[AGE_N], rb_bytes[AGE_N]; // take-backs by the time released (tex_lock) static uint64_t ra_n[AGE_N], ra_bytes, ra_why[DROP_N]; // made again by the time since the drop, and by its reason // The same split by kind and own memory, which decides whether a shorter hold for some kind could save memory without // a take-back finding its entry gone: an entry that comes back intact from the game's memory, one another target was // drawn over, one whose content only the GPU made; each under 1 MiB of own memory (none included) or more. enum { RK_INTACT, RK_INTACT_BIG, RK_OVER, RK_OVER_BIG, RK_GPU, RK_GPU_BIG, RK_N }; static const char *const rk_names[RK_N] = {"intact under 1 MiB", "intact 1 MiB or more", "overwritten under 1 MiB", "overwritten 1 MiB or more", "made by the GPU under 1 MiB", "made by the GPU 1 MiB or more"}; static uint64_t rkb_n[RK_N][AGE_N], rkb_bytes[RK_N][AGE_N], rka_n[RK_N][AGE_N]; // tex_lock #define DROPPED_N 8192 static struct { uint64_t kh, frame; uint8_t why, kind; } dropped[DROPPED_N]; // tex_lock static int age_of(uint32_t s) { // seconds return s < 2 ? AGE_YOUNG : s < 10 ? AGE_10S : s < 60 ? AGE_1M : s < 180 ? AGE_3M : s < 300 ? AGE_5M : AGE_OLD; } static int reuse_kind(const Tex *e) { int big = e->alloc && e->bytes >= (1ull << 20); return (e->overwritten ? RK_OVER : comes_back_intact(e) ? RK_INTACT : RK_GPU) + big; } static void reuse_taken_back(const Tex *e) { // tex_lock held; e is being taken back, rel_frame still its release int a = age_of(secs_since(e->rel_frame)), k = reuse_kind(e); uint64_t own = e->alloc ? e->bytes : 0; rb_n[a]++, rb_bytes[a] += own; rkb_n[k][a]++, rkb_bytes[k][a] += own; } // Drops of an entry the game may still ask for: past a bound, or under a pressure, or unsampled. static int drop_costs(int why) { return why == DROP_MAX || why == DROP_HARD || why == DROP_CAP || why == DROP_COMMIT || why == DROP_VRAM || why == DROP_IDLE_VRAM || why == DROP_IDLE_COMMIT || why == DROP_DISUSE || why == DROP_AGE; } static void reuse_dropped(const Tex *e, int why) { // tex_lock held if (!drop_costs(why)) return; uint64_t kh = key_hash(&e->key); dropped[kh % DROPPED_N].kh = kh, dropped[kh % DROPPED_N].frame = frames_now(), dropped[kh % DROPPED_N].why = (uint8_t)why; dropped[kh % DROPPED_N].kind = (uint8_t)reuse_kind(e); // the entry is gone when it is made again } static void reuse_made(const Tex *e) { // tex_lock held; e was just made uint64_t kh = key_hash(&e->key); if (dropped[kh % DROPPED_N].kh != kh || !dropped[kh % DROPPED_N].why) return; int a = age_of(secs_since(dropped[kh % DROPPED_N].frame)); ra_n[a]++, ra_bytes += e->alloc ? e->bytes : 0, ra_why[dropped[kh % DROPPED_N].why]++; rka_n[dropped[kh % DROPPED_N].kind % RK_N][a]++; dropped[kh % DROPPED_N].why = 0; } void vk_texture_reuse_log(uint64_t frame) { char line[1536]; const double M = 1048576.0; lock(); int o = snprintf(line, sizeof line, "vk: released textures frame %llu, since the last line: taken back by the time " "released:", (unsigned long long)frame); for (int a = 0; a < AGE_N; a++) o += snprintf(line + o, sizeof line - (size_t)o, " %s %llu (%.0f MiB);", age_names[a], (unsigned long long)rb_n[a], rb_bytes[a] / M); uint64_t again = 0; for (int a = 0; a < AGE_N; a++) again += ra_n[a]; o += snprintf(line + o, sizeof line - (size_t)o, " made again after a drop %llu (%.0f MiB; by the time since the drop:", (unsigned long long)again, ra_bytes / M); for (int a = 0; a < AGE_N; a++) o += snprintf(line + o, sizeof line - (size_t)o, "%s %s %llu", a ? "," : "", age_names[a], (unsigned long long)ra_n[a]); snprintf(line + o, sizeof line - (size_t)o, "; by the drop: released past the hold %llu, past the size bound " "%llu, past the count %llu (of them past the hard count %llu), VRAM pressure %llu released and %llu idle, " "commit %llu released and %llu idle, unsampled %llu); dropped past the hold %llu (%.0f MiB); now %d " "released, %.0f MiB of a bound of %.0f MiB, hold %u s", (unsigned long long)ra_why[DROP_AGE], (unsigned long long)ra_why[DROP_CAP], (unsigned long long)(ra_why[DROP_MAX] + ra_why[DROP_HARD]), (unsigned long long)ra_why[DROP_HARD], (unsigned long long)ra_why[DROP_VRAM], (unsigned long long)ra_why[DROP_IDLE_VRAM], (unsigned long long)ra_why[DROP_COMMIT], (unsigned long long)ra_why[DROP_IDLE_COMMIT], (unsigned long long)ra_why[DROP_DISUSE], (unsigned long long)n_age_drops, n_age_bytes / M, nreleased, rel_bytes / M, released_cap_bytes() == UINT64_MAX ? 0.0 : released_cap_bytes() / M, released_hold()); // the split by kind and own memory: one group per kind that had any, the take-backs and the makes again by age char kline[1536]; int ko = snprintf(kline, sizeof kline, "vk: released textures frame %llu by kind and own memory, since the last line " "(ages under 2 s/2-10 s/10-60 s/1-3 min/3-5 min/longer):", (unsigned long long)frame); int kinds = 0; for (int k = 0; k < RK_N && ko < (int)sizeof kline - 200; k++) { uint64_t any = 0, late = 0; for (int a = 0; a < AGE_N; a++) any += rkb_n[k][a] + rka_n[k][a]; for (int a = AGE_3M; a < AGE_N; a++) late += rkb_bytes[k][a]; if (!any) continue; kinds++; ko += snprintf(kline + ko, sizeof kline - (size_t)ko, "%s %s: taken back %llu/%llu/%llu/%llu/%llu/%llu (%.0f MiB " "after a minute), made again %llu/%llu/%llu/%llu/%llu/%llu", kinds > 1 ? ";" : "", rk_names[k], (unsigned long long)rkb_n[k][0], (unsigned long long)rkb_n[k][1], (unsigned long long)rkb_n[k][2], (unsigned long long)rkb_n[k][3], (unsigned long long)rkb_n[k][4], (unsigned long long)rkb_n[k][5], late / M, (unsigned long long)rka_n[k][0], (unsigned long long)rka_n[k][1], (unsigned long long)rka_n[k][2], (unsigned long long)rka_n[k][3], (unsigned long long)rka_n[k][4], (unsigned long long)rka_n[k][5]); } memset(rb_n, 0, sizeof rb_n), memset(rb_bytes, 0, sizeof rb_bytes), memset(ra_n, 0, sizeof ra_n); memset(ra_why, 0, sizeof ra_why), ra_bytes = 0, n_age_drops = n_age_bytes = 0; memset(rkb_n, 0, sizeof rkb_n), memset(rkb_bytes, 0, sizeof rkb_bytes), memset(rka_n, 0, sizeof rka_n); unlock(); rt_log("%s", line); if (kinds) rt_log("%s", kline); } // With NX2NAVI_TRACE=perf every 600 frames, after vk_memory.c's vram line: what the live entries' own memory holds. By // kind: decoded ASTC (as BC3 or RGBA8) and other textures whose content the CPU provides, all of which a re-upload // brings back (comes_back_intact), and the GPU-written ones (render targets, textures the GPU drew or copied into), // whose content lives only in the image. "idle" is the part no lookup reached for IDLE_ROUNDS pool walks and // STALE_FRAMES presents (touch). Then the entries that share their storage address with another live entry (each // descriptor of the same storage gets an image of its own), the formats holding the most, and the largest entries. void vk_texture_census_log(uint64_t frame) { enum { K_BC1, K_BC3, K_RGBA8, K_CPU, K_GPU, K_N }; static const char *const kn[K_N] = {"ASTC as BC1", "ASTC as BC3", "ASTC as RGBA8", "other CPU-written", "GPU-written"}; uint64_t n[K_N] = {0}, b[K_N] = {0}, in[K_N] = {0}, ib[K_N] = {0}, bound = 0, dup_n = 0, dup_b = 0; uint64_t sk_n[3] = {0}, sk_b[3] = {0}; // texture detail: entries made without 1 and 2 levels, and their memory struct { int fmt; uint64_t n, b; } fm[48]; int nfm = 0; struct { uint64_t b; uint32_t w, h, d, levels; int fmt, gpu; } big[4] = {{0}}; // copied: read after unlock // the addresses whose extra entries hold the most, with their largest few entries (copied: read after unlock) struct { uint64_t addr, cnt, extra; struct { uint64_t b; uint32_t w, h, d; int fmt, gpu; } e[4]; } dup[3] = {{0}}; const double M = 1048576.0; lock(); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); for (uint32_t i = 0; i < by_key.cap; i++) { const Tex *e = by_key.vals[i]; if (!e || e->released) continue; if (!e->bytes) { bound += e->bound_to != NULL; continue; } int k = !comes_back_intact(e) ? K_GPU : e->f.astc_bw ? (e->f.astc_bc == 2 ? K_BC1 : e->f.astc_bc ? K_BC3 : K_RGBA8) : K_CPU; n[k]++, b[k] += e->bytes; if (e->pub.skip < 3) sk_n[e->pub.skip]++, sk_b[e->pub.skip] += e->bytes; if (idle(e, now)) in[k]++, ib[k] += e->bytes; int j = 0; while (j < nfm && fm[j].fmt != (int)e->f.vk) j++; if (j == nfm && nfm < 48) fm[nfm++] = (typeof(fm[0])){(int)e->f.vk, 0, 0}; if (j < nfm) fm[j].n++, fm[j].b += e->bytes; for (int q = 0; q < 4; q++) if (e->bytes > big[q].b) { memmove(big + q + 1, big + q, sizeof *big * (size_t)(3 - q)); big[q] = (typeof(big[0])){e->bytes, e->pub.width, e->pub.height, e->pub.layers * e->pub.depth, e->pub.levels, (int)e->f.vk, k == K_GPU}; break; } } for (uint32_t i = 0; i < by_addr.cap; i++) { // live entries with own memory at one address, beyond the largest uint64_t cnt = 0, sum = 0, most = 0; for (const Tex *p = by_addr.vals[i]; p; p = p->next_at) if (!p->released && p->bytes) cnt++, sum += p->bytes, most = p->bytes > most ? p->bytes : most; if (cnt > 1) dup_n += cnt - 1, dup_b += sum - most; for (int q = 0; cnt > 1 && q < 3; q++) { if (sum - most <= dup[q].extra) continue; memmove(dup + q + 1, dup + q, sizeof *dup * (size_t)(2 - q)); memset(&dup[q], 0, sizeof dup[q]); dup[q].cnt = cnt, dup[q].extra = sum - most; for (const Tex *p = by_addr.vals[i]; p; p = p->next_at) { if (p->released || !p->bytes) continue; dup[q].addr = p->pub.addr; for (int x = 0; x < 4; x++) if (p->bytes > dup[q].e[x].b) { memmove(dup[q].e + x + 1, dup[q].e + x, sizeof dup[q].e[0] * (size_t)(3 - x)); dup[q].e[x].b = p->bytes, dup[q].e[x].w = p->pub.width, dup[q].e[x].h = p->pub.height; dup[q].e[x].d = p->pub.layers * p->pub.depth, dup[q].e[x].fmt = (int)p->f.vk; dup[q].e[x].gpu = !comes_back_intact(p); break; } } break; } } unlock(); char line[1024]; int o = snprintf(line, sizeof line, "mem: textures frame %llu: live with own memory by kind, idle part in brackets:", (unsigned long long)frame); for (int k = 0; k < K_N; k++) o += snprintf(line + o, sizeof line - (size_t)o, " %s %llu %.0f MiB (%llu %.0f MiB);", kn[k], (unsigned long long)n[k], b[k] / M, (unsigned long long)in[k], ib[k] / M); o += snprintf(line + o, sizeof line - (size_t)o, " bound into pool memory %llu; sharing an address with another %llu %.0f MiB", (unsigned long long)bound, (unsigned long long)dup_n, dup_b / M); if (detail_most()) // nothing more in the line at full detail snprintf(line + o, sizeof line - (size_t)o, "; texture detail: made without their top level %llu %.0f MiB, without " "two %llu %.0f MiB, made whole again for a shader %llu (%u shaders loaded read a texture by texel " "position or size)", (unsigned long long)sk_n[1], sk_b[1] / M, (unsigned long long)sk_n[2], sk_b[2] / M, (unsigned long long)n_exact, gfx_shader_exact_seen()); rt_log("%s", line); if (vk_astc_bc1()) rt_log("mem: ASTC as BC1 so far: %llu textures made BC1 at their first upload (%.0f MiB, half of what BC3 takes), " "%llu made BC3 again for content with alpha; checks in the uploads' copies %llu (%.0f MiB), of the storage " "before an upload that untiles on the CPU %llu (%.0f MiB) in %.1f ms", (unsigned long long)n_bc1_made, n_bc1_made_bytes / M, (unsigned long long)n_bc1_widened, (unsigned long long)n_bc1_up_checks, n_bc1_up_bytes / M, (unsigned long long)n_bc1_checks, n_bc1_check_bytes / M, n_bc1_check_ns / 1e6); for (int x = 0; x < nfm; x++) // by bytes, largest first for (int y = x + 1; y < nfm; y++) if (fm[y].b > fm[x].b) { typeof(fm[0]) t = fm[x]; fm[x] = fm[y], fm[y] = t; } o = snprintf(line, sizeof line, "mem: textures frame %llu: VkFormats holding the most:", (unsigned long long)frame); for (int x = 0; x < nfm && x < 8; x++) o += snprintf(line + o, sizeof line - (size_t)o, " %d: %llu %.0f MiB;", fm[x].fmt, (unsigned long long)fm[x].n, fm[x].b / M); o += snprintf(line + o, sizeof line - (size_t)o, " largest:"); for (int q = 0; q < 4 && big[q].b; q++) o += snprintf(line + o, sizeof line - (size_t)o, " %ux%ux%u fmt %d %u levels %.0f MiB%s;", big[q].w, big[q].h, big[q].d, big[q].fmt, big[q].levels, big[q].b / M, big[q].gpu ? " (GPU-written)" : ""); rt_log("%s", line); if (!dup[0].cnt) return; // Which entries share an address: each descriptor of the same storage (another size, format or level count) gets // an image of its own, and its memory beside the others'. o = snprintf(line, sizeof line, "mem: textures frame %llu: addresses with the most extra entries:", (unsigned long long)frame); for (int q = 0; q < 3 && dup[q].cnt && o < (int)sizeof line - 200; q++) { o += snprintf(line + o, sizeof line - (size_t)o, " %#llx: %llu entries, %.0f MiB beyond the largest (", (unsigned long long)dup[q].addr, (unsigned long long)dup[q].cnt, dup[q].extra / M); for (int x = 0; x < 4 && dup[q].e[x].b && o < (int)sizeof line - 80; x++) o += snprintf(line + o, sizeof line - (size_t)o, "%s%ux%ux%u fmt %d %.1f MiB%s", x ? ", " : "", dup[q].e[x].w, dup[q].e[x].h, dup[q].e[x].d, dup[q].e[x].fmt, dup[q].e[x].b / M, dup[q].e[x].gpu ? " GPU-written" : ""); o += snprintf(line + o, sizeof line - (size_t)o, ");"); } rt_log("%s", line); } void vk_texture_shutdown(void) { VkCtx *c = vk_ctx(); lock(); Tex **list = malloc(sizeof *list * (size_t)(by_key.n + 1)); int n = 0; for (uint32_t i = 0; i < by_key.cap; i++) if (by_key.vals[i]) list[n++] = by_key.vals[i]; for (int i = 0; i < n; i++) drop_tex(list[i], DROP_SHUTDOWN); free(list); for (int i = 0; i < CP.nold; i++) { // moved textures' old images (compact_old_sweep) vk_exec_image_gone(CP.old[i].img); vk_defer_image(CP.old[i].img, CP.old[i].alloc); } free(CP.item); CP.item = NULL, CP.nitem = CP.at = CP.nold = 0, CP.old_bytes = 0; __atomic_store_n(&CP.active, 0, __ATOMIC_RELEASE); for (uint32_t i = 0; i < samplers.cap; i++) if (samplers.vals[i] && c) vkDestroySampler(c->device, (VkSampler)samplers.vals[i], NULL); free(by_key.keys), free(by_key.vals), free(by_addr.keys), free(by_addr.vals), free(samplers.keys), free(samplers.vals); by_key = (nvn_hmap){.klen = sizeof(TexKey)}; by_addr = (nvn_hmap){.klen = 8}; for (uint32_t i = 0; i < img_reqs.cap; i++) free(img_reqs.vals[i]); free(img_reqs.keys), free(img_reqs.vals), free(reuse.keys), free(reuse.vals); // reuse: emptied by destroy_tex img_reqs = (nvn_hmap){.klen = sizeof(ImgKey)}; reuse = (nvn_hmap){.klen = sizeof(ImgKey)}; samplers = (nvn_hmap){.klen = sizeof(samp_par)}; free(tslots), free(sslots); tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0, pool_slots = 0; rel_head = rel_tail = NULL, rel_bytes = 0; if (sinfo) memset(sinfo, 0, sizeof *sinfo << 20); // no slot is registered: tokens 0 unlock(); } // NX2NAVI_TRACE=perf "mem: sizes" (perf.c): the texture cache's entries (the bytes are in the "mem: vram" line), the // released ones, those retired by disuse, the sampler cache, the image requirements cache, the disuse holds, the // reuse keys, and the pool's texture and sampler slots. Counters, and a walk of at most 4096 sampler slots. void vk_texture_sizes(PerfSizes *s) { lock(); uint64_t imgs = by_key.n, rel = (uint64_t)nreleased, relb = rel_bytes, ret = (uint64_t)nretiring, retb = retiring_bytes, smp = samplers.n, reqs = img_reqs.n, hold = holds.n, reuse_keys = reuse.n, tslot = pool_slots, sslot = 0; for (int i = 0; i < nsslots; i++) sslot += sslots[i].used != 0; unlock(); perf_size(s, "textures", imgs, 0, PERF_SIZE_NONE); perf_size(s, "released textures", rel, 0, relb); perf_size(s, "retiring textures", ret, 0, retb); perf_size(s, "samplers", smp, 0, PERF_SIZE_NONE); perf_size(s, "image requirements", reqs, 0, PERF_SIZE_NONE); perf_size(s, "disuse holds", hold, HOLDS_MAX, PERF_SIZE_NONE); perf_size(s, "reuse keys", reuse_keys, 0, PERF_SIZE_NONE); perf_size(s, "texture pool slots", tslot, VK_POOL_TEX, PERF_SIZE_NONE); perf_size(s, "sampler slots", sslot, VK_POOL_SAMP, PERF_SIZE_NONE); }