Files
NX2navi/runtime/vk_texture.c
chrg6 63bcb441c0 Evict the least recently used idle textures when video memory runs low.
Above the VRAM pressure threshold (NX2NAVI_TEX_VRAM_PRESSURE, 90% of the budget) only released textures went before, so a scene whose live textures alone passed the budget kept growing into system memory. Once the released ones that come back intact are gone, live textures go too, least recently looked up first: only those whose content their storage still holds (CPU-written, never drawn into) and that no lookup reached for 240 submissions and 120 presents. The texture pool refresh looks every registered slot up each submission, so no live descriptor names them; the next use creates and uploads them again. Up to 64 per check, until the use is 2% under the threshold, and the next check waits for the GPU to pass the submissions that may still name the freed images. NX2NAVI_TEX_EVICT=0 turns it off.

Measured with the budget taken as 16 GiB (NX2NAVI_VRAM_BUDGET_MB=16384), same route, the exe at this commit and the next one: with the eviction off, 17,479 MiB in use in the world, of which 12,321 MiB textures, 37 MiB of them idle; with it on, 43 idle live textures (10 MiB) went in about 6,000 frames. In the world nearly every loaded texture is named by a descriptor, so the eviction frees memory mostly once an area or a menu is left. That run predates the later fixes (the idle clock in pool walks, the free space inside blocks); an earlier run without them emptied VRAM, as the commit that stops the evictions from emptying it says.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 03:34:46 -05:00

2510 lines
130 KiB
C

// Vulkan textures of the graphics layer: a VkImage cache keyed by storage
// GPU address plus descriptor, uploads through staging buffers with block-linear deswizzle and ASTC decode (vk_astc.c),
// re-uploads when page write generations (vk_memory.c) show the storage written, ReadTexels / WriteTexels, the texture
// and sampler descriptor slots, and VkSamplers.
//
// A texture that mirrors its pool memory (tex_mirrored) stops re-uploading once the GPU rendered into it (the
// "GPU-owned" rule); a slot of a finalized texture follows the live texture at its storage address.
// Contains code Copyright (c) 2026 jhaizhou, Copyright (c) 2026 Peiikai (MIT; see THIRD_PARTY_NOTICES.md).
//
// Every image lives in VK_IMAGE_LAYOUT_GENERAL. Uploads are recorded into the upload batch of vk_memory.c; the replay
// submits it (vk_upload_flush) before a submission that samples the textures.
#include <math.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cfg.h"
#include "gfx.h"
#include "host.h"
#include "nvn_api.h"
#include "nvn_hmap.h"
#include "nvn_tex.h"
#include "tex.h"
#include "vk.h"
#include "vk_dlaa.h"
#include "vk_exec.h"
#include "platform/platform.h"
typedef struct {
uint64_t addr;
nvn_texdesc d;
uint8_t has_layout, layout[8], pad[7];
} TexKey;
typedef struct Tex {
VkTexImage pub; // first: vk_texture_get hands out &pub
TexKey key;
tex_t t; // the newest texture object seen with this key
TexVkFormat f;
VmaAllocation alloc; // own memory; NULL when bound into pool memory
uint64_t bytes; // the size of alloc (0 without own memory), for the released entries' total
VmaAllocation bound_to; // the pool memory it is bound into
uint32_t wgen; // page generation of the last upload check
uint32_t checked; // submit_epoch of the last check from a draw binding
uint32_t use_ep; // submit_epoch when a lookup last reached it (touch)
uint64_t use_frame; // gfx_counters.frames then
int uploaded, released;
uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve)
uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale)
uint32_t n_up; // uploads of this entry (vk_texture_debug)
int last_ch; // last vk_mem_changed result of make_fresh: 1 written, 0 clean, -1 not tracked
struct Tex *next_at; // other entries at the same storage address
uint8_t gpu_dirty; // mirrored, and the GPU wrote the image after the last write-back to the storage
uint32_t gpu_gen; // page generation when the GPU write that set gpu_dirty was recorded
uint64_t gpu_seq; // order of that GPU write among all of them (gpu_writes)
uint64_t gpu_fence; // fence of the submission that made that write (0: not recorded in one)
uint64_t run_frame; // mirrored: the frame of the newest GPU write
uint8_t run; // frames in a row up to run_frame with a GPU write, counted up to WB_RUN
uint64_t wr_fence; // any entry: newest submission recorded to write its image (render, copy, clear,
// storage binding); ReadTexels waits for it instead of draining the replay queue
uint8_t listed; // in owned[]
uint8_t *snap; // a texture that fills no page: its storage bytes when the GPU write was recorded
uint32_t wt_n; // WriteTexels into the storage since the last upload
uint64_t chash; // content_hash of the storage at its last upload (small textures, CHASH_MAX)
uint8_t chash_ok; // chash holds it
struct Tex *rel_prev, *rel_next; // while released: the list of released entries, oldest first
} Tex;
// A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the
// entry (ce), valid while the generation of its address (agen) is still cgen, the view and has_view handed out with it,
// and the submit_epoch of the last freshness check through the slot (cepoch). Hot fields first: a lookup that hits
// touches one or two cache lines.
typedef struct {
struct Tex *ce; // NULL: nothing cached (registered since, or never resolved)
uint32_t cgen; // agen of the slot's address when ce was resolved
uint32_t cepoch;
view_t cview;
uint8_t chas_view, used, has_view;
uint64_t khash; // hash of the descriptor key and view, 0 for CPU-written textures (vk_texture_slot_khash_locked)
view_t v;
tex_t t;
} TexSlot;
typedef struct { samp_par p; VkSampler s, sp, sc; uint8_t used; } SampSlot; // s: vk_sampler_for(&p) once looked up; sp:
// its point-sampled form (vk_sampler_point_by_id_locked),
// sc its comparing form (vk_sampler_compare_by_id_locked)
static PlatRMutex tex_lock;
static PlatOnce tex_once = PLAT_ONCE_INIT;
static nvn_hmap by_key = {.klen = sizeof(TexKey)};
static nvn_hmap by_addr = {.klen = 8};
static TexSlot *tslots;
static int ntslots;
static SampSlot *sslots;
static int nsslots;
static nvn_hmap samplers = {.klen = sizeof(samp_par)};
static int nreleased;
static Tex *rel_head, *rel_tail; // released entries in release order (sweep_released drops from the head)
static uint64_t n_uploads, n_reuploads;
// Re-uploads since the last perf line (tex_lock): all, and those of textures a WriteTexels wrote since their last upload.
static uint64_t ru_n, ru_bytes, ru_ns, ru_wt_n, ru_wt_bytes, ru_wt_ns;
static uint64_t ru_same_n, ru_same_bytes; // re-uploads skipped: the pages were written, the texture's bytes were not
// NX2NAVI_TEX_ALIAS (default on; =0: a new image over live storage starts empty): a new descriptor over the storage of a
// live image that the GPU wrote gets that image's content (alias_copy). Counts since the last log line.
static int tex_alias = -1;
static uint64_t n_alias_copied, n_alias_upload, n_alias_uninit;
static uint32_t submit_epoch = 1; // bumped per queue submit: draw bindings check a texture's storage once per submit
// Mirrored textures the GPU wrote into, sorted by address (vk_texture_cpu_stale, vk_texture_write_back). Changed under
// tex_lock and owned_lock; read under either.
static PlatMutex owned_lock = PLAT_MUTEX_INIT;
static Tex **owned;
static int nowned, cowned;
static uint64_t max_owned; // largest storage size in owned[]
static uint64_t gpu_writes; // GPU writes into mirrored textures so far: Tex.gpu_seq
// Bumped whenever the entry a slot resolves to can change: an entry created or destroyed, released or taken back into
// use (the slot lookup and the follow of a finalized slot read only by_key, by_addr and the released flags).
static uint32_t tex_gen = 1;
// Bumped with tex_gen and whenever a slot is registered: what vk_texture_by_id returns for any slot may have changed.
static uint32_t slots_gen = 1;
static uint32_t samp_gen = 1; // bumped whenever a sampler slot is registered: vk_sampler_by_id may answer otherwise
static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) since the last perf line
// tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so
// a slot's resolution stays valid while its address's generation does. NX2navi finalizes and initializes transient
// textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws,
// which emptied every cache keyed on the global generations. 65536 buckets, the most the 16-bit index of sinfo,
// slot_bucket and the dirty ring's bucket entries holds: with 4096, about 8 slots shared each bucket, and every
// transient texture made or released at one address made the texture pool refresh (vk_exec_draw.c pool_sync_entries)
// refill its neighbours too, most of its 3-3.7 ms per frame in Kokiri Forest.
#define AGEN_N 65536
static uint32_t agen[AGEN_N];
// Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered)
// and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are
// 20 bits), unlike tslots, which moves when it grows.
static uint64_t *sinfo;
static uint32_t reg_serial;
static uint64_t n_same_regs; // vk_texture_register calls that changed nothing (vk_texture_same_regs)
static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 48); }
// Slots that may resolve differently since a position, for the texture pool descriptors (vk_exec_draw.c
// pool_sync_entries), which otherwise looked at every registered slot after each change: Ocarina registers slots all
// through a submission, and the refresh walked the whole table about twenty times per submission (3.5 ms). A ring of
// changes, each a slot id (registered) or an address bucket (an entry at that agen index changed: every slot registered
// over it, kept in per-bucket lists). Lock held for all of it.
#define DIRTY_N (1u << 16)
#define DIRTY_BUCKET 0x80000000u
static uint32_t dirty[DIRTY_N];
static uint64_t ndirty;
static int32_t bucket_head[AGEN_N];
static int32_t *slot_next, *slot_prev; // per slot id: its bucket list links (slot_prev -2: in no list)
static uint16_t *slot_bucket;
static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; }
static uint64_t n_bumps, n_site[4]; // tex_gen_bump calls, by site: destroy, create, take-back, release // tex_gen_bump calls, take-backs that undid a release's bump (vk_texture_bucket_stats)
static uint32_t bump_count[AGEN_N];
static void tex_gen_bump(uint64_t addr) {
n_bumps++;
bump_count[agen_idx(addr)]++;
if (!++tex_gen) tex_gen = 1;
slots_gen++;
uint32_t b = agen_idx(addr);
uint32_t *g = &agen[b];
__atomic_store_n(g, *g + 1, __ATOMIC_RELEASE);
dirty_push(DIRTY_BUCKET | b);
}
// Moves slot id into bucket b's list (the lists exist once a slot was registered: vk_texture_register).
static void slot_rebucket(int id, uint32_t b) {
if (slot_prev[id] != -2) { // out of its old list
if (slot_prev[id] >= 0) slot_next[slot_prev[id]] = slot_next[id];
else bucket_head[slot_bucket[id]] = slot_next[id];
if (slot_next[id] >= 0) slot_prev[slot_next[id]] = slot_prev[id];
}
slot_bucket[id] = (uint16_t)b;
slot_prev[id] = -1;
slot_next[id] = bucket_head[b];
if (bucket_head[b] >= 0) slot_prev[bucket_head[b]] = id;
bucket_head[b] = id;
}
uint64_t vk_texture_dirty_pos_locked(void) { return ndirty; }
int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg) {
if (!slot_next || ndirty - *pos > DIRTY_N) {
*pos = ndirty;
return 0;
}
for (uint64_t q = *pos; q < ndirty; q++) {
uint32_t x = dirty[q % DIRTY_N];
if (x & DIRTY_BUCKET)
for (int32_t id = bucket_head[x & ~DIRTY_BUCKET]; id >= 0; id = slot_next[id]) fn(id, arg);
else fn((int)x, arg);
}
*pos = ndirty;
return 1;
}
static volatile int32_t tex_ready;
static void tex_init(void *p) {
(void)p;
__atomic_store_n(&tex_ready, 1, __ATOMIC_RELEASE);
return;
}
static void lock(void) {
if (!__atomic_load_n(&tex_ready, __ATOMIC_ACQUIRE)) plat_once(&tex_once, tex_init, NULL);
plat_rmutex_lock(&tex_lock);
}
static void unlock(void) { plat_rmutex_unlock(&tex_lock); }
static uint32_t umax(uint32_t a, uint32_t b) { return a > b ? a : b; }
static uint32_t mip(uint32_t v, unsigned lv) { return umax(v >> lv, 1); }
static void make_key(const tex_t *t, TexKey *k) {
memset(k, 0, sizeof *k);
k->addr = (uintptr_t)t->mem;
k->d = t->d;
k->has_layout = t->has_layout;
memcpy(k->layout, t->layout, 8);
}
// Keys equal apart from the descriptor's swizzle selectors: the same storage, format and geometry.
static int same_but_swizzle(const TexKey *a, const TexKey *b) {
TexKey x = *a, y = *b;
x.d.swizzle_r = x.d.swizzle_g = x.d.swizzle_b = x.d.swizzle_a = 0;
y.d.swizzle_r = y.d.swizzle_g = y.d.swizzle_b = y.d.swizzle_a = 0;
return !memcmp(&x, &y, sizeof x);
}
// Textures whose content the CPU provides in pool memory. Display, compressible, zcull,
// depth, multisample and GPU-only storage stays GPU-owned.
static int tex_mirrored(const tex_t *t) {
if (!t->mem || nvn_gpu_only((uintptr_t)t->mem)) return 0;
if (t->has_layout) return 1;
if (!(t->pool_flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) return 0;
if (t->d.flags & (NVN_TEXTURE_FLAGS_DISPLAY | NVN_TEXTURE_FLAGS_COMPRESSIBLE | NVN_TEXTURE_FLAGS_ADAPTIVE_ZCULL |
NVN_TEXTURE_FLAGS_DEPTH16_PREFER_FAST_CLEAR | NVN_TEXTURE_FLAGS_ZCULL_SUPPORT_STENCIL))
return 0;
if (t->d.samples > 1 || t->d.target == NVN_TEXTURE_TARGET_BUFFER) return 0;
if (t->d.format >= 0x31 && t->d.format <= 0x36) return 0; // stencil / depth formats
return 1;
}
static void log_limited(volatile int32_t *n, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
static void log_limited(volatile int32_t *n, const char *fmt, ...) {
if (plat_inc32(n) > 16) return;
char buf[512];
va_list ap;
va_start(ap, fmt);
vsnprintf(buf, sizeof buf, fmt, ap);
va_end(ap);
rt_log("vk: %s", buf);
}
static void barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessFlags2 sa, VkPipelineStageFlags2 dst,
VkAccessFlags2 da) {
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = src, .srcAccessMask = sa,
.dstStageMask = dst, .dstAccessMask = da};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(cb, &di);
}
#define ALL_RW VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT
// ---------------------------------------------------------------- images
static VkFormatFeatureFlags features_of(VkFormat f) {
VkFormatProperties p;
vkGetPhysicalDeviceFormatProperties(vk_ctx()->phys, f, &p);
return p.optimalTilingFeatures;
}
static VkImageAspectFlags aspect_of(const TexVkFormat *f) {
VkImageAspectFlags a = 0;
if (f->depth) a |= VK_IMAGE_ASPECT_DEPTH_BIT;
if (f->stencil) a |= VK_IMAGE_ASPECT_STENCIL_BIT;
return a ? a : VK_IMAGE_ASPECT_COLOR_BIT;
}
// Binds img into the pool memory behind its GPU address (physical pools, virtual pools through the page table) when
// the memory type, alignment and contiguous size allow; 0 otherwise.
// A depth / stencil image that needs more bytes than its NVN storage (guest_size) is not bound: AMD's RDNA2 driver
// (26.10.1) puts the depth compression metadata after the depth data, and above 1080p that tail lies over the next
// texture of the game's heap; each depth write then scrambled it (garbage band at the top, pink glow, blocks in the
// grass, a blue checkered scene in the menu). Colour images keep their place: the RX 9070 also sizes some of them
// above their storage, and nothing there showed that the GPU writes the excess.
static int format_compressed(VkFormat f) { // vkCmdClearColorImage takes no block-compressed format
return (f >= VK_FORMAT_BC1_RGB_UNORM_BLOCK && f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK) ||
(f >= VK_FORMAT_ASTC_4x4_SFLOAT_BLOCK && f <= VK_FORMAT_ASTC_12x12_SFLOAT_BLOCK);
}
static int bind_pool_memory(VkCtx *c, Tex *e, VkImage img, uint64_t guest_size) {
VkMemoryRequirements req;
vkGetImageMemoryRequirements(c->device, img, &req);
if ((e->f.depth || e->f.stencil) && req.size > guest_size) return 0;
VkMemSpan sp;
if (!req.size) return 0; // an image the device could not lay out (image_size)
if (!vk_mem_resolve(e->pub.addr, req.size, &sp) || !sp.alloc || sp.alloc_offset % req.alignment) return 0;
VmaAllocationInfo info;
vmaGetAllocationInfo(c->vma, sp.alloc, &info);
if (!(req.memoryTypeBits >> info.memoryType & 1)) return 0;
if (vmaBindImageMemory2(c->vma, sp.alloc, sp.alloc_offset, img, NULL) != VK_SUCCESS) return 0;
e->bound_to = sp.alloc;
return 1;
}
// agl's BC4 encoder draws a BC texture's blocks through an RGBA16UI view, one texel per 8-byte block.
// Vulkan does it with an uncompressed
// view of a BLOCK_TEXEL_VIEW_COMPATIBLE image; EXTENDED_USAGE lets the BC image carry the COLOR_ATTACHMENT usage that
// only such views support. Whether the device takes that combination for this format and shape:
static int block_views_ok(const VkImageCreateInfo *ci) {
VkPhysicalDeviceImageFormatInfo2 fi = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
.format = ci->format, .type = ci->imageType, .tiling = ci->tiling,
.usage = ci->usage, .flags = ci->flags};
VkImageFormatProperties2 p = {.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2};
if (vkGetPhysicalDeviceImageFormatProperties2(vk_ctx()->phys, &fi, &p) != VK_SUCCESS) return 0;
const VkImageFormatProperties *q = &p.imageFormatProperties;
return ci->extent.width <= q->maxExtent.width && ci->extent.height <= q->maxExtent.height &&
ci->extent.depth <= q->maxExtent.depth && ci->mipLevels <= q->maxMipLevels &&
ci->arrayLayers <= q->maxArrayLayers;
}
// The memory the device wants for an image made from ci: 0 when it cannot lay it out. RADV on GFX9 (Vega, Renoir)
// hands back a 0-byte image for a mipmapped block-compressed 3D image at least as deep as it is wide in blocks, with
// COLOR_ATTACHMENT usage and no STORAGE: that usage asks for a thin 3D layout (Mesa radv_image.c), for which the GFX9
// address library has no swizzle mode left (gfx9addrlib.cpp), and the error is dropped; VMA then refuses the 0-byte
// allocation. The format properties call it supported, so only the size shows it.
static VkDeviceSize image_size(const VkImageCreateInfo *ci) {
VkDeviceImageMemoryRequirements q = {.sType = VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS, .pCreateInfo = ci};
VkMemoryRequirements2 r = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2};
vkGetDeviceImageMemoryRequirements(vk_ctx()->device, &q, &r);
return r.memoryRequirements.size;
}
// UNDEFINED -> GENERAL; new depth / stencil images start cleared (LAS textures.c: the game copies depth
// without stencil, and the stencil of those images must not be whatever the memory held).
static void init_image(Tex *e, VkCommandBuffer cb) {
VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_NONE,
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = e->pub.image, .subresourceRange = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib};
vkCmdPipelineBarrier2(cb, &di);
if (e->f.depth || e->f.stencil) {
VkClearDepthStencilValue zero = {0.0f, 0};
VkImageSubresourceRange rg = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
vkCmdClearDepthStencilImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &rg);
barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
} else if (vk_fix_zero() && !e->bound_to && !e->pub.mirrored && !format_compressed(e->f.vk)) {
// GPU-only storage that got memory of its own starts at zero, as the pool it stands for (vk_memory.c
// alloc_physical). Images aliasing pool memory are not cleared: that would erase what other textures rendered
// there; mirrored images are uploaded from their storage on first use.
VkClearColorValue zero = {0};
VkImageSubresourceRange rg = {e->pub.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
vkCmdClearColorImage(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &rg);
barrier(cb, VK_PIPELINE_STAGE_2_CLEAR_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
}
}
// NX2NAVI_SHADOW_SCALE (1 to 4): the game's shadow maps are made this many times larger on each side. NX2navi renders its
// sun cascades into a 960x960 DEPTH16 2D array of 4 layers and its static shadows into 1440x1440 and 720x720 arrays of 2,
// whatever the output resolution, and shrinks the cascades under load (872, 848, 800, 480 and so on). Those are made
// larger here: render passes into them scale the game's viewports and scissors (vk_exec.c resolve_rt, pass_scale),
// copies scale their regions (vk_exec_xfer.c sub_resolve), and shaders sample them with normalised coordinates, so the
// game sees nothing of it. A map whose storage the CPU writes (mirrored) or that aliases pool memory keeps its size,
// as its texels must match the guest layout.
static int shadow_scale(void) {
static int scale = -1;
if (scale < 0) {
int s = cfg_int("NX2NAVI_SHADOW_SCALE");
s = s < 1 ? 1 : s > 4 ? 4 : s;
rt_log("vk: shadow map scale %d", s);
scale = s;
}
return scale;
}
static int shadow_scale_of(const nvn_texdesc *d, const TexVkFormat *f, uint32_t samples, int mirrored) {
int scale = shadow_scale();
if (scale == 1 || mirrored || samples > 1 || d->format != NVN_FORMAT_DEPTH16 || !f->depth) return 1;
if (d->target != NVN_TEXTURE_TARGET_2D_ARRAY || d->width != d->height || d->width < 256 || d->depth < 2 || d->depth > 4)
return 1;
uint32_t most = vk_ctx()->props.limits.maxImageDimension2D;
return d->width * (uint32_t)scale <= most ? scale : 1;
}
// defer_init: the caller records the image's first transition (init_image or alias_copy).
static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) {
static volatile int32_t n_unmapped, n_unsupported, n_failed, n_unbound, n_thick;
VkCtx *c = vk_ctx();
const nvn_texdesc *d = &t->d;
if (!c || d->target == NVN_TEXTURE_TARGET_BUFFER) return NULL;
TexVkFormat f;
if (!tex_vk_format(d->format, &f)) {
log_limited(&n_unmapped, "texture format %#x has no Vulkan format: %s", d->format, f.reason);
return NULL;
}
if (!tex_vk_format_supported(&f)) {
log_limited(&n_unsupported, "texture format %#x: VkFormat %d lacks features %#x", d->format, (int)f.vk, (unsigned)f.need);
return NULL;
}
nvn_texlayout L;
nvn_tex_layout(d, &L);
uint32_t w = umax(d->width, 1), h = umax(d->height, 1), depth = 1, layers = 1;
VkImageType type = VK_IMAGE_TYPE_2D;
switch (d->target) {
case NVN_TEXTURE_TARGET_1D: type = VK_IMAGE_TYPE_1D; h = 1; break;
case NVN_TEXTURE_TARGET_1D_ARRAY: type = VK_IMAGE_TYPE_1D; h = 1; layers = L.layers; break;
case NVN_TEXTURE_TARGET_3D: type = VK_IMAGE_TYPE_3D; depth = umax(d->depth, 1); break;
case NVN_TEXTURE_TARGET_2D_ARRAY: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY:
case NVN_TEXTURE_TARGET_CUBEMAP: case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: layers = L.layers; break;
default: break;
}
int scale = shadow_scale_of(d, &f, d->samples > 1 ? (uint32_t)d->samples : 1, tex_mirrored(t));
w *= (uint32_t)scale, h *= (uint32_t)scale;
uint32_t levels = umax(d->levels, 1), most = 1;
for (uint32_t s = umax(umax(w, h), depth); s > 1; s >>= 1) most++;
if (levels > most) levels = most;
uint32_t samples = d->samples > 1 ? (uint32_t)d->samples : 1;
if (samples > 1) levels = 1;
VkFormatFeatureFlags ff = features_of(f.vk);
VkImageUsageFlags usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
int ds = f.depth || f.stencil;
if (ds && (ff & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
if (!ds && (ff & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
if ((d->flags & NVN_TEXTURE_FLAGS_IMAGE) && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1)
usage |= VK_IMAGE_USAGE_STORAGE_BIT;
// NVN2 titles (API 200+) number the texture flags differently: Ocarina's compute targets (the title movie's RGBA8
// frames) carry 0x1, NVN1's DISPLAY. Every single-sample colour texture whose format allows it can be a storage
// image there, so a compute write into one is never dropped.
if (rt_nvn_api_version[0] >= 200 && !ds && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1)
usage |= VK_IMAGE_USAGE_STORAGE_BIT;
// contact shadows (vk_contact.c) darken the game's B10G11R11 scene colour in place from a compute shader
if (f.vk == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) && samples == 1 &&
(ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && w >= 640 && vk_contact_strength())
usage |= VK_IMAGE_USAGE_STORAGE_BIT;
VkImageCreateFlags flags = ds ? 0 : VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; // format-reinterpreting views
// Cube compatible only for cube targets: the replay makes cube views of nothing else (vk_exec_draw.c get_view checks
// the image's own view type). The flag on every square array of 6+ layers changed how AMD's Windows driver lays such
// an image out, and terrain material arrays sampled through it came out in rainbow blocks that moved with the mip
// level (seen on an RX 580 and a 7800 XT).
if (type == VK_IMAGE_TYPE_2D && layers >= 6 && w == h && samples == 1 &&
(d->target == NVN_TEXTURE_TARGET_CUBEMAP || d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY))
flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
// NVN renders into a 3D texture slice by slice (a view's min layer is the depth plane, nvn_mtl.m), e.g. the
// grading LUT of the menu background blur. Vulkan needs a 2D view of the slice for that.
if (type == VK_IMAGE_TYPE_3D && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT;
VkImageCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .flags = flags, .imageType = type, .format = f.vk,
.extent = {w, h, depth}, .mipLevels = levels, .arrayLayers = layers,
.samples = (VkSampleCountFlagBits)samples, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = usage,
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED};
int bw, bh, block_views = 0;
// NX2NAVI_BLOCK_VIEWS_3D (default 0): a 3D BC image also gets block views. On RDNA2 cards (driver 26.10.1, issue #5)
// the 32x32x32 BC4 volume the game creates at the logo, made BLOCK_TEXEL_VIEW_COMPATIBLE and 2D_ARRAY_COMPATIBLE,
// took a GPU write just past the end of its memory: device lost at the first submit after it when the image ended
// its memory block. Draws into a 3D BC image go through a stand-in instead (vk_exec.c bc_target).
static int bv3d = -1;
if (bv3d < 0) {
bv3d = rt_knob("BLOCK_VIEWS_3D", "0");
rt_log("vk: 3D BC textures %s (NX2NAVI_BLOCK_VIEWS_3D=%d)", bv3d ? "get block views (renderable)" : "have no block views",
bv3d);
}
if (!ds && samples == 1 && f.vk == f.ideal && nvn_fmt_bpb(d->format, &bw, &bh) && bw == 4 && bh == 4 &&
(type != VK_IMAGE_TYPE_3D || bv3d)) {
VkImageCreateInfo bc = ci; // BC formats: renderable through views of their blocks (block_views_ok)
bc.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
bc.flags |= VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
if (type == VK_IMAGE_TYPE_3D) bc.flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT;
VkImageUsageFlags hidden = 0; // creation usage the replay never uses (views restrict theirs, vk_exec.c)
int ok = block_views_ok(&bc);
if (ok && type == VK_IMAGE_TYPE_3D && !image_size(&bc)) {
// STORAGE usage keeps RADV on GFX9 from the thin layout (image_size), so agl's BC4 encoder still renders
// into the volume; else the image is made without block views and its encoder passes are left out
bc.usage |= VK_IMAGE_USAGE_STORAGE_BIT, hidden = VK_IMAGE_USAGE_STORAGE_BIT;
ok = block_views_ok(&bc) && image_size(&bc);
log_limited(&n_thick, "texture fmt %#x %ux%ux%u: the device cannot lay out a 3D image with block views; %s",
d->format, w, h, depth, ok ? "made with storage usage (a thick layout)" : "made without them");
}
if (ok) ci = bc, usage = bc.usage & ~hidden, flags = bc.flags, block_views = 1;
}
// Feedback loops (vk.h VkCtx.feedback): every attachable image may be sampled by a draw of a pass that renders it,
// so it gets the usage now; adding it when a loop shows up would mean recreating the image (that lost the
// grass tile maps once).
int fb_usage = c->feedback == VK_FEEDBACK_LAYOUT && !block_views &&
(usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT));
if (fb_usage) ci.usage = usage |= VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
Tex *e = calloc(1, sizeof *e);
if (!e) rt_fatal("vk: out of memory for a texture");
e->key = *k;
e->t = *t;
e->f = f;
e->pub.addr = k->addr;
VkImage img = VK_NULL_HANDLE;
if (scale == 1 && nvn_gpu_only(k->addr)) { // storage in a physical or virtual pool: alias the pool memory (section 3.2)
if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS && !bind_pool_memory(c, e, img, L.size)) {
vkDestroyImage(c->device, img, NULL);
img = VK_NULL_HANDLE;
log_limited(&n_unbound, "texture %#llx (%ux%u fmt %#x) cannot alias its pool memory; own memory instead",
(unsigned long long)k->addr, w, h, d->format);
}
}
// A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks:
// the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load
// (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that
// 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom
// thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference.
enum { SUBALLOC_MAX = 32 << 20 };
if (!img && tex_mirrored(t) && samples == 1) {
if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS) {
// an image the driver requires (not only prefers) dedicated memory for keeps it: vmaCreateImage below
VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS};
VkMemoryRequirements2 mr2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr};
VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img};
vkGetImageMemoryRequirements2(c->device, &ri, &mr2);
VkMemoryRequirements mr = mr2.memoryRequirements;
VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT};
int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX &&
vmaAllocateMemory(c->vma, &mr, &ai, &e->alloc, NULL) == VK_SUCCESS;
if (ok && vmaBindImageMemory(c->vma, e->alloc, img) != VK_SUCCESS) {
vmaFreeMemory(c->vma, e->alloc);
ok = 0;
}
if (!ok) {
vkDestroyImage(c->device, img, NULL);
img = VK_NULL_HANDLE, e->alloc = NULL;
}
} else {
img = VK_NULL_HANDLE;
}
}
if (!img) {
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL);
if (r != VK_SUCCESS) {
log_limited(&n_failed, "image creation failed (%d): nvn fmt %#x %ux%ux%u target %d levels %u layers %u", (int)r,
d->format, w, h, depth, d->target, levels, layers);
free(e);
return NULL;
}
}
if (e->alloc) {
VmaAllocationInfo own;
vmaGetAllocationInfo(c->vma, e->alloc, &own);
e->bytes = own.size;
}
VkTexImage *p = &e->pub;
p->image = img;
p->format = f.vk;
p->type = type;
p->usage = usage;
p->aspect = aspect_of(&f);
p->width = w, p->height = h, p->depth = depth, p->levels = levels, p->layers = layers, p->samples = samples;
p->view_type = type == VK_IMAGE_TYPE_1D ? (d->target == NVN_TEXTURE_TARGET_1D_ARRAY ? VK_IMAGE_VIEW_TYPE_1D_ARRAY : VK_IMAGE_VIEW_TYPE_1D)
: type == VK_IMAGE_TYPE_3D ? VK_IMAGE_VIEW_TYPE_3D
: d->target == NVN_TEXTURE_TARGET_CUBEMAP && (flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) ? VK_IMAGE_VIEW_TYPE_CUBE
: d->target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY && (flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) ? VK_IMAGE_VIEW_TYPE_CUBE_ARRAY
: layers > 1 || d->target == NVN_TEXTURE_TARGET_2D_ARRAY || d->target == NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY
? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D;
p->alpha_one = f.alpha_one;
memcpy(p->chan, f.chan, sizeof p->chan);
p->swz[0] = (uint8_t)d->swizzle_r, p->swz[1] = (uint8_t)d->swizzle_g;
p->swz[2] = (uint8_t)d->swizzle_b, p->swz[3] = (uint8_t)d->swizzle_a;
if (!(p->swz[0] | p->swz[1] | p->swz[2] | p->swz[3])) // descriptors built without nvnTextureBuilderSetDefaults
p->swz[0] = NVN_TEXTURE_SWIZZLE_R, p->swz[1] = NVN_TEXTURE_SWIZZLE_G, p->swz[2] = NVN_TEXTURE_SWIZZLE_B,
p->swz[3] = NVN_TEXTURE_SWIZZLE_A;
p->mirrored = (uint8_t)tex_mirrored(t);
p->bound = e->bound_to != NULL;
p->block_views = (uint8_t)block_views;
p->fb_usage = (uint8_t)fb_usage;
p->scale = (uint8_t)scale;
if (!defer_init) {
init_image(e, vk_upload_begin());
vk_upload_end();
}
return e;
}
static int owned_lower(uint64_t a) {
int lo = 0, hi = nowned;
while (lo < hi) {
int m = (lo + hi) / 2;
if (owned[m]->pub.addr < a) lo = m + 1;
else hi = m;
}
return lo;
}
static void own_add(Tex *e) {
if (e->listed) return;
plat_mutex_lock(&owned_lock);
if (nowned == cowned) {
cowned = cowned ? 2 * cowned : 256;
if (!(owned = realloc(owned, sizeof *owned * (size_t)cowned))) rt_fatal("vk: out of memory for the owned-texture list");
}
int i = owned_lower(e->pub.addr);
memmove(owned + i + 1, owned + i, sizeof *owned * (size_t)(nowned - i));
owned[i] = e;
__atomic_store_n(&nowned, nowned + 1, __ATOMIC_RELEASE);
if (e->t.L.size > max_owned) max_owned = e->t.L.size;
e->listed = 1;
plat_mutex_unlock(&owned_lock);
}
static void own_remove(Tex *e) {
if (!e->listed) return;
plat_mutex_lock(&owned_lock);
for (int i = owned_lower(e->pub.addr); i < nowned && owned[i]->pub.addr == e->pub.addr; i++)
if (owned[i] == e) {
memmove(owned + i, owned + i + 1, sizeof *owned * (size_t)(nowned - i - 1));
__atomic_store_n(&nowned, nowned - 1, __ATOMIC_RELEASE);
break;
}
e->listed = 0;
plat_mutex_unlock(&owned_lock);
}
static uint64_t rel_bytes; // own memory of the released entries (Tex.bytes)
static void rel_append(Tex *e) {
e->rel_next = NULL;
e->rel_prev = rel_tail;
if (rel_tail) rel_tail->rel_next = e;
else rel_head = e;
rel_tail = e;
rel_bytes += e->bytes;
}
static void rel_unlink(Tex *e) {
rel_bytes -= e->bytes;
if (e->rel_prev) e->rel_prev->rel_next = e->rel_next;
else rel_head = e->rel_next;
if (e->rel_next) e->rel_next->rel_prev = e->rel_prev;
else rel_tail = e->rel_prev;
e->rel_prev = e->rel_next = NULL;
}
static void destroy_tex(Tex *e) {
own_remove(e);
n_site[0]++;
tex_gen_bump(e->key.addr);
nvn_hm_del(&by_key, &e->key);
Tex *head = nvn_hm_get(&by_addr, &e->key.addr);
if (head == e) {
if (e->next_at) nvn_hm_put(&by_addr, &e->key.addr, e->next_at);
else nvn_hm_del(&by_addr, &e->key.addr);
} else {
for (Tex *p = head; p; p = p->next_at)
if (p->next_at == e) { p->next_at = e->next_at; break; }
}
if (e->released) nreleased--, rel_unlink(e);
vk_exec_image_gone(e->pub.image);
vk_defer_image(e->pub.image, e->alloc);
free(e->snap);
free(e);
}
// Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries the
// oldest one goes at each release (it comes back on demand, a mirrored one re-uploaded from its storage). NX2navi
// finalizes and initializes transient textures every frame, so a released entry is mostly taken back within frames.
// Dropping all of them at once when the count passed 4096 destroyed thousands of images in one frame: 333-422 ms
// frames every 40-50 s on a GTX 1050 Ti, then the textures still in use were created and uploaded again.
#define RELEASED_MAX 2048
static void sweep_released(void) {
while (nreleased > RELEASED_MAX && rel_head) destroy_tex(rel_head);
}
// NX2NAVI_TEX_VRAM_PRESSURE (percent of the device-local budget, default 90; 0 = off): above it, each submit drops up to
// PRESSURE_STEP of the oldest released entries, so a 4 GiB card does not keep RELEASED_MAX released images (each ASTC
// one decoded to RGBA8) while the heap spills into system memory. A few per submit, for the reason above. Only entries
// that come back intact are dropped: mirrored ones the GPU never wrote, re-uploaded from their storage. A render target
// lives only in its image; dropping it brought the texture back empty (black flashes in a Depths boss fight, ~170
// "left uninitialised" aliases a minute).
#define PRESSURE_STEP 32
static int vram_pressure = -1;
static uint64_t n_pressure_drops;
static int comes_back_intact(const Tex *e) { return e->pub.mirrored && !e->pub.gpu_owned && !e->gpu_dirty; }
// When a lookup last reached an entry: entry_of (every texture get and slot resolution) and the per-submit check of a
// slot lookup (vk_texture_by_id_locked). The texture pool refresh looks every registered slot up once per submission
// (vk_exec_draw.c pool_sync_entries), so an entry no lookup reached for IDLE_SUBMITS submissions and STALE_FRAMES
// presents (below) has no live descriptor naming it: the game keeps the texture object, nothing samples it.
#define IDLE_SUBMITS 240
static void touch(Tex *e) {
e->use_ep = submit_epoch;
e->use_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
}
static int idle(const Tex *e, uint64_t now);
// NX2NAVI_TEX_RELEASED_MB (default a sixteenth of the installed memory, 1 to 4 GiB; 0 = no limit) bounds the own memory
// of the released entries: RELEASED_MAX alone kept 5-7 GiB of released render targets and decoded textures after an
// hour of play at a 2160p game resolution, device memory the driver also charges to the system's commit (the process's
// private bytes reached 31 GiB). NX2NAVI_TEX_COMMIT_FREE_MB (default the larger of 4 GiB and a tenth of the commit
// limit; 0 = off) lowers that bound to COMMIT_LOW_KEEP while less of the system's commit is free: an allocation that
// fails there ends the game. Both drop only stale entries (drop_stale), a few per check like the VRAM pressure.
#define COMMIT_LOW_KEEP ((uint64_t)256 << 20)
static uint64_t released_cap; // bytes; 0 = not read yet, UINT64_MAX = no limit
static long long commit_free_mb = -2; // -2: not read yet, -1: the default
static int commit_low, commit_logs;
static uint64_t n_cap_drops, n_commit_drops;
static uint64_t released_cap_bytes(void) {
if (!released_cap) {
const char *s = cfg_get("NX2NAVI_TEX_RELEASED_MB");
long long mb;
if (s && *s) mb = atoll(s);
else {
float used, total, proc;
plat_mem_info(&used, &total, &proc);
mb = total > 0 ? (long long)(total / 16) : 2048;
mb = mb < 1024 ? 1024 : mb > 4096 ? 4096 : mb;
}
released_cap = mb > 0 ? (uint64_t)mb << 20 : UINT64_MAX;
if (mb > 0) rt_log("vk: released textures keep at most %lld MiB of their own memory", mb);
else rt_log("vk: released textures: no size limit");
}
return released_cap;
}
static int commit_is_low(void) {
if (commit_free_mb == -2) {
const char *s = cfg_get("NX2NAVI_TEX_COMMIT_FREE_MB");
commit_free_mb = s && *s ? atoll(s) : -1;
if (commit_free_mb < -1) commit_free_mb = -1;
if (!plat_commit_enforced()) commit_free_mb = 0; // a limit no allocation fails at
}
if (!commit_free_mb) return 0;
double proc, used, limit;
plat_commit_info(&proc, &used, &limit);
if (used < 0 || limit <= 0) return 0;
double want = commit_free_mb > 0 ? (double)commit_free_mb : limit / 10 > 4096 ? limit / 10 : 4096;
int low = limit - used < want;
if (low != commit_low && commit_logs < 40) { // a few lines, however often it flips
commit_logs++;
rt_log("vk: system commit %s: %.0f of %.0f MB in use (this process %.0f MB); released textures kept %d, %.0f MiB "
"of own memory%s", low ? "low" : "no longer low", used, limit, proc, nreleased, rel_bytes / 1048576.0,
low ? ": dropping the stale ones" : "");
}
return commit_low = low;
}
// Drops up to PRESSURE_STEP released entries that own memory and were released STALE_FRAMES presents ago or earlier,
// oldest first and the intact ones before the others, while the released entries own more than keep. A transient
// texture is taken back within frames, so dropping it frees nothing: the game initializes it again and it is created
// again, empty if it is a render target. Under commit pressure that a system-managed page file never relieves, dropping
// young ones destroyed 2160p render targets about every frame for whole sessions. Frame generation also reads a frame's
// depth target at the host present (vk_dlssg.c vk_dlssg_frame tags it), after the blit deferred destruction waits for.
#define STALE_FRAMES 120
static int idle(const Tex *e, uint64_t now) {
return submit_epoch - e->use_ep >= IDLE_SUBMITS && now - e->use_frame >= STALE_FRAMES;
}
static void drop_stale(uint64_t keep, uint64_t *count) {
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
int dropped = 0;
for (int any = 0; any < 2; any++)
for (Tex *e = rel_head, *next; e && now - e->rel_frame >= STALE_FRAMES; e = next) { // in release order
if (dropped == PRESSURE_STEP || rel_bytes <= keep) return;
next = e->rel_next;
if (!e->bytes || (!any && !comes_back_intact(e))) continue;
destroy_tex(e);
dropped++, (*count)++;
}
}
// NX2NAVI_TEX_EVICT (default on; 0 = only released entries go under VRAM pressure, as before): above the VRAM pressure
// threshold, once the released entries that come back intact are gone, live entries go too, least recently looked up
// first: those that come back intact and that no lookup reached for IDLE_SUBMITS submissions and STALE_FRAMES presents
// (touch: no live descriptor names them). A later lookup creates the image again and uploads it from its storage, as
// for a texture seen the first time. Up to EVICT_STEP entries per check, until the device-local use is 2% of the budget
// under the threshold; the next check waits until the GPU is past the submissions that may still name the images, as
// their memory goes back to the driver only then (vk_defer_image) and the budget would show the old use.
#define EVICT_STEP 64
static int tex_evict = -1;
static uint64_t evict_wait; // vk_sync_fence_done value the next eviction waits for
static uint64_t n_evict, n_evict_bytes; // since the last perf line
static int evict_logs;
static void evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex_lock held
if (vk_sync_fence_done() < evict_wait) return;
Tex *v[EVICT_STEP];
int nv = 0;
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
for (uint32_t i = 0; i < by_key.cap; i++) { // the EVICT_STEP oldest candidates, oldest first
Tex *e = by_key.vals[i];
if (!e || e->released || !e->bytes || !comes_back_intact(e) || !idle(e, now)) continue;
uint32_t age = submit_epoch - e->use_ep;
int at = nv;
while (at > 0 && submit_epoch - v[at - 1]->use_ep < age) at--;
if (at == EVICT_STEP) continue;
if (nv < EVICT_STEP) nv++;
memmove(v + at + 1, v + at, sizeof *v * (size_t)(nv - 1 - at));
v[at] = e;
}
uint64_t freed = 0;
int n = 0;
for (; n < nv && freed < want; n++) freed += v[n]->bytes, destroy_tex(v[n]);
if (!n) return;
n_evict += (uint64_t)n, n_evict_bytes += freed;
evict_wait = vk_sync_issued() + 2;
if (evict_logs < 8 || ((rt_trace & RT_TRACE_PERF) && evict_logs < 64)) {
evict_logs++;
rt_log("vk: VRAM %.0f of %.0f MiB in use: %d idle textures evicted (%.1f MiB), least recently used first; they "
"are uploaded again when the game uses them", used / 1048576.0, budget / 1048576.0, n, freed / 1048576.0);
}
}
static void sweep_pressure(void) { // tex_lock held
if (vram_pressure < 0) {
const char *s = cfg_get("NX2NAVI_TEX_VRAM_PRESSURE");
vram_pressure = s && *s ? atoi(s) : 90;
if (vram_pressure < 0 || vram_pressure > 100) vram_pressure = 90;
tex_evict = rt_knob("TEX_EVICT", "1");
rt_log("vk: textures under VRAM pressure: %s%d%%%s", vram_pressure ? "dropped above " : "off", vram_pressure,
!vram_pressure ? "" : tex_evict ? ", released ones first, then idle live ones (NX2NAVI_TEX_EVICT=0: released only)"
: ", released ones only (NX2NAVI_TEX_EVICT=0)");
}
if (submit_epoch & 7) return; // the budget and commit queries every 8 submits
uint64_t used, budget;
int exact, dropped = 0;
if (vram_pressure && (rel_head || tex_evict) && vk_mem_vram(&used, &budget, &exact) &&
used > budget / 100 * (uint64_t)vram_pressure) {
uint64_t freed = 0, over = used - budget / 100 * (uint64_t)vram_pressure + budget / 50;
for (Tex *e = rel_head, *next; e && dropped < PRESSURE_STEP; e = next) {
next = e->rel_next;
if (!comes_back_intact(e)) continue;
freed += e->bytes;
destroy_tex(e);
dropped++, n_pressure_drops++;
}
if (tex_evict && freed < over) evict_idle(over - freed, used, budget);
}
if (!rel_head) return;
uint64_t cap = released_cap_bytes();
if (rel_bytes > cap) drop_stale(cap, &n_cap_drops);
if (rel_bytes > COMMIT_LOW_KEEP && commit_is_low()) drop_stale(COMMIT_LOW_KEEP, &n_commit_drops);
}
// ---------------------------------------------------------------- uploads
static uint32_t texel_alpha_one(int format, uint32_t *elem) {
switch (format) {
case 0x19: *elem = 1; return 0xFF;
case 0x1A: *elem = 1; return 0x7F;
case 0x1B: case 0x1C: *elem = 1; return 1;
case 0x1D: *elem = 2; return 0x3C00;
case 0x1E: *elem = 2; return 0xFFFF;
case 0x1F: *elem = 2; return 0x7FFF;
case 0x20: case 0x21: *elem = 2; return 1;
case 0x22: *elem = 4; return 0x3F800000;
default: *elem = 4; return 1; // 0x23, 0x24
}
}
// 3-component texels to 4 (n texels).
static void expand_rgb(int format, const uint8_t *src, uint8_t *dst, size_t n) {
uint32_t e, one = texel_alpha_one(format, &e);
for (size_t i = 0; i < n; i++, src += 3 * e, dst += 4 * e) {
memcpy(dst, src, 3 * e);
memcpy(dst + 3 * e, &one, e);
}
}
static void shrink_rgb(int format, const uint8_t *src, uint8_t *dst, size_t n) {
uint32_t e;
texel_alpha_one(format, &e);
for (size_t i = 0; i < n; i++, src += 4 * e, dst += 3 * e) memcpy(dst, src, 3 * e);
}
// NX2NAVI_ASTC=check: the compute decoder's texels of n regions (gpu: the regions one after the other) against
// astc_decode_ldr of the same raw blocks (raw: the staging bytes the regions point into).
static void astc_check(const Tex *e, const uint8_t *raw, const VkAstcRegion *ar, int n, const uint8_t *gpu) {
static volatile int32_t n_bad;
static uint64_t n_tex, n_texels, n_diff_tex;
uint32_t bw = e->f.astc_bw, bh = e->f.astc_bh;
uint64_t diff = 0, first = UINT64_MAX;
uint32_t want = 0, got = 0;
int first_k = 0;
for (int k = 0; k < n; k++) {
const VkAstcRegion *r = &ar[k];
size_t sb = (size_t)((r->w + bw - 1) / bw) * ((r->h + bh - 1) / bh) * 16, st = (size_t)r->w * r->h * 4;
uint8_t *cpu = malloc(st);
for (uint32_t z = 0; z < r->slices; z++, gpu += st) {
astc_decode_ldr(raw + r->src + z * sb, r->w, r->h, bw, bh, e->f.astc_srgb, cpu);
if (!memcmp(cpu, gpu, st)) continue;
for (size_t i = 0; i < st / 4; i++)
if (memcmp(cpu + 4 * i, gpu + 4 * i, 4)) {
if (first == UINT64_MAX) {
first = i, first_k = k;
memcpy(&want, cpu + 4 * i, 4), memcpy(&got, gpu + 4 * i, 4);
}
diff++;
}
}
free(cpu);
n_texels += (uint64_t)r->w * r->h * r->slices;
}
n_tex++;
if (diff) {
n_diff_tex++;
log_limited(&n_bad, "ASTC check: texture %#llx fmt %#x %ux%u: %llu texels differ, first region %d texel %llu "
"cpu %08x gpu %08x", (unsigned long long)e->pub.addr, e->t.d.format, bw, bh,
(unsigned long long)diff, first_k, (unsigned long long)first, want, got);
}
if (n_tex == 1 || !(n_tex & 63) || diff)
rt_log("vk: ASTC check: %llu textures, %llu texels compared, %llu textures differ", (unsigned long long)n_tex,
(unsigned long long)n_texels, (unsigned long long)n_diff_tex);
}
// Staging bytes of one part of a texture upload decoded or copied on the CPU (see upload). With NX2NAVI_ASTC=cpu a
// large ASTC array decodes to RGBA8 in one go: a staging buffer of 676682336 bytes failed (-2) on an Intel UHD with
// 8 GB of shared memory and the game closed before the title screen.
#define UPLOAD_PART (32ull << 20) // fits the staging pool (vk_staging takes up to half of its 64 MiB block)
// A texture of at most CHASH_MAX bytes keeps a hash of its storage as it was uploaded. A write to its pages then
// re-uploads it only when its own bytes changed: Ocarina's frames rewrite hundreds of small textures' pages (shared
// with data the CPU writes, or written with the same bytes), about 420 re-uploads a frame of 1.5 KiB each, and one
// costs a staging buffer, a copy and a barrier, several microseconds, where hashing 1.5 KiB takes a fraction of one.
// A CPU write during the hash sets the pages' write bits again, so the next check sees it.
enum { CHASH_MAX = 64 << 10 };
static uint64_t content_hash(const void *p, uint64_t n) {
const uint8_t *b = p;
uint64_t h = 0x9e3779b97f4a7c15ull ^ n, i = 0;
for (; i + 8 <= n; i += 8) {
uint64_t w;
memcpy(&w, b + i, 8);
h = (h ^ w) * 0xbf58476d1ce4e5b9ull;
h ^= h >> 29;
}
for (; i < n; i++) h = (h ^ b[i]) * 0x94d049bb133111ebull;
return h ^ (h >> 31);
}
// Copies the whole storage into the image (every level and layer the image has).
static void upload(Tex *e) {
static volatile int32_t n_bad, n_undecoded, n_split;
const tex_t *t = &e->t;
const nvn_texdesc *d = &t->d;
int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh);
if (!bpb || !t->mem) return;
// A stale slot (the bindless pools look every slot up): no upload; a later write to mapped storage uploads it
// (make_fresh_snap).
if (!mem_readable(e->pub.addr, e->t.L.size)) {
static volatile int32_t n_unmapped;
log_limited(&n_unmapped, "texture %#llx (%llu bytes): storage not mapped; not uploaded",
(unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size);
e->uploaded = 1;
return;
}
e->chash_ok = e->t.L.size <= CHASH_MAX; // the hash of what this upload copies (make_fresh_snap compares it)
if (e->chash_ok) e->chash = content_hash((const void *)(uintptr_t)e->pub.addr, e->t.L.size);
static __thread char ctx[160]; // what this thread copies, for a crash report
snprintf(ctx, sizeof ctx, "texture upload %#llx (%llu bytes, fmt %#x %ux%u)", (unsigned long long)e->pub.addr,
(unsigned long long)e->t.L.size, d->format, d->width, d->height);
uint64_t p0 = perf_begin(), u0 = kern_now_ns();
int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D;
uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers;
int astc = e->f.astc_bw != 0, rgb = e->f.expand_rgb;
int agpu = astc && vk_astc_gpu(), acheck = agpu && vk_astc_mode() == VK_ASTC_CHECK; // raw blocks to the GPU
uint32_t n = levels * layers, k = 0;
VkBufferImageCopy *rg = calloc(n, sizeof *rg);
VkAstcRegion *ar = agpu ? calloc(n, sizeof *ar) : NULL;
VkDeviceSize total = 0, decoded = 0;
for (uint32_t lv = 0; lv < levels; lv++)
for (uint32_t ly = 0; ly < layers; ly++) {
uint32_t w = mip(d->width, lv), h = one_d ? 1 : mip(d->height, lv), sl = is3d ? mip(d->depth, lv) : 1;
uint64_t texels = (uint64_t)w * h * sl, raw = (uint64_t)(w + bw - 1) / bw * bpb * ((h + bh - 1) / bh) * sl;
uint64_t out = agpu ? raw : astc ? texels * 4 : rgb ? texels * (uint64_t)bpb / 3 * 4 : raw;
rg[k++] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {e->pub.aspect, lv, ly, 1},
.imageExtent = {w, h, sl}};
total += (out + 15) & ~15ull;
decoded += texels * 4;
}
VkBuffer sb, rbuf = VK_NULL_HANDLE;
VmaAllocation ralloc = NULL;
uint8_t *rmap = NULL, *raw_copy = NULL;
int checked = 0, parts = 1;
// A CPU decode or copy stages at most UPLOAD_PART bytes per staging buffer (a region larger than that alone) and
// records one copy per buffer; every part stays in the same upload batch, submitted as before. The GPU decoder
// takes the raw blocks of every region from one buffer.
for (uint32_t k0 = 0, k1; k0 < n; k0 = k1) {
VkDeviceSize base = rg[k0].bufferOffset, size;
for (k1 = k0 + 1; k1 < n && (agpu || (k1 + 1 < n ? rg[k1 + 1].bufferOffset : total) - base <= UPLOAD_PART);) k1++;
size = (k1 < n ? rg[k1].bufferOffset : total) - base;
if (k0) parts++;
VkCommandBuffer cb = vk_upload_begin();
uint8_t *stg = vk_staging(size, &sb, NULL);
if (acheck) rmap = vk_staging(decoded, &rbuf, &ralloc);
// The part's copies out of the storage, with it held mapped (mem_read_hold): the game releases a texture's
// memory on its own thread, and a decommit in the middle of a copy faulted (tex_bl_copy). Only the copies run
// held, so a waiting unmap and the guest's memory calls behind it never wait for the staging block or a decode.
// Storage released since the check above uploads as zeros, as the game no longer has that texture.
uint8_t **tmps = calloc(k1 - k0, sizeof *tmps);
int held = mem_read_hold(e->pub.addr, e->t.L.size);
if (held) plat_crash_context(ctx);
else {
static volatile int32_t n_gone;
log_limited(&n_gone, "texture %#llx (%llu bytes): storage released during the upload; uploaded as zeros",
(unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size);
}
for (k = k0; k < k1; k++) {
uint32_t lv = rg[k].imageSubresource.mipLevel, ly = rg[k].imageSubresource.baseArrayLayer;
uint32_t w = rg[k].imageExtent.width, h = rg[k].imageExtent.height, sl = rg[k].imageExtent.depth;
uint32_t rb = (w + bw - 1) / bw * (uint32_t)bpb, rows = (h + bh - 1) / bh;
size_t raw = (size_t)rb * rows * sl;
rg[k].bufferOffset -= base;
uint8_t *dst = stg + rg[k].bufferOffset, *tmp = (astc && !agpu) || rgb ? malloc(raw) : NULL;
tmps[k - k0] = tmp;
if (!held) {
memset(tmp ? tmp : dst, 0, raw);
} else if (nvn_deswizzle(d, lv, ly, t->mem, tmp ? tmp : dst, rb)) {
log_limited(&n_bad, "texture %#llx fmt %#x target %d: level %u layer %u has no CPU layout; uploaded as zeros",
(unsigned long long)e->pub.addr, d->format, d->target, lv, ly);
memset(tmp ? tmp : dst, 0, raw);
}
}
if (held) {
plat_crash_context(NULL);
mem_read_release(); // the part's reads of the storage are done
}
for (k = k0; k < k1; k++) { // the decodes, from the copies
uint32_t lv = rg[k].imageSubresource.mipLevel, ly = rg[k].imageSubresource.baseArrayLayer;
uint32_t w = rg[k].imageExtent.width, h = rg[k].imageExtent.height, sl = rg[k].imageExtent.depth;
uint32_t rb = (w + bw - 1) / bw * (uint32_t)bpb, rows = (h + bh - 1) / bh;
uint8_t *dst = stg + rg[k].bufferOffset, *tmp = tmps[k - k0];
if (agpu) {
ar[k] = (VkAstcRegion){.src = rg[k].bufferOffset, .w = w, .h = h, .slices = sl, .level = lv, .layer = ly};
} else if (astc) {
uint64_t a0 = perf_begin();
for (uint32_t z = 0; z < sl; z++)
astc_decode_ldr_mt(tmp + (size_t)z * rb * rows, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb,
dst + (size_t)z * w * h * 4);
perf_end(PERF_ASTC, a0);
rg[k].bufferRowLength = w, rg[k].bufferImageHeight = h;
} else if (rgb) {
expand_rgb(d->format, tmp, dst, (size_t)w * h * sl);
rg[k].bufferRowLength = w, rg[k].bufferImageHeight = h;
} else {
rg[k].bufferRowLength = rb / (uint32_t)bpb * (uint32_t)bw, rg[k].bufferImageHeight = rows * (uint32_t)bh;
}
free(tmp);
}
free(tmps);
if (acheck) {
raw_copy = malloc(size);
memcpy(raw_copy, stg, size);
}
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
if (agpu) {
uint64_t a0 = perf_begin();
checked = vk_astc_record(cb, sb, size, ar, (int)n, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc,
e->pub.image, e->pub.aspect, rbuf);
perf_end(PERF_ASTC, a0);
if (!checked)
log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was",
(unsigned long long)e->pub.addr, d->format, d->width, d->height);
} else {
vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, k1 - k0, rg + k0);
}
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
vk_upload_end();
}
if (parts > 1)
log_limited(&n_split, "texture %#llx fmt %#x %ux%u, %u levels, %u layers: %.1f MiB staged in %d parts",
(unsigned long long)e->pub.addr, d->format, d->width, d->height, levels, layers, total / 1048576.0,
parts);
if (acheck) { // waits for the GPU
vk_upload_flush(1);
if (checked) {
vmaInvalidateAllocation(vk_ctx()->vma, ralloc, 0, VK_WHOLE_SIZE);
astc_check(e, raw_copy, ar, (int)n, rmap);
}
vmaDestroyBuffer(vk_ctx()->vma, rbuf, ralloc);
}
free(raw_copy);
free(ar);
free(rg);
uint64_t dt = kern_now_ns() - u0;
if (e->uploaded) {
n_reuploads++, perf_count(PERF_CNT_TEX_REUP);
ru_n++, ru_bytes += t->L.size, ru_ns += dt;
if (e->wt_n) {
ru_wt_n++, ru_wt_bytes += t->L.size, ru_wt_ns += dt;
if (rt_trace & RT_TRACE_PERF)
rt_log("perf: frame %llu: re-upload after %u WriteTexels: texture %#llx fmt %#x %ux%u, %.1f KiB, %.3f ms",
(unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), e->wt_n,
(unsigned long long)e->pub.addr, d->format, d->width, d->height, t->L.size / 1024.0, dt / 1e6);
}
} else {
n_uploads++, perf_count(PERF_CNT_TEX_UP);
}
e->wt_n = 0;
e->uploaded = 1;
e->n_up++;
perf_end(PERF_TEX, p0);
}
// Whether e's storage fills no 4 KiB page of its own, so the write tracking can only see writes to pages it shares.
static int fills_no_page(const Tex *e) {
return ((e->pub.addr + e->t.L.size) & ~0xfffull) <= ((e->pub.addr + 0xfff) & ~0xfffull);
}
// A texture that fills no page keeps a copy of its storage bytes from when a GPU write to it was recorded
// (vk_texture_gpu_written), up to SNAP_MAX bytes; wb_exact (NX2NAVI_WB_EXACT, default on) enables it.
enum { SNAP_MAX = 4096 };
static int wb_exact = -1;
static volatile int32_t n_kept;
// vk_mem_changed over the pages wholly inside e's storage (all of its pages when it fills none): a neighbour that shares
// the first or last page does not count as a write to e. A texture that fills no page shares all of its pages, so
// there a page write counts only when e's own bytes differ from the copy taken at the GPU write. NX2navi's Purah Pad
// camera renders the ID, screen rect and depth of what is under the reticle into 4x4 targets of 512 bytes, two
// slots used in turn, and the CPU reads them a frame later. One slot's rect target shares its page with data the CPU
// writes every frame: without the copy its renders were never written back, the game read a zero rect every other
// frame and the target's name label blinked. NX2NAVI_WB_EXACT=0 restores the page-only decision.
static int storage_rewritten(const Tex *e, uint32_t since, uint32_t *now) {
uint64_t lo = (e->pub.addr + 0xfff) & ~0xfffull, hi = (e->pub.addr + e->t.L.size) & ~0xfffull;
if (hi <= lo) lo = e->pub.addr, hi = e->pub.addr + e->t.L.size;
int ch = vk_mem_changed(lo, hi - lo, since, now);
if (ch != 1 || !since || !e->snap || memcmp(e->snap, (const void *)(uintptr_t)e->pub.addr, e->t.L.size)) return ch == 1;
log_limited(&n_kept, "write-back of %#llx kept: fmt %#x %ux%u, %llu bytes; a neighbour wrote its page",
(unsigned long long)e->pub.addr, e->t.d.format, e->t.d.width, e->t.d.height,
(unsigned long long)e->t.L.size);
return 0;
}
// The pages of an uploaded texture show a write (or are not tracked): whether its own bytes are still those it was
// uploaded with (upload, content_hash). WriteTexels since the upload always counts as a change.
static int same_content(const Tex *e) {
static int on = -1; // NX2NAVI_TEX_SAME=0: every write to the pages uploads again (diagnosis)
if (on < 0) { const char *v = cfg_get("NX2NAVI_TEX_SAME"); on = !(v && !strcmp(v, "0")); }
if (!on || !e->chash_ok || e->wt_n || e->t.L.size > CHASH_MAX) return 0;
if (content_hash((const void *)(uintptr_t)e->pub.addr, e->t.L.size) != e->chash) return 0;
ru_same_n++, ru_same_bytes += e->t.L.size;
return 1;
}
// A mirrored texture is uploaded on first use and again when its storage pages carry a newer write generation.
static void make_fresh(Tex *e) {
if (!e->pub.mirrored || e->pub.gpu_owned) return;
uint64_t size = e->t.L.size;
if (!e->uploaded) {
vk_mem_changed(e->pub.addr, size, 0, &e->wgen); // resets the write bits before the storage is read
upload(e);
return;
}
uint32_t now;
int ch = vk_mem_changed(e->pub.addr, size, e->wgen, &now);
e->wgen = now;
e->last_ch = ch;
if (ch && !same_content(e)) upload(e); // 1 written, -1 not tracked: both upload again, unless the bytes match
}
// make_fresh for draw bindings: the page check answers from this submission's write-watch snapshot (vk_memory.c
// vk_mem_changed_snap) instead of a system call per texture. A write the snapshot of the texture's region missed (made
// after it, during this replay) keeps its write-watch bit and is caught by the next submission's snapshot.
static void make_fresh_snap(Tex *e) {
if (!e->pub.mirrored || e->pub.gpu_owned) return;
uint32_t now;
n_checks++;
int ch = vk_mem_changed_snap(e->pub.addr, e->t.L.size, e->uploaded ? e->wgen : 0, submit_epoch, &now);
e->wgen = now;
if (!e->uploaded) { upload(e); return; }
e->last_ch = ch;
if (ch && !same_content(e)) upload(e);
}
// ---------------------------------------------------------------- aliases (NX2NAVI_TEX_ALIAS)
// Bytes per block and block size of the image's VkFormat (ASTC decodes to RGBA8, RGB expands to RGBA); 0 when the
// VkFormat substitutes another one of unknown size.
static int img_block(const Tex *e, int *bw, int *bh) {
if (e->f.astc_bc) { *bw = *bh = 4; return 16; }
if (e->f.astc_bw) { *bw = *bh = 1; return 4; }
int b = nvn_fmt_bpb(e->t.d.format, bw, bh);
if (e->f.expand_rgb) { *bw = *bh = 1; return b / 3 * 4; }
return e->f.vk == e->f.ideal ? b : 0;
}
// The new entry e can take src's content texel for texel: colour, one sample, the same shape and storage layout, and
// image formats of one block size.
static int alias_compatible(const Tex *e, const Tex *src) {
const VkTexImage *p = &e->pub, *q = &src->pub;
if (e->f.depth || e->f.stencil || src->f.depth || src->f.stencil || p->samples > 1 || q->samples > 1) return 0;
if (p->type != q->type || p->width != q->width || p->height != q->height || p->depth != q->depth) return 0;
if (e->t.L.size != src->t.L.size || nvn_tex_is_linear(&e->t.d) != nvn_tex_is_linear(&src->t.d)) return 0;
int ew, eh, sw, sh, eb = img_block(e, &ew, &eh), sb = img_block(src, &sw, &sh);
return eb && eb == sb && ew == sw && eh == sh;
}
// src holds content the storage bytes do not: the GPU wrote it and, for a mirrored texture, it was not written back
// and the CPU has not rewritten the storage since (the new entry's upload then reads the newer bytes).
static int alias_has_gpu_content(const Tex *src) {
if (!src->pub.gpu_owned) return 0;
if (!src->pub.mirrored) return 1;
uint32_t now;
return src->gpu_dirty && !storage_rewritten(src, src->gpu_gen, &now);
}
// Maxwell reads the same bytes through both descriptors: src's content goes into the new image e at the replay's
// current position, before any later command of the submission uses e. Through a buffer, since bound images share
// their memory and image copies may not overlap; e leaves UNDEFINED between the two halves, so its first transition
// cannot clobber src's bytes before they are read. 0 when no replay is recording (e is then not initialised yet).
static int alias_copy(Tex *e, const Tex *src) {
int reopen;
VkCommandBuffer cb = vk_exec_inline_begin(&reopen);
if (!cb) return 0;
int bw, bh, bpb = img_block(e, &bw, &bh);
uint32_t levels = e->pub.levels < src->pub.levels ? e->pub.levels : src->pub.levels;
uint32_t layers = e->pub.layers < src->pub.layers ? e->pub.layers : src->pub.layers, n = levels * layers, k = 0;
VkBufferImageCopy *rg = calloc(n, sizeof *rg);
if (!rg) rt_fatal("vk: out of memory for an alias copy");
VkDeviceSize total = 0;
for (uint32_t lv = 0; lv < levels; lv++)
for (uint32_t ly = 0; ly < layers; ly++, k++) {
uint32_t w = mip(e->pub.width, lv), h = mip(e->pub.height, lv), z = mip(e->pub.depth, lv);
rg[k] = (VkBufferImageCopy){.bufferOffset = total, .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, lv, ly, 1},
.imageExtent = {w, h, z}};
total += ((VkDeviceSize)(w + bw - 1) / bw * ((h + bh - 1) / bh) * z * bpb + 15) & ~15ull;
}
VkBuffer buf = vk_exec_inline_buffer(total);
if (!buf) {
free(rg);
vk_exec_inline_end(reopen);
return 0;
}
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
vkCmdCopyImageToBuffer(cb, src->pub.image, VK_IMAGE_LAYOUT_GENERAL, buf, n, rg);
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT};
VkImageMemoryBarrier2 ib = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT,
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = e->pub.image,
.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb,
.imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &ib};
vkCmdPipelineBarrier2(cb, &di);
vkCmdCopyBufferToImage(cb, buf, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, n, rg);
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
vk_exec_inline_end(reopen);
free(rg);
return 1;
}
// Descriptors whose image creation failed (VRAM exhausted, an unsupported format): not tried again for 2 s, so a
// draw that names one every frame does not repeat a failing allocation on every use. Texture lock held.
#define FAILED_HOLD 32
static struct {
TexKey k;
uint64_t until_ns;
} failed_hold[FAILED_HOLD];
static unsigned failed_next;
static int create_held(const TexKey *k) {
uint64_t now = plat_now_ns();
for (int i = 0; i < FAILED_HOLD; i++)
if (failed_hold[i].until_ns > now && !memcmp(&failed_hold[i].k, k, sizeof *k)) return 1;
return 0;
}
static void create_failed(const TexKey *k) {
unsigned i = failed_next++ % FAILED_HOLD;
failed_hold[i].k = *k, failed_hold[i].until_ns = plat_now_ns() + 2000000000ull;
}
static Tex *entry_of(const tex_t *t) {
TexKey k;
make_key(t, &k);
Tex *e = nvn_hm_get(&by_key, &k);
if (!e) {
if (create_held(&k)) return NULL;
if (tex_alias < 0) {
tex_alias = cfg_bool("NX2NAVI_TEX_ALIAS");
rt_log("vk: texture aliases: %s", tex_alias ? "a new image takes the live image's content" : "a new image starts empty");
}
if (!(e = create_tex(t, &k, tex_alias))) return create_failed(&k), NULL;
perf_count(PERF_CNT_TEX_NEW);
// released entries of another descriptor at this address are replaced by the new one; a live one holding
// GPU-written content is the alias source, preferring the one that differs only in swizzle (same_but_swizzle)
int alias = 0;
Tex *src = NULL;
for (Tex *p = nvn_hm_get(&by_addr, &k.addr), *nx; p; p = nx) {
nx = p->next_at;
if (p->released) { destroy_tex(p); continue; }
alias = 1;
if (tex_alias && (!src || (same_but_swizzle(&p->key, &k) && !same_but_swizzle(&src->key, &k))) &&
alias_compatible(e, p) && alias_has_gpu_content(p))
src = p;
}
if (alias) perf_count(PERF_CNT_TEX_ALIAS);
if (tex_alias) {
if (src && alias_copy(e, src)) {
n_alias_copied++;
if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes
vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen);
e->uploaded = 1;
e->pub.gpu_owned = 1;
}
e->chash_ok = 0;
} else {
init_image(e, vk_upload_begin());
vk_upload_end();
if (alias) {
if (e->pub.mirrored) n_alias_upload++;
else n_alias_uninit++;
}
}
}
e->next_at = nvn_hm_get(&by_addr, &k.addr);
nvn_hm_put(&by_addr, &k.addr, e);
nvn_hm_put(&by_key, &k, e);
n_site[1]++;
tex_gen_bump(k.addr);
}
if (e->released) e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_bump(k.addr);
e->t = *t;
touch(e);
return e;
}
// vk_dump: upload bookkeeping of an image vk_texture_get / vk_texture_by_id returned. pages_written: whether its
// storage pages were written since the last check (1, 0, -1 not tracked); the query leaves that check's state alone.
void vk_texture_debug(const VkTexImage *img, uint32_t *uploads, int *last_change, int *pages_written) {
lock();
const Tex *e = (const Tex *)img; // pub is the first member
uint32_t now;
*uploads = e->n_up;
*last_change = e->last_ch;
*pages_written = e->pub.mirrored && e->uploaded ? vk_mem_changed(e->pub.addr, e->t.L.size, e->wgen, &now) : 0;
unlock();
}
const VkTexImage *vk_texture_get(const tex_t *t) {
if (!t || !t->mem) return NULL;
lock();
Tex *e = entry_of(t);
if (e) make_fresh(e);
unlock();
return e ? &e->pub : NULL;
}
// Render targets the CPU reads without NVN's invalidate (vk_texture_write_back_small); small_pending: one may be dirty.
// wb_limit: targets up to this size (bytes) always go back when the CPU waits for a fence, NX2NAVI_WB_FENCE_LIMIT, read
// once. A larger one goes back unless the GPU wrote it in each of the last WB_RUN frames (wb_wanted). wb_trace:
// NX2NAVI_TRACE=gfx, read together with it.
enum { WB_SMALL_DEFAULT = 64 << 10, WB_RUN = 16 };
static int small_pending;
static uint64_t wb_limit = WB_SMALL_DEFAULT;
static int wb_trace;
static int wb_cfg_read;
static void wb_cfg_init(void) {
if (wb_cfg_read) return;
int v = cfg_int("NX2NAVI_WB_FENCE_LIMIT");
wb_limit = v < 0 ? 0 : (uint64_t)v;
wb_trace = (rt_trace & RT_TRACE_GFX) != 0;
wb_cfg_read = 1;
if (wb_limit != WB_SMALL_DEFAULT) rt_log("vk: fence write-back limit %llu bytes", (unsigned long long)wb_limit);
}
// Whether the fence write-back takes a dirty target. A target the GPU draws every frame is scratch the CPU does not
// read: in Hyrule Field three 256x256 RGBA8 targets and a 256x256 RG8 one, which cost 2.7 ms a frame to write back.
// NX2navi's 260x260 RG8 grass maps, which its fire spread reads after a fence wait (#132), are drawn once per change, or
// up to 9 frames in a row while a tile loads.
static int wb_wanted(const Tex *e) { return e->t.L.size <= wb_limit || e->run < WB_RUN; }
void vk_texture_gpu_written(const tex_t *t) {
if (!t || !t->mem) return;
vk_exec_frame_written(t);
lock();
Tex *e = entry_of(t);
if (e) {
make_fresh(e); // a partial render keeps what the storage held before
e->pub.gpu_owned = 1;
e->chash_ok = 0; // the image now holds the GPU's content: matching storage bytes no longer mean "unchanged"
uint64_t f = vk_exec_recording_fence();
if (f > e->wr_fence) e->wr_fence = f;
if (e->pub.mirrored) {
e->gpu_dirty = 1;
storage_rewritten(e, 0, &e->gpu_gen); // CPU writes up to here are older than this render
if (wb_exact < 0) wb_exact = cfg_bool("NX2NAVI_WB_EXACT");
if (wb_exact && !e->snap && e->t.L.size <= SNAP_MAX && fills_no_page(e) && !(e->snap = malloc(SNAP_MAX)))
rt_fatal("vk: out of memory for a texture's storage copy");
if (e->snap) memcpy(e->snap, (const void *)(uintptr_t)e->pub.addr, e->t.L.size);
e->gpu_seq = ++gpu_writes;
e->gpu_fence = vk_exec_recording_fence();
uint64_t fr = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
if (fr != e->run_frame || !e->run) {
e->run = fr == e->run_frame + 1 && e->run ? (uint8_t)(e->run < WB_RUN ? e->run + 1 : WB_RUN) : 1;
e->run_frame = fr;
}
wb_cfg_init();
if (wb_wanted(e)) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE);
own_add(e);
}
}
unlock();
}
void vk_texture_release(const tex_t *t) {
if (!t || !t->mem) return;
TexKey k;
make_key(t, &k);
lock();
Tex *e = nvn_hm_get(&by_key, &k);
if (e && !e->released) {
e->released = 1;
e->rel_serial = reg_serial;
e->rel_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
nreleased++;
rel_append(e);
n_site[3]++;
tex_gen_bump(k.addr);
sweep_released();
}
unlock();
}
void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc) {
lock();
Tex **list = malloc(sizeof *list * (size_t)(by_key.n + 1));
int n = 0;
for (uint32_t i = 0; i < by_key.cap; i++) {
Tex *e = by_key.vals[i];
if (e && ((e->key.addr >= lo && e->key.addr - lo < size) || (alloc && e->bound_to == alloc))) list[n++] = e;
}
for (int i = 0; i < n; i++) destroy_tex(list[i]);
free(list);
unlock();
}
// ---------------------------------------------------------------- descriptor slots
void vk_texture_register(int id, const tex_t *t, const view_t *v) {
if (id < 0 || id >= (1 << 20)) rt_fatal("vk: texture slot %d out of range", id);
lock();
if (id >= ntslots) {
int n = ntslots ? ntslots : 1024;
while (n <= id) n *= 2;
TexSlot *s = realloc(tslots, sizeof *s * (size_t)n);
if (!s) rt_fatal("vk: out of memory for texture slots");
memset(s + ntslots, 0, sizeof *s * (size_t)(n - ntslots));
tslots = s, ntslots = n;
}
if (!sinfo) {
uint64_t *p = calloc(1 << 20, sizeof *p);
slot_next = malloc(sizeof *slot_next << 20), slot_prev = malloc(sizeof *slot_prev << 20);
slot_bucket = calloc(1 << 20, sizeof *slot_bucket);
if (!p || !slot_next || !slot_prev || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens");
for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1;
for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1;
__atomic_store_n(&sinfo, p, __ATOMIC_RELEASE);
}
TexSlot *s = &tslots[id];
// The same texture and view registered again (Ocarina rewrites much of its descriptor pool every frame): the slot
// keeps its serial and resolved entry, so the texture pool refresh (vk_exec_draw.c pool_sync_entries) does not
// resolve it again. Only while that entry is live: after a release, a registration is what makes a slot current
// again (slot_resolve's stale rule), so it goes the full way.
if (s->used && s->ce && !s->ce->released && !memcmp(&s->t, t, sizeof *t) && s->has_view == (v != NULL) &&
(!v || !memcmp(&s->v, v, sizeof *v))) {
n_same_regs++;
unlock();
return;
}
s->t = *t;
s->used = 1;
s->ce = NULL;
slots_gen++;
if (!++reg_serial) reg_serial = 1;
__atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE);
slot_rebucket(id, agen_idx((uintptr_t)t->mem));
dirty_push((uint32_t)id);
s->has_view = v != NULL;
if (v) s->v = *v;
else memset(&s->v, 0, sizeof s->v);
s->khash = 0;
if (!tex_mirrored(t)) { // FNV-1a over the key (make_key zeroes its padding) and the view
TexKey k;
make_key(t, &k);
uint64_t h = 0xcbf29ce484222325ull;
const uint8_t *b = (const uint8_t *)&k;
for (size_t i = 0; i < sizeof k; i++) h = (h ^ b[i]) * 0x100000001b3ull;
b = (const uint8_t *)&s->v;
for (size_t i = 0; i < sizeof s->v; i++) h = (h ^ b[i]) * 0x100000001b3ull;
s->khash = (h ^ s->has_view) | 1;
}
unlock();
}
// The descriptor hash of a GPU-written slot: two slots with equal hashes, the same address generation and no entry
// created, destroyed, released or taken back between their lookups (vk_texture_entry_gen) resolve alike. 0 for an
// unregistered or CPU-written slot (its resolution depends on when it was registered: slot_resolve's stale rule).
uint64_t vk_texture_slot_khash_locked(int id) { return id >= 0 && id < ntslots && tslots[id].used ? tslots[id].khash : 0; }
uint32_t vk_texture_entry_gen(void) { return tex_gen; }
// The entry slot s resolves to, and the view and has_view a caller gets with it (the slot's, with the descriptor's
// swizzle when the entry is another descriptor at the same storage). NULL when the slot names no texture.
// Entry creation budget of slot lookups (vk_texture_create_budget_locked): -1 none. The bindless pools look up every
// registered slot, and most of a pool is stale descriptors (storage freed or reused); with a budget a lookup creates at
// most that many entries, only for mapped storage, and notes the slots it left for later (create_deferred).
static int create_budget = -1, create_deferred;
void vk_texture_create_budget_locked(int n) { create_budget = n, create_deferred = 0; }
int vk_texture_create_deferred_locked(void) { int d = create_deferred; create_deferred = 0; return d; }
static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) {
*view = s->v;
*has_view = s->has_view;
TexKey k;
make_key(&s->t, &k);
Tex *e = nvn_hm_get(&by_key, &k);
int stale = 0;
if (!e || e->released) { // a finalized slot follows the live texture at its storage
// Prefer the live entry that differs from the slot descriptor only in its swizzle: transient render targets
// share storage, and the newest one at the address is usually another size. The map screen samples its
// 2250x1080 map target through an (R,G,B,1) descriptor and followed a 960x60 target at the same address.
Tex *own = e, *any = NULL;
e = NULL;
// A slot whose content the CPU writes (tex_mirrored) follows another image only for content its own bytes do
// not hold, the GPU's: Ocarina rasterizes each UI caption on the CPU into a small r8 glyph atlas and reuses the
// memory of a freed one for the next, of another width. Following the older 256x39 atlas at the address, the
// replay uploaded the new 128-wide bytes with a 256-texel row, and the Basic Controls screen's ZL caption
// ("Target") sampled a mix of its glyphs and stale ones. Without such an image the slot gets its own below
// (entry_of), uploaded from its storage with its own geometry.
int cpu_slot = tex_mirrored(&s->t);
for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) {
if (p->released) continue;
// Only an entry of the slot's target: Ocarina's volumetric fog registers a 3D two-channel volume (32x24x64)
// over the storage of the bloom pass's 480x270 2D target; followed, the fog computes and the fog pass got
// that 2D image, no 3D view can be made of it (vk_exec_draw.c get_view) and they read and wrote stand-ins.
// With the target kept they get the volume itself.
if (p->key.d.target != k.d.target) continue;
if (same_but_swizzle(&p->key, &k)) { e = p; break; }
if (!any && (!cpu_slot || alias_has_gpu_content(p))) any = p;
}
// own is missing or released here; for a CPU slot entry_of below finds or revives it by key. A CPU slot whose
// own entry was released after the slot was registered is stale: the game finalized that texture and is
// freeing its storage, and reviving it uploaded from that storage while another thread unmapped it (an
// access violation in tex_bl_copy from the pool refresh, Kokiri Forest). It resolves to nothing; a slot
// registered again after the release is live and revives the entry.
if (!e && any) e = any;
else if (!e && !cpu_slot) e = own;
else if (!e && own && (uint32_t)(sinfo[s - tslots] >> 16) <= own->rel_serial) stale = 1;
}
if (!e && s->t.mem && !stale) {
if (create_budget < 0) e = entry_of(&s->t);
else if (!create_budget) create_deferred = 1;
else if (mem_readable((uint64_t)(uintptr_t)s->t.mem, s->t.L.size)) e = entry_of(&s->t), create_budget--;
}
// A followed texture is another descriptor at the same storage: its image holds the content, but sampling takes the
// slot descriptor's swizzle. The UI renders text into RGBA8 and samples it through a (A,A,A,A) descriptor; with the
// render target's RGBA swizzle the text blur saw R = 1 everywhere and drew solid boxes behind tips and labels.
if (e && !(s->has_view && s->v.set_swizzle)) {
const nvn_texdesc *d = &s->t.d;
uint8_t sw[4] = {(uint8_t)d->swizzle_r, (uint8_t)d->swizzle_g, (uint8_t)d->swizzle_b, (uint8_t)d->swizzle_a};
if (!(sw[0] | sw[1] | sw[2] | sw[3])) // descriptors built without nvnTextureBuilderSetDefaults
sw[0] = NVN_TEXTURE_SWIZZLE_R, sw[1] = NVN_TEXTURE_SWIZZLE_G, sw[2] = NVN_TEXTURE_SWIZZLE_B,
sw[3] = NVN_TEXTURE_SWIZZLE_A;
if (memcmp(sw, e->pub.swz, 4)) {
if (!s->has_view) memset(view, 0, sizeof *view);
memcpy(view->swz, sw, 4);
view->set_swizzle = 1;
*has_view = 1;
}
}
return e;
}
void vk_texture_lock(void) { lock(); }
void vk_texture_unlock(void) { unlock(); }
uint32_t vk_texture_slots_gen(void) { return slots_gen; }
uint64_t vk_texture_same_regs(void) { return __atomic_load_n(&n_same_regs, __ATOMIC_RELAXED); }
// Diagnosis for the texture pool log: address-generation bumps since the last call, the slots of the fullest buckets
// and the bumps those buckets took. Texture lock held by none: takes it.
void vk_texture_bucket_stats(char *out, size_t n) {
lock();
static uint32_t occ[AGEN_N];
memset(occ, 0, sizeof occ);
for (int i = 0; i < ntslots && sinfo; i++)
if (tslots[i].used) occ[sinfo[i] & 0xffff]++;
int top[4] = {-1, -1, -1, -1};
for (int b = 0; b < AGEN_N; b++)
for (int k = 0; k < 4; k++)
if (top[k] < 0 || occ[b] > occ[top[k]]) {
for (int j = 3; j > k; j--) top[j] = top[j - 1];
top[k] = b;
break;
}
uint64_t hot = 0; // bumps weighted by the slots they invalidate
for (int b = 0; b < AGEN_N; b++) hot += (uint64_t)bump_count[b] * occ[b];
snprintf(out, n, "%llu bumps (destroy %llu, create %llu, take-back %llu, release %llu), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u",
(unsigned long long)n_bumps, (unsigned long long)n_site[0], (unsigned long long)n_site[1],
(unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]],
bump_count[top[0]], bump_count[top[1]], bump_count[top[2]], bump_count[top[3]]);
n_bumps = 0;
memset(n_site, 0, sizeof n_site);
memset(bump_count, 0, sizeof bump_count);
unlock();
}
uint64_t vk_texture_slot_token(int id) {
const uint64_t *si = __atomic_load_n(&sinfo, __ATOMIC_ACQUIRE);
if (!si || id < 0 || id >= (1 << 20)) return 0;
uint64_t v = __atomic_load_n(&si[id], __ATOMIC_ACQUIRE);
return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0;
}
uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); }
int vk_texture_slot_count_locked(void) { return ntslots; }
int vk_sampler_slot_count_locked(void) { return nsslots; }
const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) {
lock();
const VkTexImage *r = vk_texture_by_id_locked(id, view, has_view);
unlock();
return r;
}
const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
if (id < 0 || id >= ntslots || !tslots[id].used) return NULL;
TexSlot *s = &tslots[id];
uint32_t g = agen[sinfo[id] & 0xffff];
if (!s->ce || s->cgen != g) { // resolved again: an entry at its address was created, dropped or released, or the
view_t v; // slot registered (s->ce cleared); an empty result is not kept: image creation may
int hv; // succeed later
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it
s->cview = v, s->chas_view = (uint8_t)hv;
s->cgen = g;
s->cepoch = 0;
}
Tex *e = s->ce;
// Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a
// replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not
// mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do.
if (e && s->cepoch != submit_epoch) {
if (!e->uploaded || e->checked != submit_epoch) {
make_fresh_snap(e);
e->checked = submit_epoch;
touch(e);
}
s->cepoch = submit_epoch;
}
if (view) *view = has_view ? s->cview : s->v;
if (has_view) *has_view = view ? s->chas_view : s->has_view;
return e ? &e->pub : NULL;
}
// vk_dump: the swizzle selectors of the descriptor registered in slot id; 0 when the slot is empty.
int vk_texture_slot_swz(int id, uint8_t swz[4]) {
lock();
int ok = id >= 0 && id < ntslots && tslots[id].used;
if (ok) {
const nvn_texdesc *d = &tslots[id].t.d;
swz[0] = (uint8_t)d->swizzle_r, swz[1] = (uint8_t)d->swizzle_g, swz[2] = (uint8_t)d->swizzle_b;
swz[3] = (uint8_t)d->swizzle_a;
}
unlock();
return ok;
}
uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC_RELAXED); }
void vk_texture_new_submit(void) {
lock();
if (!++submit_epoch) submit_epoch = 1;
sweep_pressure();
static uint64_t log_at;
uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) / 60;
if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) {
log_at = f;
rt_log("perf: frame %llu: texture aliases: content copied %llu, filled from storage %llu, left uninitialised %llu",
(unsigned long long)f * 60, (unsigned long long)n_alias_copied, (unsigned long long)n_alias_upload,
(unsigned long long)n_alias_uninit);
n_alias_copied = n_alias_upload = n_alias_uninit = 0;
rt_log("perf: frame %llu: texture re-uploads %llu (%.1f MiB, %.2f ms), after WriteTexels %llu (%.1f MiB, %.2f ms); "
"skipped with unchanged bytes %llu (%.1f MiB)",
(unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6,
(unsigned long long)ru_wt_n, ru_wt_bytes / 1048576.0, ru_wt_ns / 1e6, (unsigned long long)ru_same_n,
ru_same_bytes / 1048576.0);
ru_n = ru_bytes = ru_ns = ru_wt_n = ru_wt_bytes = ru_wt_ns = ru_same_n = ru_same_bytes = 0;
uint64_t ck, calls, untr;
vk_mem_snap_stats(&ck, &calls, &untr);
rt_log("perf: frame %llu: binding storage checks %llu (texture %llu), write-watch snapshot queries %llu, untracked %llu",
(unsigned long long)f * 60, (unsigned long long)ck, (unsigned long long)n_checks, (unsigned long long)calls,
(unsigned long long)untr);
n_checks = 0;
rt_log("perf: frame %llu: released textures kept %d (%.0f MiB of own memory), dropped past the size limit %llu, "
"under VRAM pressure %llu, under commit pressure %llu; idle live textures evicted %llu (%.1f MiB)",
(unsigned long long)f * 60, nreleased, rel_bytes / 1048576.0, (unsigned long long)n_cap_drops,
(unsigned long long)n_pressure_drops, (unsigned long long)n_commit_drops, (unsigned long long)n_evict,
n_evict_bytes / 1048576.0);
n_cap_drops = n_pressure_drops = n_commit_drops = n_evict = n_evict_bytes = 0;
}
unlock();
}
// ---------------------------------------------------------------- texel reads and writes
typedef struct { unsigned level, layer0; int nz, is3d, bw, bh, bpb; size_t wb, hb, row, img, x0, y0; } Region;
static int region_of(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img, Region *g) {
g->bpb = nvn_fmt_bpb(t->d.format, &g->bw, &g->bh);
if (!g->bpb || r->width <= 0 || r->height <= 0 || r->depth < 0 || r->xoffset < 0 || r->yoffset < 0 || r->zoffset < 0 ||
r->xoffset % g->bw || r->yoffset % g->bh)
return 0;
g->level = v && v->set_levels ? v->base_level : 0;
g->layer0 = v && v->set_layers ? v->min_layer : 0;
g->nz = r->depth ? r->depth : 1;
g->is3d = t->d.target == NVN_TEXTURE_TARGET_3D;
g->wb = (size_t)((r->width + g->bw - 1) / g->bw) * (size_t)g->bpb;
g->hb = (size_t)(r->height + g->bh - 1) / (size_t)g->bh;
g->row = row ? row : g->wb;
g->img = img ? img : g->row * g->hb;
g->x0 = (size_t)(r->xoffset / g->bw) * (size_t)g->bpb;
g->y0 = (size_t)(r->yoffset / g->bh);
return 1;
}
// Pool bytes <-> caller rows of the region alone (nvn_texels_box), not through a
// linear copy of the whole level: for a small region of a large texture that cost tens of milliseconds while the guest
// thread held its virtual core and the caller the texture lock (monsters2 replay, frames 765-790).
static int storage_texels(const tex_t *t, const Region *g, const NVNcopyRegion *r, uint8_t *mem, int write) {
for (int z = 0; z < g->nz; z++) {
unsigned layer = g->is3d ? 0 : g->layer0 + (unsigned)r->zoffset + (unsigned)z;
uint32_t slice = g->is3d ? (uint32_t)(r->zoffset + z) : 0;
if (nvn_texels_box(&t->d, g->level, layer, (uint32_t)g->x0, (uint32_t)g->y0, slice, (uint32_t)g->wb,
(uint32_t)g->hb, t->mem, mem + (size_t)z * g->img, g->row, !write))
return 0;
}
return 1;
}
static int read_aspect(const Tex *e, VkImageAspectFlags *a) {
if (!e->f.depth && !e->f.stencil) { *a = VK_IMAGE_ASPECT_COLOR_BIT; return 1; }
switch (e->t.d.format) { // packed depth-stencil storage has no single-aspect copy
case 0x31: *a = VK_IMAGE_ASPECT_STENCIL_BIT; return 1;
case 0x32: case 0x34: *a = VK_IMAGE_ASPECT_DEPTH_BIT; return 1;
default: return 0;
}
}
// An image region read through a staging buffer. texels_read_record records the copy into the open upload batch;
// texels_read_take copies the rows out once that batch is done (vk_upload_flush(1)) and frees the buffer. Several
// records before one flush cost one GPU round trip instead of one each.
typedef struct {
VkBuffer sb;
VmaAllocation sa;
const uint8_t *stg;
size_t slice, ib; // ib: bytes per texel of an expanded RGB image, 0 when the image holds the storage layout
uint32_t w;
} TexRead;
static int texels_read_record(Tex *e, const Region *g, const NVNcopyRegion *r, TexRead *t) {
static volatile int32_t n_astc, n_ds;
VkImageAspectFlags aspect;
if (e->f.astc_bw) {
log_limited(&n_astc, "ReadTexels of ASTC texture %#llx: the image holds decoded RGBA8; zeros returned",
(unsigned long long)e->pub.addr);
return 0;
}
if (!read_aspect(e, &aspect)) {
log_limited(&n_ds, "texels of packed depth-stencil format %#x are not copied", e->t.d.format);
return 0;
}
if (g->level >= e->pub.levels) return 0;
uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height;
t->w = w;
t->ib = e->f.expand_rgb ? (size_t)g->bpb / 3 * 4 : 0;
t->slice = t->ib ? (size_t)w * h * t->ib : g->wb * g->hb;
t->stg = vk_staging(t->slice * (size_t)g->nz, &t->sb, &t->sa);
VkBufferImageCopy *rg = calloc((size_t)g->nz, sizeof *rg);
if (!rg) rt_fatal("vk: out of memory for a texture read");
for (int z = 0; z < g->nz; z++)
rg[z] = (VkBufferImageCopy){.bufferOffset = t->slice * (size_t)z,
.imageSubresource = {aspect, g->level, g->is3d ? 0 : g->layer0 + (uint32_t)r->zoffset + (uint32_t)z, 1},
.imageOffset = {r->xoffset, r->yoffset, g->is3d ? r->zoffset + z : 0},
.imageExtent = {w, h, 1}};
VkCommandBuffer cb = vk_upload_begin();
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT);
vkCmdCopyImageToBuffer(cb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, t->sb, (uint32_t)g->nz, rg);
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_HOST_BIT,
VK_ACCESS_2_HOST_READ_BIT);
vk_upload_end();
free(rg);
return 1;
}
static void texels_read_take(const Tex *e, const Region *g, const TexRead *t, uint8_t *mem) {
VkCtx *c = vk_ctx();
vmaInvalidateAllocation(c->vma, t->sa, 0, VK_WHOLE_SIZE);
for (int z = 0; z < g->nz; z++) {
const uint8_t *s = t->stg + t->slice * (size_t)z;
uint8_t *m = mem + (size_t)z * g->img;
for (size_t y = 0; y < g->hb; y++) {
if (t->ib) shrink_rgb(e->t.d.format, s + y * t->w * t->ib, m + y * g->row, t->w);
else memcpy(m + y * g->row, s + y * g->wb, g->wb);
}
}
vmaDestroyBuffer(c->vma, t->sb, t->sa);
}
// Image <-> caller rows through a staging buffer; the GPU copy is waited for.
static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t *mem, int write) {
static volatile int32_t n_astc, n_ds;
VkCtx *c = vk_ctx();
VkImageAspectFlags aspect;
if (!write) {
TexRead t;
if (!texels_read_record(e, g, r, &t)) return 0;
vk_upload_flush(1);
texels_read_take(e, g, &t, mem);
return 1;
}
if (!read_aspect(e, &aspect)) {
log_limited(&n_ds, "texels of packed depth-stencil format %#x are not copied", e->t.d.format);
return 0;
}
if (g->level >= e->pub.levels) return 0;
uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height;
int rgb = e->f.expand_rgb, astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu();
size_t texels = (size_t)w * h, ib = agpu ? 0 : astc ? 4 : rgb ? (size_t)g->bpb / 3 * 4 : 0;
size_t slice = ib ? texels * ib : g->wb * g->hb; // agpu: the raw blocks, decoded by vk_astc_record
VkBuffer sb;
VmaAllocation sa;
uint8_t *stg = vk_staging(slice * (size_t)g->nz, &sb, &sa);
VkBufferImageCopy *rg = calloc((size_t)g->nz, sizeof *rg);
for (int z = 0; z < g->nz; z++) {
uint8_t *s = stg + slice * (size_t)z;
const uint8_t *m = mem + (size_t)z * g->img;
if (agpu) {
for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb);
} else if (astc) {
uint8_t *tight = malloc(g->wb * g->hb);
for (size_t y = 0; y < g->hb; y++) memcpy(tight + y * g->wb, m + y * g->row, g->wb);
uint64_t a0 = perf_begin();
astc_decode_ldr_mt(tight, w, h, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, s);
perf_end(PERF_ASTC, a0);
free(tight);
} else if (rgb) {
for (size_t y = 0; y < g->hb; y++) expand_rgb(e->t.d.format, m + y * g->row, s + y * w * ib, w);
} else {
for (size_t y = 0; y < g->hb; y++) memcpy(s + y * g->wb, m + y * g->row, g->wb);
}
rg[z] = (VkBufferImageCopy){.bufferOffset = slice * (size_t)z,
.imageSubresource = {aspect, g->level, g->is3d ? 0 : g->layer0 + (uint32_t)r->zoffset + (uint32_t)z, 1},
.imageOffset = {r->xoffset, r->yoffset, g->is3d ? r->zoffset + z : 0},
.imageExtent = {w, h, 1}};
}
VkCommandBuffer cb = vk_upload_begin();
vmaFlushAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
barrier(cb, ALL_RW, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT);
int done = 0;
// A BC3 image takes whole 4x4 blocks, or a part that ends on the mip's edge; a region of a 5x5 or 6x6 footprint
// may end inside a block (vk_astc_record checks the start corner).
uint32_t mw = e->t.d.width >> g->level, mh = e->t.d.height >> g->level;
int bc_cut = e->f.astc_bc && (((w & 3) && (uint32_t)r->xoffset + w != (mw ? mw : 1)) ||
((h & 3) && (uint32_t)r->yoffset + h != (mh ? mh : 1)));
if (agpu && !bc_cut) { // bc_cut: nothing recorded, logged below as not decoded
VkAstcRegion *ar = calloc((size_t)g->nz, sizeof *ar);
for (int z = 0; z < g->nz; z++)
ar[z] = (VkAstcRegion){.src = slice * (size_t)z, .w = w, .h = h, .slices = 1, .level = rg[z].imageSubresource.mipLevel,
.layer = rg[z].imageSubresource.baseArrayLayer, .offset = rg[z].imageOffset};
uint64_t a0 = perf_begin();
done = vk_astc_record(cb, sb, slice * (size_t)g->nz, ar, g->nz, e->f.astc_bw, e->f.astc_bh, e->f.astc_srgb, e->f.astc_bc,
e->pub.image, aspect, VK_NULL_HANDLE);
perf_end(PERF_ASTC, a0);
free(ar);
}
if (!agpu) vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)g->nz, rg);
else if (!done) log_limited(&n_astc, "WriteTexels of ASTC texture %#llx not decoded on the GPU: left as it was",
(unsigned long long)e->pub.addr);
barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW);
vk_upload_end();
vk_upload_flush(1);
free(rg);
vmaDestroyBuffer(c->vma, sb, sa);
return 1;
}
static void zero_rows(const Region *g, uint8_t *dst) {
for (int z = 0; z < g->nz; z++)
for (size_t y = 0; y < g->hb; y++) memset(dst + (size_t)z * g->img + y * g->row, 0, g->wb);
}
int vk_texture_read_image(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) {
Region g;
if (!t || !region_of(t, v, r, row, img, &g)) return 0;
lock();
Tex *e = entry_of(t);
int ok = 0;
if (e) {
make_fresh(e);
ok = image_texels(e, &g, r, dst, 0);
}
unlock();
if (!ok) zero_rows(&g, dst);
return ok;
}
// ReadTexels: the pool bytes are the content of a mirrored texture the GPU has not rendered into; every other texture
// is read back from its image.
void vk_read_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint8_t *dst, uint64_t row, uint64_t img) {
static volatile int32_t n_bad;
Region g;
if (!region_of(t, v, r, row, img, &g)) {
log_limited(&n_bad, "ReadTexels: invalid region %d,%d,%d %dx%dx%d (format %#x)", r->xoffset, r->yoffset,
r->zoffset, r->width, r->height, r->depth, t->d.format);
return;
}
lock();
Tex *e = entry_of(t);
int ok = 0;
if (e && e->pub.mirrored && !e->pub.gpu_owned) ok = storage_texels(t, &g, r, dst, 0);
else if (e) {
make_fresh(e);
ok = image_texels(e, &g, r, dst, 0);
}
unlock();
if (!ok) zero_rows(&g, dst);
}
// WriteTexels: a mirrored texture gets the texels in its pool bytes (and in its image once the GPU owns it); every
// other texture in its image.
void vk_write_texels(const tex_t *t, const view_t *v, const NVNcopyRegion *r, const uint8_t *src, uint64_t row, uint64_t img) {
static volatile int32_t n_bad;
Region g;
if (!region_of(t, v, r, row, img, &g) || (v && v->set_format && v->format != t->d.format)) {
log_limited(&n_bad, "WriteTexels: invalid region %d,%d,%d %dx%dx%d or view format (format %#x)", r->xoffset,
r->yoffset, r->zoffset, r->width, r->height, r->depth, t->d.format);
return;
}
lock();
Tex *e = entry_of(t);
if (e && e->pub.mirrored) {
if (!storage_texels(t, &g, r, (uint8_t *)src, 1))
log_limited(&n_bad, "WriteTexels: region outside level %u of texture %#llx", g.level, (unsigned long long)e->pub.addr);
vk_guest_written((uintptr_t)t->mem, t->L.size);
if (e->pub.gpu_owned) image_texels(e, &g, r, (uint8_t *)src, 1);
else if (e->uploaded) e->wt_n++;
} else if (e) {
image_texels(e, &g, r, (uint8_t *)src, 1);
}
unlock();
}
// ---------------------------------------------------------------- CPU reads of GPU-written storage
// A mirrored texture the GPU rendered into holds its texels in its image only; its storage keeps what the CPU last put
// there. The game may still read that storage with the CPU: NX2navi moves its generated grass maps between pools with
// memcpy once the GPU is done, and the copy then carried zeros. Such a read writes the image back first.
//
// Every level and layer of every texture in v is copied to staging in one upload batch with one wait: one GPU round
// trip per level and layer took 0.1 ms each, and a 260x260 grass map has 9 levels. The storage is then written in the
// order of v, so where two textures share storage the later one wins.
typedef struct {
Tex *e;
Region g;
NVNcopyRegion r;
TexRead t;
} WbPart;
static void write_back_n(Tex **v, int n) {
int np = 0, cap = 0;
WbPart *p = NULL;
for (int i = 0; i < n; i++) {
Tex *e = v[i];
const nvn_texdesc *d = &e->t.d;
int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D;
uint32_t levels = linear ? 1 : e->pub.levels, layers = linear || is3d ? 1 : e->pub.layers;
for (uint32_t lv = 0; lv < levels; lv++)
for (uint32_t ly = 0; ly < layers; ly++) {
if (np == cap) {
cap = cap ? cap * 2 : 16;
if (!(p = realloc(p, sizeof *p * (size_t)cap))) rt_fatal("vk: out of memory for a texture write-back");
}
WbPart *q = &p[np];
view_t vw = {.set_levels = 1, .set_layers = 1, .base_level = (uint8_t)lv, .min_layer = (uint16_t)ly};
q->e = e;
q->r = (NVNcopyRegion){.width = (int)mip(d->width, lv),
.height = d->target == 0 || d->target == 3 ? 1 : (int)mip(d->height, lv),
.depth = is3d ? (int)mip(d->depth, lv) : 1};
if (region_of(&e->t, &vw, &q->r, 0, 0, &q->g) && texels_read_record(e, &q->g, &q->r, &q->t)) np++;
}
}
if (np) vk_upload_flush(1);
for (int i = 0; i < np; i++) {
WbPart *q = &p[i];
uint8_t *buf = malloc(q->g.img * (size_t)q->g.nz);
if (!buf) rt_fatal("vk: out of memory for a texture write-back");
texels_read_take(q->e, &q->g, &q->t, buf);
storage_texels(&q->e->t, &q->g, &q->r, buf, 1);
free(buf);
}
free(p);
for (int i = 0; i < n; i++) v[i]->gpu_dirty = 0;
}
// Mirrored textures overlapping [addr, addr+size) that the GPU wrote since their last write-back.
static int owned_dirty(uint64_t addr, uint64_t size, Tex **hit) {
int n = 0;
for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) {
Tex *e = owned[i];
if (__atomic_load_n(&e->gpu_dirty, __ATOMIC_RELAXED) && e->pub.addr + e->t.L.size > addr) {
if (!hit) return 1;
hit[n++] = e;
}
}
return n;
}
// A storage image may be written by the shaders of the submission being recorded (texture lock held, vk_exec_draw.c
// tex_info). Its storage is not marked GPU-owned: only the write fence for ReadTexels.
void vk_texture_storage_bound_locked(const VkTexImage *img) {
Tex *e = (Tex *)img; // pub is the first member
uint64_t f = vk_exec_recording_fence();
if (f > e->wr_fence) e->wr_fence = f;
}
uint64_t vk_texture_last_write(const tex_t *t) {
lock();
Tex *e = entry_of(t);
uint64_t f = e ? e->wr_fence : 0;
unlock();
return f;
}
size_t vk_texture_texel_span(const tex_t *t, const view_t *v, const NVNcopyRegion *r, uint64_t row, uint64_t img) {
Region g;
if (!region_of(t, v, r, row, img, &g)) return 0;
return (size_t)(g.nz - 1) * g.img + (g.hb - 1) * g.row + g.wb;
}
int vk_texture_cpu_stale(uint64_t addr, uint64_t size) {
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
plat_mutex_lock_shared(&owned_lock);
int dirty = owned_dirty(addr, size, NULL);
plat_mutex_unlock_shared(&owned_lock);
return dirty;
}
// The newest submission recorded to write a GPU-written mirrored texture overlapping [addr, addr+size); 0: none.
uint64_t vk_texture_dirty_fence(uint64_t addr, uint64_t size) {
if (!size || !__atomic_load_n(&nowned, __ATOMIC_ACQUIRE)) return 0;
lock();
uint64_t f = 0;
for (int i = owned_lower(addr > max_owned ? addr - max_owned : 0); i < nowned && owned[i]->pub.addr < addr + size; i++) {
const Tex *e = owned[i];
if (!e->gpu_dirty || e->pub.addr + e->t.L.size <= addr) continue;
if (e->gpu_fence > f) f = e->gpu_fence;
if (e->wr_fence > f) f = e->wr_fence;
}
unlock();
return f;
}
// The image is the newest content only while nothing wrote the storage since the GPU render: NX2navi finalizes grass maps,
// reuses their storage for new data and memcpys that data on, and writing the old image back first handed the copy
// a stale map or stale resource bytes (wrong textures in the distance). Every hit is checked before any is written
// back, since a write-back is itself a CPU write to storage another hit may share; the hits then go back oldest render
// first, so where two descriptors share storage the newer render wins.
void vk_texture_write_back(uint64_t addr, uint64_t size) {
lock();
Tex **hit = malloc(sizeof *hit * (size_t)(nowned + 1));
if (!hit) rt_fatal("vk: out of memory for a texture write-back");
int n = owned_dirty(addr, size, hit), k = 0; // owned[] cannot change while tex_lock is held
for (int i = 0; i < n; i++) {
uint32_t now;
if (storage_rewritten(hit[i], hit[i]->gpu_gen, &now) == 1) hit[i]->gpu_dirty = 0;
else hit[k++] = hit[i];
}
for (int i = 1; i < k; i++)
for (int j = i; j > 0 && hit[j - 1]->gpu_seq > hit[j]->gpu_seq; j--) {
Tex *x = hit[j];
hit[j] = hit[j - 1], hit[j - 1] = x;
}
write_back_n(hit, k);
for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB);
for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP);
free(hit);
unlock();
}
// NX2navi's grass system renders a 128x128 RGBA8 map of cut and trampled grass into a CPU-cached pool every frame and its
// CPU code reads the pool bytes without nvnMemoryPoolInvalidateMappedRange (cut reactions: blades, insects, the cut
// itself). On the Switch the bytes are there once the frame's fence has passed. So when the
// CPU waits for a fence, mirrored targets rendered by work up to that fence go back to their storage (wb_wanted:
// targets over wb_limit only when not drawn every frame), unless the CPU rewrote it since. Targets drawn every frame
// still go back on an invalidate, memcpy or ReadTexels.
int vk_texture_small_dirty(void) { return __atomic_load_n(&small_pending, __ATOMIC_ACQUIRE); }
// NX2NAVI_TRACE=gfx: a dirty target the fence write-back leaves alone because it is over the limit and drawn every frame
// (what a CPU read after the wait would miss), or sends back although it is over the default limit. One line per
// target address and outcome; the table is small and a full one stops logging.
static void wb_note(const Tex *e, int skipped) {
static uint64_t seen[2][64];
static int n_seen[2];
for (int i = 0; i < n_seen[skipped]; i++)
if (seen[skipped][i] == e->pub.addr) return;
if (n_seen[skipped] == 64) return;
seen[skipped][n_seen[skipped]++] = e->pub.addr;
const pool_t *p = e->t.pool;
unsigned fl = p ? p->flags : 0;
rt_log("vk: fence write-back %s %#llx %ux%u fmt %#x size %llu pool %s (limit %llu)",
skipped ? "skipped-drawn-every-frame" : "over-default", (unsigned long long)e->pub.addr, e->t.d.width, e->t.d.height,
e->t.d.format, (unsigned long long)e->t.L.size,
!p ? "none" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_CACHED) ? "cpu-cached" : (fl & NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED) ? "cpu-uncached" : "other",
(unsigned long long)wb_limit);
}
void vk_texture_write_back_small(uint64_t fence) {
lock();
wb_cfg_init();
__atomic_store_n(&small_pending, 0, __ATOMIC_RELAXED);
plat_mutex_lock_shared(&owned_lock);
int n = 0, later = 0;
for (int i = 0; i < nowned; i++) n += owned[i]->gpu_dirty && wb_wanted(owned[i]);
Tex **hit = n ? malloc(sizeof *hit * (size_t)n) : NULL;
if (n && !hit) rt_fatal("vk: out of memory for a texture write-back");
n = 0;
for (int i = 0; i < nowned; i++) {
Tex *e = owned[i];
if (!e->gpu_dirty) continue;
if (!wb_wanted(e)) {
if (wb_trace && e->gpu_fence <= fence) {
wb_note(e, 1);
perf_count(PERF_CNT_WB_SKIP);
}
continue;
}
if (e->gpu_fence > fence) later = 1;
else {
if (wb_trace && e->t.L.size > WB_SMALL_DEFAULT) wb_note(e, 0);
hit[n++] = e;
}
}
plat_mutex_unlock_shared(&owned_lock);
if (later) __atomic_store_n(&small_pending, 1, __ATOMIC_RELEASE);
int k = 0;
for (int i = 0; i < n; i++) {
uint32_t now;
if (storage_rewritten(hit[i], hit[i]->gpu_gen, &now) == 1) hit[i]->gpu_dirty = 0;
else hit[k++] = hit[i];
}
for (int i = 1; i < k; i++)
for (int j = i; j > 0 && hit[j - 1]->gpu_seq > hit[j]->gpu_seq; j--) {
Tex *x = hit[j];
hit[j] = hit[j - 1], hit[j - 1] = x;
}
write_back_n(hit, k);
for (int i = 0; i < k; i++) perf_count(PERF_CNT_WB);
for (int i = k; i < n; i++) perf_count(PERF_CNT_WB_SKIP);
free(hit);
unlock();
}
// ---------------------------------------------------------------- samplers (LAS textures.c)
static VkCompareOp compare_op(uint32_t f) {
switch (f) {
case NVN_COMPARE_FUNC_NEVER: return VK_COMPARE_OP_NEVER;
case NVN_COMPARE_FUNC_LESS: return VK_COMPARE_OP_LESS;
case NVN_COMPARE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL;
case NVN_COMPARE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL;
case NVN_COMPARE_FUNC_GREATER: return VK_COMPARE_OP_GREATER;
case NVN_COMPARE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL;
case NVN_COMPARE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL;
default: return VK_COMPARE_OP_ALWAYS;
}
}
static struct { int init, mirror_clamp_edge, minmax, border_no_format; float aniso; } sf; // aniso: NX2NAVI_ANISO, 0 = the game's
static void sampler_features(VkCtx *c) {
if (sf.init) return;
VkPhysicalDeviceCustomBorderColorFeaturesEXT cbf = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT};
VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES,
.pNext = c->has_custom_border_color ? &cbf : NULL};
VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12};
vkGetPhysicalDeviceFeatures2(c->phys, &f2);
sf.mirror_clamp_edge = f12.samplerMirrorClampToEdge;
sf.minmax = f12.samplerFilterMinmax;
sf.border_no_format = c->has_custom_border_color && cbf.customBorderColors && cbf.customBorderColorWithoutFormat;
const char *an = cfg_value("NX2NAVI_ANISO");
sf.aniso = strcmp(an, "game") ? (float)atoi(an) : 0;
rt_log("vk: anisotropic filtering: %s", sf.aniso > 1 ? an : "the game's");
sf.init = 1;
}
static VkSampler make_sampler(VkCtx *c, const samp_par *d) {
sampler_features(c);
VkSamplerCreateInfo si = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO};
si.magFilter = d->magf == NVN_MAG_FILTER_LINEAR ? VK_FILTER_LINEAR : VK_FILTER_NEAREST;
uint32_t mf = d->minf;
si.minFilter = mf == 1 || mf == 3 || mf == 5 ? VK_FILTER_LINEAR : VK_FILTER_NEAREST;
si.mipmapMode = mf == 4 || mf == 5 ? VK_SAMPLER_MIPMAP_MODE_LINEAR : VK_SAMPLER_MIPMAP_MODE_NEAREST;
VkSamplerAddressMode modes[3];
const uint8_t wrap[3] = {d->ws, d->wt, d->wr};
int border = 0;
for (int i = 0; i < 3; i++) {
switch (wrap[i]) {
case NVN_WRAP_MODE_CLAMP: case NVN_WRAP_MODE_CLAMP_TO_EDGE: modes[i] = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; break;
case NVN_WRAP_MODE_MIRROR_CLAMP: case NVN_WRAP_MODE_MIRROR_CLAMP_TO_EDGE:
modes[i] = sf.mirror_clamp_edge ? VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE : VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
break;
case NVN_WRAP_MODE_MIRROR_CLAMP_TO_BORDER: case NVN_WRAP_MODE_CLAMP_TO_BORDER:
modes[i] = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
border = 1;
break;
case NVN_WRAP_MODE_MIRRORED_REPEAT: modes[i] = VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT; break;
default: modes[i] = VK_SAMPLER_ADDRESS_MODE_REPEAT;
}
}
si.addressModeU = modes[0], si.addressModeV = modes[1], si.addressModeW = modes[2];
si.minLod = d->lod_min;
si.maxLod = d->lod_max > 1000 ? VK_LOD_CLAMP_NONE : d->lod_max;
if (mf <= NVN_MIN_FILTER_LINEAR) si.maxLod = si.minLod + 0.25f; // not mipmapped: stay on the base level
if (si.maxLod < si.minLod) si.maxLod = si.minLod;
float lim = c->props.limits.maxSamplerLodBias;
si.mipLodBias = fmaxf(-lim, fminf(lim, d->lod_bias));
// NX2NAVI_ANISO raises the level for filtered, mipmapped colour samplers that repeat on U and V: the surface textures,
// which tile, so ground and walls seen at a shallow angle stay sharp into the distance. Depth-compare (shadow)
// samplers, nearest or unmipmapped ones (data maps, LUTs) and clamped ones keep the game's level: clamped samplers
// read screen-sized buffers, whose coordinates can stretch steeply at an object's edge, where anisotropic taps reach
// across it and leave a speckled fringe (the inventory's character preview).
float aniso = d->aniso;
// The game's own wrap modes: a mirror-clamp one maps to MIRRORED_REPEAT where the device lacks mirror-clamp-to-edge.
int tiled = (d->ws == NVN_WRAP_MODE_REPEAT || d->ws == NVN_WRAP_MODE_MIRRORED_REPEAT) &&
(d->wt == NVN_WRAP_MODE_REPEAT || d->wt == NVN_WRAP_MODE_MIRRORED_REPEAT);
if (sf.aniso > aniso && tiled && !d->cmp_mode && si.magFilter == VK_FILTER_LINEAR && si.minFilter == VK_FILTER_LINEAR &&
(mf == NVN_MIN_FILTER_LINEAR_MIPMAP_NEAREST || mf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR))
aniso = sf.aniso;
if (aniso > 1) {
si.anisotropyEnable = VK_TRUE;
si.maxAnisotropy = fminf(aniso, c->props.limits.maxSamplerAnisotropy);
}
if (d->cmp_mode) {
si.compareEnable = VK_TRUE;
si.compareOp = compare_op(d->cmp_func);
// A depth-compare sampler set to nearest returns 0 or 1 per tap, and the game's shaders smooth shadow edges by
// a few taps spaced for its own shadow map texels. With NX2NAVI_SHADOW_SCALE those taps cover a fraction of the
// area and the edges turn hard and stepped; linear filtering makes every tap a 2x2 PCF instead.
if (shadow_scale() > 1) si.magFilter = si.minFilter = VK_FILTER_LINEAR;
}
VkSamplerCustomBorderColorCreateInfoEXT cb = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT};
VkSamplerReductionModeCreateInfo rm = {.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO};
const void **tail = &si.pNext;
if (border) {
const float *b = d->border;
if (b[0] == 0 && b[1] == 0 && b[2] == 0 && b[3] == 0) si.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
else if (b[0] == 0 && b[1] == 0 && b[2] == 0 && b[3] == 1) si.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK;
else if (b[0] == 1 && b[1] == 1 && b[2] == 1 && b[3] == 1) si.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
else if (sf.border_no_format) {
memcpy(cb.customBorderColor.float32, b, 16);
si.borderColor = VK_BORDER_COLOR_FLOAT_CUSTOM_EXT;
*tail = &cb;
tail = &cb.pNext;
} else si.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
}
if (d->reduction && sf.minmax && !si.compareEnable) {
rm.reductionMode = d->reduction == NVN_SAMPLER_REDUCTION_MIN ? VK_SAMPLER_REDUCTION_MODE_MIN : VK_SAMPLER_REDUCTION_MODE_MAX;
*tail = &rm;
}
VkSampler s;
VkResult r = vkCreateSampler(c->device, &si, NULL, &s);
if (r != VK_SUCCESS) rt_fatal("vk: vkCreateSampler failed (%d)", (int)r);
return s;
}
void vk_sampler_register(int id, const samp_par *s) {
if (id < 0 || id >= (1 << 12)) rt_fatal("vk: sampler slot %d out of range", id);
lock();
if (id >= nsslots) {
int n = nsslots ? nsslots : 256;
while (n <= id) n *= 2;
SampSlot *p = realloc(sslots, sizeof *p * (size_t)n);
if (!p) rt_fatal("vk: out of memory for sampler slots");
memset(p + nsslots, 0, sizeof *p * (size_t)(n - nsslots));
sslots = p, nsslots = n;
}
sslots[id].p = *s;
sslots[id].s = sslots[id].sp = sslots[id].sc = VK_NULL_HANDLE;
sslots[id].used = 1;
samp_gen++;
unlock();
}
// DLSS upscaling (vk_sampler_dlss_bias): the scene's material textures, the filtered mipmapped ones that repeat, are
// sampled for the output's resolution; clamped ones (post effects, LUTs, the UI) keep the game's bias.
static float dlss_bias;
static int dlss_biased(const samp_par *k) {
int rep_s = k->ws == NVN_WRAP_MODE_REPEAT || k->ws == NVN_WRAP_MODE_MIRRORED_REPEAT;
int rep_t = k->wt == NVN_WRAP_MODE_REPEAT || k->wt == NVN_WRAP_MODE_MIRRORED_REPEAT;
return !k->cmp_mode && k->minf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR && (rep_s || rep_t);
}
void vk_sampler_dlss_bias(float b) {
lock();
if (b != dlss_bias) {
dlss_bias = b;
for (int i = 0; i < nsslots; i++) sslots[i].s = sslots[i].sp = sslots[i].sc = VK_NULL_HANDLE; // looked up again (the key moves)
samp_gen++;
}
unlock();
}
// Caller holds the lock.
static VkSampler sampler_for(VkCtx *c, const samp_par *s) {
samp_par k;
memset(&k, 0, sizeof k);
k = *s;
if (dlss_bias != 0 && dlss_biased(&k)) k.lod_bias += dlss_bias; // part of the key: samplers live until shutdown
VkSampler v = (VkSampler)nvn_hm_get(&samplers, &k);
if (!v) {
v = make_sampler(c, &k);
nvn_hm_put(&samplers, &k, (void *)v);
}
return v;
}
VkSampler vk_sampler_for(const samp_par *s) {
VkCtx *c = vk_ctx();
if (!c || !s) return VK_NULL_HANDLE;
lock();
VkSampler v = sampler_for(c, s);
unlock();
return v;
}
// Samplers live until shutdown, so a slot keeps the one its parameters map to until it is registered again.
VkSampler vk_sampler_by_id_locked(int id) {
VkCtx *c = vk_ctx();
if (!c) return VK_NULL_HANDLE;
VkSampler v = VK_NULL_HANDLE;
if (id >= 0 && id < nsslots && sslots[id].used) {
SampSlot *s = &sslots[id];
if (!s->s) s->s = sampler_for(c, &s->p);
v = s->s;
}
return v;
}
VkSampler vk_sampler_point_by_id_locked(int id) {
VkCtx *c = vk_ctx();
if (!c || id < 0 || id >= nsslots || !sslots[id].used) return VK_NULL_HANDLE;
SampSlot *s = &sslots[id];
if (!s->sp) {
samp_par k = s->p;
k.magf = NVN_MAG_FILTER_NEAREST;
k.minf = k.minf == NVN_MIN_FILTER_LINEAR ? NVN_MIN_FILTER_NEAREST :
k.minf > NVN_MIN_FILTER_LINEAR ? NVN_MIN_FILTER_NEAREST_MIPMAP_NEAREST : k.minf;
k.aniso = 1;
s->sp = sampler_for(c, &k);
}
return s->sp;
}
// A depth-compare read through a sampler without a compare is undefined in Vulkan. NVIDIA's driver compares anyway; AMD's
// returns the raw depth, so the game's shadow tests (which sample this way) come out wrong there. Such a read gets the
// slot's sampler with the compare on: the game's function when it set a valid one, LEQUAL otherwise, as NVIDIA's driver
// compares. NX2NAVI_SHADOW_COMPARE=0 binds the game's sampler as it is.
VkSampler vk_sampler_compare_by_id_locked(int id) {
VkCtx *c = vk_ctx();
if (!c || id < 0 || id >= nsslots || !sslots[id].used) return VK_NULL_HANDLE;
SampSlot *s = &sslots[id];
static int on = -1;
if (on < 0) on = cfg_int("NX2NAVI_SHADOW_COMPARE") != 0;
if (s->p.cmp_mode || !on) return vk_sampler_by_id_locked(id);
if (!s->sc) {
samp_par k = s->p;
k.cmp_mode = 1;
if (k.cmp_func < NVN_COMPARE_FUNC_NEVER || k.cmp_func > NVN_COMPARE_FUNC_ALWAYS) k.cmp_func = NVN_COMPARE_FUNC_LEQUAL;
static int told;
if (told++ < 8)
rt_log("vk: a depth-compare read through sampler slot %d without a compare (function %u): compared with %u", id,
(unsigned)s->p.cmp_func, (unsigned)k.cmp_func);
s->sc = sampler_for(c, &k);
}
return s->sc;
}
VkSampler vk_sampler_by_id(int id) {
lock();
VkSampler v = vk_sampler_by_id_locked(id);
unlock();
return v;
}
int vk_sampler_params(int id, samp_par *out) {
lock();
int ok = id >= 0 && id < nsslots && sslots[id].used;
if (ok) *out = sslots[id].p;
unlock();
return ok;
}
// ---------------------------------------------------------------- statistics and shutdown
void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) {
lock();
if (images) *images = by_key.n;
if (uploads) *uploads = n_uploads;
if (reuploads) *reuploads = n_reuploads;
unlock();
}
void vk_texture_census(VkTexCensus *s) {
VkCtx *c = vk_ctx();
memset(s, 0, sizeof *s);
lock();
for (uint32_t i = 0; i < by_key.cap; i++) {
const Tex *e = by_key.vals[i];
if (!e) continue;
uint64_t bytes = 0;
if (e->alloc && c) {
VmaAllocationInfo ai;
vmaGetAllocationInfo(c->vma, e->alloc, &ai);
bytes = ai.size;
}
if (e->released) s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes;
else s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes;
}
unlock();
}
// With NX2NAVI_TRACE=perf every 600 frames, after vk_memory.c's vram line: what the live entries' own memory holds. By
// kind: decoded ASTC (as BC3 or RGBA8) and other textures whose content the CPU provides, all of which a re-upload
// brings back (comes_back_intact), and the GPU-written ones (render targets, textures the GPU drew or copied into),
// whose content lives only in the image. "idle" is the part no lookup reached for IDLE_SUBMITS submissions and
// STALE_FRAMES presents (touch). Then the entries that share their storage address with another live entry (each
// descriptor of the same storage gets an image of its own), the formats holding the most, and the largest entries.
void vk_texture_census_log(uint64_t frame) {
enum { K_BC3, K_RGBA8, K_CPU, K_GPU, K_N };
static const char *const kn[K_N] = {"ASTC as BC3", "ASTC as RGBA8", "other CPU-written", "GPU-written"};
uint64_t n[K_N] = {0}, b[K_N] = {0}, in[K_N] = {0}, ib[K_N] = {0}, bound = 0, dup_n = 0, dup_b = 0;
struct { int fmt; uint64_t n, b; } fm[48];
int nfm = 0;
struct { uint64_t b; uint32_t w, h, d, levels; int fmt, gpu; } big[4] = {{0}}; // copied: read after unlock
const double M = 1048576.0;
lock();
uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
for (uint32_t i = 0; i < by_key.cap; i++) {
const Tex *e = by_key.vals[i];
if (!e || e->released) continue;
if (!e->bytes) { bound += e->bound_to != NULL; continue; }
int k = !comes_back_intact(e) ? K_GPU : e->f.astc_bw ? (e->f.astc_bc ? K_BC3 : K_RGBA8) : K_CPU;
n[k]++, b[k] += e->bytes;
if (idle(e, now)) in[k]++, ib[k] += e->bytes;
int j = 0;
while (j < nfm && fm[j].fmt != (int)e->f.vk) j++;
if (j == nfm && nfm < 48) fm[nfm++] = (typeof(fm[0])){(int)e->f.vk, 0, 0};
if (j < nfm) fm[j].n++, fm[j].b += e->bytes;
for (int q = 0; q < 4; q++)
if (e->bytes > big[q].b) {
memmove(big + q + 1, big + q, sizeof *big * (size_t)(3 - q));
big[q] = (typeof(big[0])){e->bytes, e->pub.width, e->pub.height, e->pub.layers * e->pub.depth,
e->pub.levels, (int)e->f.vk, k == K_GPU};
break;
}
}
for (uint32_t i = 0; i < by_addr.cap; i++) { // live entries with own memory at one address, beyond the largest
uint64_t cnt = 0, sum = 0, most = 0;
for (const Tex *p = by_addr.vals[i]; p; p = p->next_at)
if (!p->released && p->bytes) cnt++, sum += p->bytes, most = p->bytes > most ? p->bytes : most;
if (cnt > 1) dup_n += cnt - 1, dup_b += sum - most;
}
unlock();
char line[1024];
int o = snprintf(line, sizeof line, "mem: textures frame %llu: live with own memory by kind, idle part in brackets:",
(unsigned long long)frame);
for (int k = 0; k < K_N; k++)
o += snprintf(line + o, sizeof line - (size_t)o, " %s %llu %.0f MiB (%llu %.0f MiB);", kn[k], (unsigned long long)n[k],
b[k] / M, (unsigned long long)in[k], ib[k] / M);
snprintf(line + o, sizeof line - (size_t)o, " bound into pool memory %llu; sharing an address with another %llu %.0f MiB",
(unsigned long long)bound, (unsigned long long)dup_n, dup_b / M);
rt_log("%s", line);
for (int x = 0; x < nfm; x++) // by bytes, largest first
for (int y = x + 1; y < nfm; y++)
if (fm[y].b > fm[x].b) { typeof(fm[0]) t = fm[x]; fm[x] = fm[y], fm[y] = t; }
o = snprintf(line, sizeof line, "mem: textures frame %llu: VkFormats holding the most:", (unsigned long long)frame);
for (int x = 0; x < nfm && x < 8; x++)
o += snprintf(line + o, sizeof line - (size_t)o, " %d: %llu %.0f MiB;", fm[x].fmt, (unsigned long long)fm[x].n,
fm[x].b / M);
o += snprintf(line + o, sizeof line - (size_t)o, " largest:");
for (int q = 0; q < 4 && big[q].b; q++)
o += snprintf(line + o, sizeof line - (size_t)o, " %ux%ux%u fmt %d %u levels %.0f MiB%s;", big[q].w, big[q].h,
big[q].d, big[q].fmt, big[q].levels, big[q].b / M, big[q].gpu ? " (GPU-written)" : "");
rt_log("%s", line);
}
void vk_texture_shutdown(void) {
VkCtx *c = vk_ctx();
lock();
Tex **list = malloc(sizeof *list * (size_t)(by_key.n + 1));
int n = 0;
for (uint32_t i = 0; i < by_key.cap; i++)
if (by_key.vals[i]) list[n++] = by_key.vals[i];
for (int i = 0; i < n; i++) destroy_tex(list[i]);
free(list);
for (uint32_t i = 0; i < samplers.cap; i++)
if (samplers.vals[i] && c) vkDestroySampler(c->device, (VkSampler)samplers.vals[i], NULL);
free(by_key.keys), free(by_key.vals), free(by_addr.keys), free(by_addr.vals), free(samplers.keys), free(samplers.vals);
by_key = (nvn_hmap){.klen = sizeof(TexKey)};
by_addr = (nvn_hmap){.klen = 8};
samplers = (nvn_hmap){.klen = sizeof(samp_par)};
free(tslots), free(sslots);
tslots = NULL, sslots = NULL, ntslots = nsslots = 0, nreleased = 0;
rel_head = rel_tail = NULL, rel_bytes = 0;
if (sinfo) memset(sinfo, 0, sizeof *sinfo << 20); // no slot is registered: tokens 0
unlock();
}