From 8e656b85a44c7c0fc59a40e3f4cc114d2eaca0ee Mon Sep 17 00:00:00 2001 From: chrg6 Date: Mon, 9 Nov 2026 14:52:44 -0500 Subject: [PATCH] Hold less video memory in empty space: smaller VMA blocks, spare blocks only where textures grow, a smaller ASTC scratch, and DLSS features freed once unused, with a VRAM breakdown in the perf log. VMA's blocks on large heaps are 64 MiB instead of 256 (vma_block_mb): one session held 2.1 GiB free inside the device blocks beside 8.2 GiB of allocations after a few area changes, and the driver counts a block whole until it empties. The headroom thread keeps its spare block only for the device-local types textures and mirrors grow in; the spare blocks it made for the host types and the PCIe BAR type at boot stayed empty for the whole run. The ASTC decoder's scratch is 32 MiB, so it stays inside a block, and an area load that makes one per upload batch holds half as much at once. A DLSS feature no frame used for 3600 evaluations is freed with its work images, and a model change now frees all five sizes. The perf log gets a "vram by use" line (textures, released textures, mirrors, physical pools, ASTC scratch, headroom spare, free space inside the blocks, memory outside VMA and DLSS's own share), the free ranges per memory type, the addresses whose duplicate texture entries hold the most, and a line when memory outside VMA moves by 256 MiB. astc_bc_probe in nx2navi.ini turns on the ASTC opacity probe. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 99cfc2fa14ae208b7b23e057ff9b87986cd6a5e0) --- nx2navi.ini.example | 7 ++++ runtime/cfg.c | 4 +++ runtime/vk.h | 3 ++ runtime/vk_astc.c | 18 ++++++++-- runtime/vk_device.c | 1 + runtime/vk_dlaa.c | 45 +++++++++++++++++++++++-- runtime/vk_dlaa.h | 2 ++ runtime/vk_headroom.c | 62 ++++++++++++++++++++++++++++++---- runtime/vk_memory.c | 77 +++++++++++++++++++++++++++++++++++++++++-- runtime/vk_texture.c | 43 +++++++++++++++++++++--- 10 files changed, 243 insertions(+), 19 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 09933df0..de38b485 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -522,6 +522,13 @@ # Each walk holds the address-space lock for 100-240 ms and delays the next frame by as much, so they are off unless # asked for. # mem_census = 0 +# The size in MiB (16 to 256) of the blocks the renderer takes video memory in. A block goes back to the driver only once +# everything in it is freed, so smaller blocks hold less free space after area changes; larger ones need fewer +# allocations while an area loads. Default: 64. +# vma_block_mb = 64 +# 1: samples each new ASTC texture's blocks on the CPU and logs how many are opaque ("ASTC alpha probe" lines), which +# could be stored at half the video memory. Slows area loads. Default: 0. +# astc_bc_probe = 0 # Flight recorder: keeps the last frames and saves them to a flight folder next to the folder of nx2navi.exe when a frame # takes longer than flight_spike_ms (0 to 60000). # flight = 0 diff --git a/runtime/cfg.c b/runtime/cfg.c index f2b0ac92..5981ee02 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -67,6 +67,7 @@ static const char *const known[] = { "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", + "VMA_BLOCK_MB", "ASTC_BC_PROBE", }; // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). @@ -119,6 +120,7 @@ static const struct { const char *key, *section; } sections[] = { {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, + {"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"}, }; // The settings the menu shows: the value in effect when the key is unset, and what the key accepts. The only place a @@ -231,6 +233,8 @@ static const CfgMeta metas[] = { {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, {"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART}, {"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame + {"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size + {"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha {"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only {"INPUT_RECORD", "", NULL, RESTART}, }; diff --git a/runtime/vk.h b/runtime/vk.h index 47035fdb..2218b8f2 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -336,6 +336,8 @@ int vk_mem_vram(uint64_t *used, uint64_t *budget, int *exact); const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void); void vk_headroom_start(void); void vk_headroom_stop(void); // before vmaDestroyAllocator +uint64_t vk_headroom_held(void); // bytes the spare blocks' anchors hold +VkDeviceSize vk_vma_block_size(void); // VMA's block size on heaps over 1 GiB (NX2NAVI_VMA_BLOCK_MB) int vk_mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out); // 1 when all of [addr, addr+size) is backed // Changes whenever vk_mem_resolve plus VK_MEM_USE may give another span for an address: a pool added, finalized or // mapped, a mirror created, refused or forgotten, a submission started. @@ -403,6 +405,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback); VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh); // source bytes the region reads void vk_astc_stats(uint64_t *dispatches, uint64_t *texels); // since start +uint64_t vk_astc_scratch_bytes(void); // the decoder's shared scratch now // NX2NAVI_ASTC_TIME=1: GPU timestamps around each vk_astc_record's dispatches (logged per footprint and the largest one) void vk_astc_time_enable(int on); int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls, double *us); // totals so far diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index a19a276f..daa4e5b9 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -10,7 +10,7 @@ // device cannot. cpu: astc_decode_ldr_mt on the uploading thread, as before. check: gpu, and every upload is also // decoded on the CPU and compared (waits for the GPU; for testing only). // -// The decode is recorded into the upload batch (vk_memory.c) in chunks of at most 32 MiB of texels: a chunk's raw +// The decode is recorded into the upload batch (vk_memory.c) in chunks of at most 16 MiB of texels: a chunk's raw // blocks go from staging to a device-local scratch buffer, its dispatches write RGBA8 texels behind them, and a // buffer-to-image copy puts them into the image; the next chunk reuses the scratch after a barrier. The next texture // reuses it too, behind the same barrier, while it records into the same command buffer (scratch_get): a buffer per @@ -430,8 +430,12 @@ static void mem_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessF // Scratch per vk_astc_record: texels of at most this many bytes at a time (or one region, when larger), so a large // texture array (169 M texels, 677 MB as RGBA8, on monsters2) does not need its whole RGBA8 copy as scratch. -#define CHUNK_OUT (32ull << 20) -#define SCRATCH_MIN (64ull << 20) // the shared scratch's least size: one chunk's texels plus its blocks and BC3 +// The scratch is made per upload batch and kept until that batch completes (scratch_keep), and an area load records +// many batches in one frame: 12 scratches in the frame that uploaded 2617 textures entering an area, all alive at once. +// So each is kept to half a VMA block (vk_vma_block_size's default 64 MiB), where VMA still places it in a block, not in +// memory of its own: 16 MiB of texels, their blocks and BC3 fit in 32 MiB. +#define CHUNK_OUT (16ull << 20) +#define SCRATCH_MIN (32ull << 20) // the shared scratch's least size: one chunk's texels plus its blocks and BC3 // The shared scratch and the command buffer it was last recorded into. An upload batch's command buffer is used again // only after that batch completed (vk_memory.c: completed batches go to the spare list), so the same handle means the @@ -497,6 +501,14 @@ static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int return out; } +// The shared scratch's size, for the log's VRAM breakdown (scratches still owned by an upload batch not counted). +uint64_t vk_astc_scratch_bytes(void) { + plat_mutex_lock(&g_lock); + uint64_t n = S.buf ? S.size : 0; + plat_mutex_unlock(&g_lock); + return n; +} + static VkDeviceSize region_out(const VkAstcRegion *r) { return (VkDeviceSize)r->w * r->h * r->slices * 4; } static VkDeviceSize tight_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { return (VkDeviceSize)((r->w + bw - 1) / bw) * ((r->h + bh - 1) / bh) * r->slices * 16; diff --git a/runtime/vk_device.c b/runtime/vk_device.c index e1398278..8926bd69 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -446,6 +446,7 @@ static int create_device(void) { .vulkanApiVersion = VK_API_VERSION_1_3, .flags = (g_vk.has_buffer_device_address ? VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT : 0) | (g_vk.has_memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0), + .preferredLargeHeapBlockSize = vk_vma_block_size(), .pDeviceMemoryCallbacks = vk_headroom_callbacks()}; if ((r = vmaCreateAllocator(&ai, &g_vk.vma)) != VK_SUCCESS) { rt_log("vk: vmaCreateAllocator failed (%d)", (int)r); return 0; } diff --git a/runtime/vk_dlaa.c b/runtime/vk_dlaa.c index 04f6ac09..64b0e18e 100644 --- a/runtime/vk_dlaa.c +++ b/runtime/vk_dlaa.c @@ -72,6 +72,7 @@ static struct { PFN_Release release; int tried, loaded, available; int ngx_up; // vk_dlaa_init brought NGX up: DLSS serves this run + NgxParam *capsp; // the capability parameters (vk_dlaa_init), which DLSS's statistics callback reads and fills int wanted; // -1 not read yet } N = {.wanted = -1}; @@ -83,6 +84,7 @@ typedef NgxResult (*GetI)(const NgxParam *, const char *, int *); typedef NgxResult (*GetU)(const NgxParam *, const char *, unsigned *); typedef NgxResult (*GetF)(const NgxParam *, const char *, float *); typedef NgxResult (*GetP)(const NgxParam *, const char *, void **); +typedef NgxResult (*GetU64)(const NgxParam *, const char *, unsigned long long *); // The SDK declares Set(u64), Set(float), Set(double), Set(u32), Set(int), Set(ID3D11Resource *), Set(ID3D12Resource *), // Set(void *), then the Get overloads in the same order, then Reset. #ifdef _WIN32 @@ -98,6 +100,9 @@ static int p_geti(const NgxParam *p, const char *n, int *v) { return NGX_OK(((Ge static int p_getu(const NgxParam *p, const char *n, unsigned *v) { return NGX_OK(((GetU)p->vt->slot[G_U32])(p, n, v)); } static int p_getf(const NgxParam *p, const char *n, float *v) { return NGX_OK(((GetF)p->vt->slot[G_F])(p, n, v)); } static int p_getp(const NgxParam *p, const char *n, void **v) { return NGX_OK(((GetP)p->vt->slot[G_P])(p, n, v)); } +static int p_getull(const NgxParam *p, const char *n, unsigned long long *v) { + return NGX_OK(((GetU64)p->vt->slot[G_U64])(p, n, v)); +} #ifdef _WIN32 #define DLSS_MODEL "nvngx_dlss.dll" @@ -279,6 +284,7 @@ void vk_dlaa_init(void) { p_geti(caps, "SuperSampling.NeedsUpdatedDriver", &driver); } N.available = avail != 0; + N.capsp = caps; N.ngx_up = 1; rt_log("dlaa: NGX up; DLSS %s%s", avail ? "available" : "NOT available (is " DLSS_MODEL " next to the executable?)", driver ? "; the driver needs an update" : ""); @@ -309,6 +315,10 @@ int vk_dlaa_available(void) { return vk_dlaa_wanted() && N.available; } // (vk_dlaa_swap_img, applied by vk_exec_draw.c tex_info): whichever shader scales, it scales DLSS's picture. #define OCAM_BLOCK 0x3500u // Ocarina's scene context block (vk_dlaa_draw) #define OCAM_BYTES 0xb0u // its rows 0-10: the view and the projection +#define DLSS_SIZES 5 // DLSS features kept, one per scene and output size (D.sizes) +// A size no evaluation used for this many goes with its feature (stale_sizes): DLSS's history buffers and the work +// images at the output size held their VRAM after a switch of the DLSS mode, until four more sizes had come and gone. +#define SIZE_STALE 3600 static struct { VkExecTarget color, depth; // the armed pair, valid in the frame it was armed int armed, done, gbuf; // this frame: armed on the light target, DLAA ran, the G-buffer was drawn @@ -328,7 +338,7 @@ static struct { int dist_cur, dist_ok; // as chars_cur and chars_ok NgxHandle *feat; uint64_t used; // evals count when it last ran - } sizes[5], *last; // last: the one the previous DLAA frame ran with + } sizes[DLSS_SIZES], *last; // last: the one the previous DLAA frame ran with VkImage last_color; // the target DLAA ran on last frame: another one means a new history NgxParam *params; int failed, reset; @@ -487,7 +497,7 @@ static void free_size(struct Size *e) { // goes for a new one. static struct Size *size_for(VkCommandBuffer cb, uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) { struct Size *e = NULL, *lru = &D.sizes[0]; - for (int i = 0; i < 5; i++) { + for (int i = 0; i < DLSS_SIZES; i++) { if (D.sizes[i].w == w && D.sizes[i].h == h && D.sizes[i].ow == ow && D.sizes[i].oh == oh) return &D.sizes[i]; if (!D.sizes[i].out && !e) e = &D.sizes[i]; if (D.sizes[i].used < lru->used) lru = &D.sizes[i]; @@ -1282,6 +1292,33 @@ static void target_changed(const VkExecTarget *t) { D.reset = 1, D.why[2]++, D.last_color = t->image; } +// DLSS's own video memory, the features and their history (NGX's DLSS statistics callback with the capability +// parameters), for the log's VRAM breakdown (vk_dlaa_vram); -1 until known. Asked on the replay thread, which makes every +// other NGX call, every 600 evaluations. +static volatile int64_t dlss_vram = -1; +int64_t vk_dlaa_vram(void) { return __atomic_load_n(&dlss_vram, __ATOMIC_RELAXED); } + +static void query_vram(void) { + void *cb = NULL; + unsigned long long n = 0; + if (fsr3() || !N.capsp || !p_getp(N.capsp, "DLSSGetStatsCallback", &cb) || !cb) return; + if (NGX_OK(((NgxResult (*)(NgxParam *))cb)(N.capsp)) && p_getull(N.capsp, "SizeInBytes", &n)) + __atomic_store_n(&dlss_vram, (int64_t)n, __ATOMIC_RELAXED); +} + +// Frees the sizes other than cur that no evaluation used for SIZE_STALE evaluations: the GPU finished with them long +// ago (the CPU is never that many presents ahead), so their images and feature go at once. +static void stale_sizes(const struct Size *cur) { + for (int i = 0; i < DLSS_SIZES; i++) { + struct Size *s = &D.sizes[i]; + if (s == cur || !s->out || D.evals < s->used + SIZE_STALE) continue; + rt_log("dlaa: the DLSS feature at %ux%u -> %ux%u went unused for %d frames: freed", s->w, s->h, s->ow, s->oh, + SIZE_STALE); + if (D.last == s) D.last = NULL; + free_size(s); + } +} + static void evaluate(VkExec *ex, const VkExecTarget *dst) { const VkExecTarget *t = &D.color, *z = &D.depth; if (!dst) dst = t; @@ -1289,7 +1326,7 @@ static void evaluate(VkExec *ex, const VkExecTarget *dst) { VkCommandBuffer cb = ex->cb; vk_exec_barrier(ex); // what drew the scene is done (no pass is open here) if (D.preset_dirty) { // another model: features are created with it, so every size gets a new one - for (int i = 0; i < 3; i++) + for (int i = 0; i < DLSS_SIZES; i++) if (D.sizes[i].out) free_size(&D.sizes[i]); D.last = NULL, D.preset_dirty = 0; } @@ -1404,6 +1441,8 @@ evaluated: if (timed) times_mark(cb, T_DL1); // through the recorder, after NGX's own commands D.reset = 0; D.last = e, e->used = D.evals + 1; + if (D.evals % 600 == 0) query_vram(); + stale_sizes(e); full_barrier(cb); vk_mvcheck_output(cb, e->out, ow, oh); // NX2NAVI_MV_CHECK (test) // the result into the scene target (DLAA: the bloom, the post passes and the HUD read it and draw over it) or into diff --git a/runtime/vk_dlaa.h b/runtime/vk_dlaa.h index b225190b..6c233a42 100644 --- a/runtime/vk_dlaa.h +++ b/runtime/vk_dlaa.h @@ -35,6 +35,8 @@ void vk_dlaa_device_exts(const VkExtensionProperties *have, uint32_t nhave, cons void vk_dlaa_init(void); // 1 when dlaa is on and NGX came up with DLSS (after vk_dlaa_init): a DLSS mode's smaller scene is scaled up by DLSS. int vk_dlaa_available(void); +// The video memory DLSS reports for its features (NGX's statistics), for the log; -1 while unknown. +int64_t vk_dlaa_vram(void); // vk_exec.c vk_exec_pass_begin, once the pass's targets are resolved and before its barrier: arms the frame on the // scene's light target or, at the first quarter-width pass after it, runs DLAA on that target. Each present ends the // frame (vk_dlaa_frame). diff --git a/runtime/vk_headroom.c b/runtime/vk_headroom.c index cf09a823..7bd24714 100644 --- a/runtime/vk_headroom.c +++ b/runtime/vk_headroom.c @@ -1,8 +1,9 @@ // VRAM headroom: the next VMA block of a memory type is allocated on a background thread, not on the render thread. // -// VMA places an allocation in an existing block of its memory type and allocates a new block (vkAllocateMemory, 32 to -// 256 MiB) only when none has room. On the render thread that took 8-19 ms per block, several times in the first -// minute of play while VRAM grows (textures from create_tex, mirrors). A new VkDeviceMemory made by any other thread +// VMA places an allocation in an existing block of its memory type and allocates a new block (vkAllocateMemory, an +// eighth of the default block size up to all of it, vk_vma_block_size) only when none has room. On the render thread +// that took 8-19 ms per 256 MiB block, several times in the first minute of play while VRAM grows (textures from +// create_tex, mirrors). A new VkDeviceMemory made by any other thread // (VMA's pfnAllocate callback) wakes this thread for its memory type, which it then checks every 500 ms. A check // allocates half a default block of that type (the largest size VMA still places in blocks, not in dedicated memory). // When a block has that much room the allocation lands there and is freed at once. When none has, VMA allocates a new @@ -10,6 +11,12 @@ // type (VmaBlockVector::Free, "we have another empty block"), so a spare block left empty lasted until the next // texture went. The previous anchor is freed then, which gives its half block to the render thread, so at most half // a block per memory type is held back. +// Only the memory types textures and VRAM mirrors grow in get a spare (spare_type): the device-local types the CPU +// cannot map, or on a GPU that has none of those (integrated, every type mappable), the device-local ones. In one +// session the others, the host heap's two types (system memory, charged to the commit) and the device-local mappable +// one (VRAM behind the PCIe BAR), each got a 256 MiB spare early on and never needed another; to the end the BAR type +// held 320 MiB of blocks for 37 MiB of allocations and one host type 288 MiB for 0.1 MiB, as VMA keeps an empty block +// until the next free of its type, and these types seldom free anything. // Under VRAM pressure no headroom is kept: the spare block counts against the budget that decides whether a pool gets // its VRAM mirror (vk_memory.c), so while use plus two blocks passes 85% of the budget the anchor goes and nothing new // is allocated. Nor while the system's commit is low (vk_texture_commit_low): the driver charges device memory to it, @@ -34,8 +41,43 @@ static struct { volatile int64_t made; // blocks this thread brought in volatile int64_t other; // device memory of 32 MiB or more other threads allocated (VMA blocks, large dedicated) VmaAllocation anchor[64]; // per memory type: the half block that keeps the newest spare block from being deleted + volatile int64_t held; // bytes of the anchors, for the log's VRAM breakdown (vk_headroom_held) } H; +// NX2NAVI_VMA_BLOCK_MB (16 to 256, default 64): the size of VMA's blocks on heaps over 1 GiB (its +// preferredLargeHeapBlockSize; VMA's own default is 256 MiB). A freed texture or mirror leaves a hole its block keeps, +// and the driver counts the whole block against the VRAM budget and the system's commit until its last allocation +// goes. With 256 MiB blocks one session held 2.1 GiB free inside them beside 8.2 GiB of allocations after a few area +// changes, every block pinned by a texture or mirror that outlived its neighbours; a smaller block empties, and goes +// back to the driver, once fewer allocations have gone. Allocations over half a block get memory of their own (VMA), +// which matches the 32 MiB from which create_tex gives images their own memory. +VkDeviceSize vk_vma_block_size(void) { + static VkDeviceSize size; + if (!size) { + int mb = cfg_int("NX2NAVI_VMA_BLOCK_MB"); + mb = mb < 16 ? 16 : mb > 256 ? 256 : mb; + size = (VkDeviceSize)mb << 20; + rt_log("vk: VMA blocks of %d MiB on heaps over 1 GiB (NX2NAVI_VMA_BLOCK_MB)", mb); + } + return size; +} + +// The memory types a spare block is kept for (see the top): device-local without host access, or device-local at all +// where the device has no such type. +static int spare_type(const VkCtx *c, uint32_t type) { + const VkPhysicalDeviceMemoryProperties *m = &c->mem_props; + VkMemoryPropertyFlags f = m->memoryTypes[type].propertyFlags; + if (!(f & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) return 0; + if (!(f & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) return 1; + for (uint32_t t = 0; t < m->memoryTypeCount; t++) + if ((m->memoryTypes[t].propertyFlags & (VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) == + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) + return 0; + return 1; +} + +uint64_t vk_headroom_held(void) { return (uint64_t)__atomic_load_n(&H.held, __ATOMIC_RELAXED); } + static void VKAPI_PTR on_allocate(VmaAllocator a, uint32_t type, VkDeviceMemory m, VkDeviceSize size, void *user) { (void)a, (void)m, (void)size, (void)user; if (!H.on || type >= 64) return; @@ -60,7 +102,7 @@ const VmaDeviceMemoryCallbacks *vk_headroom_callbacks(void) { static VkDeviceSize block_size(const VkCtx *c, uint32_t type) { VkDeviceSize heap = c->mem_props.memoryHeaps[c->mem_props.memoryTypes[type].heapIndex].size; - return heap <= HR_SMALL_HEAP ? heap / 8 : 256ull << 20; // VMA's default preferredLargeHeapBlockSize + return heap <= HR_SMALL_HEAP ? heap / 8 : vk_vma_block_size(); // VMA's preferredLargeHeapBlockSize (vk_device.c) } static int room(const VkCtx *c, uint32_t type, VkDeviceSize block) { @@ -72,7 +114,12 @@ static int room(const VkCtx *c, uint32_t type, VkDeviceSize block) { } static void drop_anchor(const VkCtx *c, uint32_t type) { - if (H.anchor[type]) vmaFreeMemory(c->vma, H.anchor[type]); + if (H.anchor[type]) { + VmaAllocationInfo ai; + vmaGetAllocationInfo(c->vma, H.anchor[type], &ai); + plat_add64(&H.held, -(int64_t)ai.size); + vmaFreeMemory(c->vma, H.anchor[type]); + } H.anchor[type] = NULL; } @@ -93,6 +140,9 @@ static void top_up(const VkCtx *c, uint32_t type) { } drop_anchor(c, type); H.anchor[type] = a; + VmaAllocationInfo info; + vmaGetAllocationInfo(c->vma, a, &info); + plat_add64(&H.held, (int64_t)info.size); static volatile int32_t logged; if (plat_inc32(&logged) <= 32) rt_log("vk: headroom: a %.0f MiB block of memory type %u allocated ahead (%lld so far; %lld allocations of 32 MiB " @@ -111,7 +161,7 @@ static void headroom_main(void *arg) { plat_mutex_lock(&H.lock); const VkCtx *c = H.stop ? NULL : vk_ctx(); for (uint32_t t = 0; c && t < c->mem_props.memoryTypeCount && t < 64; t++) - if (watch >> t & 1) top_up(c, t); + if ((watch >> t & 1) && spare_type(c, t)) top_up(c, t); int stop = H.stop; if (stop && (c = vk_ctx())) for (uint32_t t = 0; t < 64; t++) drop_anchor(c, t); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 5fa1c921..effe0686 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -18,6 +18,7 @@ #include "nvn_api.h" #include "nvn_pool_index.h" #include "vk.h" +#include "vk_dlaa.h" #include "vk_exec.h" #include "vk_gpa.h" #include "wb_merge_spv.h" @@ -576,6 +577,7 @@ static void shadow_log(uint64_t frame); static int dead_pending(void); static void vram_log(uint64_t frame); +static void outside_watch(uint64_t frame); static volatile uint64_t mirror_frame; // the frame vk_mem_census last saw, for mirror_sync's log lines @@ -583,6 +585,7 @@ void vk_mem_census(uint64_t frame) { mirror_frame = frame; static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); + if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame), shadow_log(frame); // mirror_log also logs the write-watch line @@ -2528,9 +2531,11 @@ static void heap_log(VkCtx *c, uint64_t frame, const VmaBudget *vb) { for (uint32_t t = 0; t < c->mem_props.memoryTypeCount && t < VK_MAX_MEMORY_TYPES && o < (int)sizeof line - 120; t++) { const VmaDetailedStatistics *d = &ts.memoryType[t]; if (!d->statistics.blockCount) continue; - o += snprintf(line + o, sizeof line - (size_t)o, " type %u: %u blocks %.0f MiB, %u allocations %.0f MiB, largest %.1f MiB;", t, - d->statistics.blockCount, d->statistics.blockBytes / M, d->statistics.allocationCount, - d->statistics.allocationBytes / M, d->allocationSizeMax / M); + o += snprintf(line + o, sizeof line - (size_t)o, + " type %u: %u blocks %.0f MiB, %u allocations %.0f MiB, largest %.1f MiB, %u free ranges (largest %.1f MiB);", + t, d->statistics.blockCount, d->statistics.blockBytes / M, d->statistics.allocationCount, + d->statistics.allocationBytes / M, d->allocationSizeMax / M, d->unusedRangeCount, + d->unusedRangeCount ? d->unusedRangeSizeMax / M : 0.0); } rt_log("%s", line); // The address-space walk of the census (plat_mem_census, 100-240 ms of VirtualQuery) runs on its own thread, but @@ -2569,6 +2574,71 @@ static void pool_census_log(uint64_t frame) { (unsigned long long)nn, nb / M, (unsigned long long)mn, mb / M, MIRROR_IDLE, (unsigned long long)in, ib / M); } +// After the heap lines: the driver's figure for the device-local heaps split by what holds it. VMA's blocks hold the +// textures with their own memory (live and released), the VRAM mirrors, the physical pools, the ASTC decoder's scratch, +// the headroom's spare blocks (their anchors), everything else VMA placed, and the free space inside the blocks, which +// the driver counts as used until a block empties. The rest of the driver's figure was never allocated through VMA: +// the swapchain, DLSS and frame generation (Streamline), compiled pipelines and the driver's own memory; DLSS's share +// as its statistics report it, when known. +static void vram_use_log(VkCtx *c, uint64_t frame, const VmaBudget *vb, const VkTexCensus *t) { + uint64_t usage = 0, blocks = 0, allocs = 0, phys = 0; + VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + if (c->has_memory_budget) { + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + } + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + usage += c->has_memory_budget ? bp.heapUsage[h] : vb[h].statistics.blockBytes, + blocks += vb[h].statistics.blockBytes, allocs += vb[h].statistics.allocationBytes; + plat_mutex_lock_shared(&pool_lock); + for (int i = 0; i < nphys; i++) + if (phys_list[i]->alloc) phys += phys_list[i]->size; + plat_mutex_unlock_shared(&pool_lock); + uint64_t mir = (uint64_t)__atomic_load_n(&m_live_bytes, __ATOMIC_RELAXED), held = vk_headroom_held(), + scratch = vk_astc_scratch_bytes(); + uint64_t known = t->bytes_live + t->bytes_released + mir + phys + held + scratch; + int64_t dlss = vk_dlaa_vram(); + char ds[48] = "n/a"; + if (dlss >= 0) snprintf(ds, sizeof ds, "%.0f MiB", dlss / 1048576.0); + const double M = 1048576.0; + rt_log("mem: vram by use frame %llu: in use %.0f MiB = VMA blocks %.0f MiB (textures %.0f, released textures %.0f, " + "VRAM mirrors %.0f, physical pools %.0f, ASTC scratch %.0f, headroom spare %.0f, other %.0f, free inside the " + "blocks %.0f) + outside VMA %.0f MiB (swapchain, DLSS and frame generation, pipelines, driver; DLSS reports %s)", + (unsigned long long)frame, usage / M, blocks / M, t->bytes_live / M, t->bytes_released / M, mir / M, phys / M, + scratch / M, held / M, allocs > known ? (allocs - known) / M : 0.0, blocks > allocs ? (blocks - allocs) / M : 0.0, + usage > blocks ? (usage - blocks) / M : 0.0, ds); +} + +// With NX2NAVI_TRACE=perf every 60 frames: a line when the device-local memory VMA did not allocate (vram_use_log's +// "outside VMA") moved by OUTSIDE_STEP or more since the last line, so the log places it beside what did it. In one +// session it grew from 0.6 to 2.6 GiB at the title screen, in two steps of about 1 GiB while the boot's compute +// pipelines were being built (vk_pipeline.c, stage library prebuild), and DLSS and frame generation add theirs when +// they start. +#define OUTSIDE_STEP (256ull << 20) +static void outside_watch(uint64_t frame) { + VkCtx *c = vk_ctx(); + if (!c || !c->has_memory_budget) return; + VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + uint64_t usage = 0, blocks = 0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + usage += bp.heapUsage[h], blocks += vb[h].statistics.blockBytes; + uint64_t out = usage > blocks ? usage - blocks : 0; + static uint64_t last; + static int seen; + if (seen && (out > last ? out - last : last - out) < OUTSIDE_STEP) return; + const double M = 1048576.0; + rt_log("mem: frame %llu: device memory outside VMA (swapchain, DLSS and frame generation, pipelines, driver) %s%.0f " + "MiB (%s%.0f MiB)", (unsigned long long)frame, seen ? "now " : "", out / M, out >= last ? "+" : "-", + (out >= last ? out - last : last - out) / M); + last = out, seen = 1; +} + // With NX2NAVI_TRACE=perf every 600 frames: what holds device-local memory, to see what grows over a long session. // VMA's blocks and allocations on the device-local heaps, texture entries with their own memory, the VRAM mirrors and // the objects waiting for their last submission to retire. @@ -2594,6 +2664,7 @@ static void vram_log(uint64_t frame) { vk_texture_census_log(frame); pool_census_log(frame); heap_log(c, frame, vb); + vram_use_log(c, frame, vb, &t); } // Device class: INTEGRATED -> uma; a discrete GPU with a DEVICE_LOCAL heap of 1 GiB or more beside a diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index c00d0e3d..1696c7b9 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -483,8 +483,9 @@ static int shadow_scale_of(const nvn_texdesc *d, const TexVkFormat *f, uint32_t return d->width * (uint32_t)scale <= most ? scale : 1; } -// NX2NAVI_ASTC_BC_PROBE=1: measures, without changing any format, how much of the ASTC set could be compressed if the -// alpha rule were data-driven instead of footprint-driven. Each texture's source blocks are decoded on the CPU with +// NX2NAVI_ASTC_BC_PROBE=1 (astc_bc_probe in nx2navi.ini): measures, without changing any format, how much of the ASTC +// set could be compressed if the alpha rule were data-driven instead of footprint-driven: the opaque part could be BC1, +// half the size of the BC3 every decoded ASTC texture is now. Each texture's source blocks are decoded on the CPU with // tex_astc.c astc_decode_block (up to 512 spread over the level) and counted by whether any block has alpha below 250. #define ASTC_BC_PROBES 512 static uint64_t n_probe_calls, n_probe_alpha, n_probe_opaque, n_probe_bytes_alpha, n_probe_bytes_opaque; @@ -530,9 +531,9 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { log_limited(&n_unsupported, "texture format %#x: VkFormat %d lacks features %#x", d->format, (int)f.vk, (unsigned)f.need); return NULL; } - // NX2NAVI_ASTC_BC_PROBE (environment; diagnostics only): measure what a data-driven alpha rule would compress + // NX2NAVI_ASTC_BC_PROBE (diagnostics only): measure what a data-driven alpha rule would compress static int bc_probe = -1; // read once: this runs for every texture created - if (bc_probe < 0) bc_probe = getenv("NX2NAVI_ASTC_BC_PROBE") != NULL; + if (bc_probe < 0) bc_probe = cfg_bool("NX2NAVI_ASTC_BC_PROBE"); if (f.astc_bw && bc_probe) astc_probe_alpha(t, &f); nvn_texlayout L; nvn_tex_layout(d, &L); @@ -3025,6 +3026,8 @@ void vk_texture_census_log(uint64_t frame) { struct { int fmt; uint64_t n, b; } fm[48]; int nfm = 0; struct { uint64_t b; uint32_t w, h, d, levels; int fmt, gpu; } big[4] = {{0}}; // copied: read after unlock + // the addresses whose extra entries hold the most, with their largest few entries (copied: read after unlock) + struct { uint64_t addr, cnt, extra; struct { uint64_t b; uint32_t w, h, d; int fmt, gpu; } e[4]; } dup[3] = {{0}}; const double M = 1048576.0; lock(); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); @@ -3052,6 +3055,25 @@ void vk_texture_census_log(uint64_t frame) { for (const Tex *p = by_addr.vals[i]; p; p = p->next_at) if (!p->released && p->bytes) cnt++, sum += p->bytes, most = p->bytes > most ? p->bytes : most; if (cnt > 1) dup_n += cnt - 1, dup_b += sum - most; + for (int q = 0; cnt > 1 && q < 3; q++) { + if (sum - most <= dup[q].extra) continue; + memmove(dup + q + 1, dup + q, sizeof *dup * (size_t)(2 - q)); + memset(&dup[q], 0, sizeof dup[q]); + dup[q].cnt = cnt, dup[q].extra = sum - most; + for (const Tex *p = by_addr.vals[i]; p; p = p->next_at) { + if (p->released || !p->bytes) continue; + dup[q].addr = p->pub.addr; + for (int x = 0; x < 4; x++) + if (p->bytes > dup[q].e[x].b) { + memmove(dup[q].e + x + 1, dup[q].e + x, sizeof dup[q].e[0] * (size_t)(3 - x)); + dup[q].e[x].b = p->bytes, dup[q].e[x].w = p->pub.width, dup[q].e[x].h = p->pub.height; + dup[q].e[x].d = p->pub.layers * p->pub.depth, dup[q].e[x].fmt = (int)p->f.vk; + dup[q].e[x].gpu = !comes_back_intact(p); + break; + } + } + break; + } } unlock(); char line[1024]; @@ -3075,6 +3097,19 @@ void vk_texture_census_log(uint64_t frame) { o += snprintf(line + o, sizeof line - (size_t)o, " %ux%ux%u fmt %d %u levels %.0f MiB%s;", big[q].w, big[q].h, big[q].d, big[q].fmt, big[q].levels, big[q].b / M, big[q].gpu ? " (GPU-written)" : ""); rt_log("%s", line); + if (!dup[0].cnt) return; + // Which entries share an address: each descriptor of the same storage (another size, format or level count) gets + // an image of its own, and its memory beside the others'. + o = snprintf(line, sizeof line, "mem: textures frame %llu: addresses with the most extra entries:", (unsigned long long)frame); + for (int q = 0; q < 3 && dup[q].cnt && o < (int)sizeof line - 200; q++) { + o += snprintf(line + o, sizeof line - (size_t)o, " %#llx: %llu entries, %.0f MiB beyond the largest (", + (unsigned long long)dup[q].addr, (unsigned long long)dup[q].cnt, dup[q].extra / M); + for (int x = 0; x < 4 && dup[q].e[x].b && o < (int)sizeof line - 80; x++) + o += snprintf(line + o, sizeof line - (size_t)o, "%s%ux%ux%u fmt %d %.1f MiB%s", x ? ", " : "", dup[q].e[x].w, + dup[q].e[x].h, dup[q].e[x].d, dup[q].e[x].fmt, dup[q].e[x].b / M, dup[q].e[x].gpu ? " GPU-written" : ""); + o += snprintf(line + o, sizeof line - (size_t)o, ");"); + } + rt_log("%s", line); } void vk_texture_shutdown(void) {