From 2820781fb702d1231224b4688b9e4de80d7d11b5 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Thu, 12 Nov 2026 23:10:09 -0500 Subject: [PATCH] Log the memory by owner every 3600 frames, and how often released textures come back. Every VMA allocation is counted under the file that made it until it is freed, so the new "mem: by owner" line splits the device-local memory the driver reports into textures (live, GPU-drawn, idle, released, released too lately to drop), pools and VRAM mirrors, upscalers, replay buffers, uniform snapshots, the device address table, the ASTC decoder, present and post-processing, headroom spares, the free space inside VMA's blocks and what VMA never allocated. "mem: host" gives the host-visible allocations, the process's commit and working set and the game's heap. "vk: released textures" counts take-backs by how long the texture was released and textures made again after a drop by a bound or a pressure, with the drop's reason. Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 3 ++ runtime/memory.c | 6 +++ runtime/vk.h | 35 ++++++++++---- runtime/vk_memory.c | 85 +++++++++++++++++++++++++++++++++- runtime/vk_texture.c | 96 ++++++++++++++++++++++++++++++++++---- runtime/vk_vma_budget.c | 100 ++++++++++++++++++++++++++++++++++++---- tests/gpa_check.c | 7 +-- 7 files changed, 302 insertions(+), 30 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index 275dffc2..33ca3fe3 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -196,6 +196,9 @@ int mem_start_note(int probe); // a game's start: a // below the game's need (mem_need.h); 1 when it is void mem_guest_census(uint64_t frame); // NX2NAVI_TRACE=perf: committed bytes per guest region void mem_lazy_heap_report(uint64_t frame); // NX2NAVI_TRACE=perf: the lazy heap's faults since the last line +// The game's own memory for the memory report (vk_memory.c owner_log): its heap's size, the part of it committed (the +// size itself unless the heap is lazy), and the physical memory it mapped. No lock: a log may read a figure late. +void mem_guest_usage(uint64_t *heap, uint64_t *heap_committed, uint64_t *phys_maps); uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr); // 0x01 uint32_t mem_set_permission(uint64_t a, uint64_t n, uint32_t perm); // 0x02 uint32_t mem_set_attribute(uint64_t a, uint64_t n, uint32_t mask, uint32_t at); // 0x03 diff --git a/runtime/memory.c b/runtime/memory.c index 6940c49c..08bbb0c0 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -1806,6 +1806,12 @@ void mem_guest_census(uint64_t frame) { __atomic_load_n(&heap_size, __ATOMIC_ACQUIRE) / 1048576.0, (unsigned long long)heap_lazy_faults); } +void mem_guest_usage(uint64_t *heap, uint64_t *heap_committed, uint64_t *phys_maps) { + *heap = __atomic_load_n(&heap_size, __ATOMIC_RELAXED); + *heap_committed = heap_lazy ? __atomic_load_n(&heap_lazy_bytes, __ATOMIC_RELAXED) : *heap; + *phys_maps = __atomic_load_n(&alias_bytes, __ATOMIC_RELAXED); +} + // NX2NAVI_TRACE=perf "mem: sizes" (perf.c): the guest heap and physical maps, the kernel region's bump and commit, the // moved stacks, and the lazily written ranges with their array's room. void mem_sizes(PerfSizes *s) { diff --git a/runtime/vk.h b/runtime/vk.h index 3a30a322..3552b339 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -8,17 +8,30 @@ #pragma clang diagnostic pop // Every VMA allocation goes by the device-local heaps' live budgets (vk_vma_budget.c): a heap within the reserve of its // budget takes nothing new while another heap can. +// Each allocation is also counted under its owner, by the source file that made it, until it is freed through the +// wrappers below (vk_vma_owner_bytes, for the memory report: vk_memory.c owner_log). VkResult vk_vma_create_image(VmaAllocator vma, const VkImageCreateInfo *ici, const VmaAllocationCreateInfo *ai, VkImage *img, - VmaAllocation *a, VmaAllocationInfo *info); + VmaAllocation *a, VmaAllocationInfo *info, const char *file); VkResult vk_vma_create_buffer(VmaAllocator vma, const VkBufferCreateInfo *bci, const VmaAllocationCreateInfo *ai, - VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info); + VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info, const char *file); VkResult vk_vma_allocate_memory(VmaAllocator vma, const VkMemoryRequirements *req, const VmaAllocationCreateInfo *ai, - VmaAllocation *a, VmaAllocationInfo *info); + VmaAllocation *a, VmaAllocationInfo *info, const char *file); +void vk_vma_destroy_image(VmaAllocator vma, VkImage img, VmaAllocation a); +void vk_vma_destroy_buffer(VmaAllocator vma, VkBuffer buf, VmaAllocation a); +void vk_vma_free_memory(VmaAllocator vma, VmaAllocation a); uint64_t vk_vma_moved(void); // allocations placed outside a full heap so far +// The owners of VMA allocations (vk_vma_budget.c owner_of says which files make each one's). +enum { VK_OWN_TEXTURES, VK_OWN_POOLS, VK_OWN_UPSCALER, VK_OWN_REPLAY, VK_OWN_USNAP, VK_OWN_GPA, VK_OWN_ASTC, + VK_OWN_PRESENT, VK_OWN_HEADROOM, VK_OWN_OTHER, VK_OWN_N }; +// The bytes of an owner's live allocations on the device-local heaps (local 1) or on the others (local 0). No lock. +uint64_t vk_vma_owner_bytes(int owner, int local); #ifndef VK_VMA_BUDGET_IMPL -#define vmaCreateImage vk_vma_create_image -#define vmaCreateBuffer vk_vma_create_buffer -#define vmaAllocateMemory vk_vma_allocate_memory +#define vmaCreateImage(v, ici, ai, img, a, info) vk_vma_create_image(v, ici, ai, img, a, info, __FILE__) +#define vmaCreateBuffer(v, bci, ai, buf, a, info) vk_vma_create_buffer(v, bci, ai, buf, a, info, __FILE__) +#define vmaAllocateMemory(v, req, ai, a, info) vk_vma_allocate_memory(v, req, ai, a, info, __FILE__) +#define vmaDestroyImage vk_vma_destroy_image +#define vmaDestroyBuffer vk_vma_destroy_buffer +#define vmaFreeMemory vk_vma_free_memory #endif #include "nvn_obj.h" // pool_t, tex_t, view_t, samp_par, NVNcopyRegion (memory and textures block) #include "gfx.h" // GfxShader, GfxShaderRes, GfxShaderAnswers (pipelines block) @@ -762,10 +775,16 @@ void vk_texture_write_back(uint64_t addr, uint64_t size); // writes those imag void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc); // vk_pool_fini: drop the images there void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads); // Entries now, live and released (TextureFinalize, kept until sweep_released), with the bytes of their own memory -// (images not bound into pool memory). -typedef struct { uint64_t live, released, own_live, own_released, bytes_live, bytes_released; } VkTexCensus; +// (images not bound into pool memory). Of the live bytes: those whose content only the GPU made (render targets and +// what the GPU drew or copied into), and the idle ones no lookup reached lately (vk_texture.c idle, what the VRAM +// pressure may evict); of the released bytes: those released too lately for any bound to drop them (stale_after). +typedef struct { + uint64_t live, released, own_live, own_released, bytes_live, bytes_released; + uint64_t bytes_live_gpu, bytes_live_idle, bytes_released_young; +} VkTexCensus; void vk_texture_census(VkTexCensus *s); void vk_texture_census_log(uint64_t frame); // vk_memory.c vram_log: the live entries' memory by kind and format +void vk_texture_reuse_log(uint64_t frame); // vk_memory.c owner_log: released textures taken back, made again // The holes in VMA's blocks that hold textures (vk_texture.c "compaction", vk_tex_compact.h), for vk_memory.c holes_log. // A VkBlockRow is an allocation of another kind that the caller knows to lie in one of VMA's blocks (a VRAM mirror): // the block's device memory handle and size, the allocation's size and memory type. diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 164a7b55..91e75c13 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -966,6 +966,7 @@ static int dead_pending(void); static void vram_log(uint64_t frame); #define HOLES_PERIOD 3600 // frames between two "mem: holes" lines (holes_log) static void holes_log(uint64_t frame); +static void owner_log(uint64_t frame); // the memory report, after the staging pool (vk_mem_give_back) static void compact_look(void); static void outside_watch(uint64_t frame); static void up_ts_log(uint64_t frame); @@ -983,7 +984,7 @@ void vk_mem_census(uint64_t frame) { if (dmabuf_on && frame % 60 == 0) span_gc(frame, 0); static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); - if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) holes_log(frame); + if (frame && frame % ((rt_trace & RT_TRACE_PERF) ? 600 : HOLES_PERIOD) == 0) holes_log(frame), owner_log(frame); if (vk_texture_compact_due()) compact_look(); // NX2NAVI_TEX_COMPACT: 0 at once with the setting off if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); @@ -5346,6 +5347,88 @@ uint64_t vk_mem_give_back(void) { return freed + vk_headroom_release(); } +// ---- The memory report: every HOLES_PERIOD presents (600 with NX2NAVI_TRACE=perf), after the holes line, in every run. +// The device-local memory the driver counts for the game split by what holds it: VMA's allocations by owner +// (vk_vma_budget.c, "owners"), with the textures split further by the texture cache's census (vk_texture_census) and +// the pools into VRAM mirrors and physical pools; the free space inside VMA's blocks; and what VMA never allocated +// (compiled pipelines, descriptor memory, the swapchain, the DLSS and frame generation features, the driver's own), +// with DLSS's share as its statistics report it. Then the host-visible allocations (staging above all) and the +// process's system memory with the game's own heap. Then the released textures' reuse (vk_texture_reuse_log). +// A category that only climbs from line to line over a long session is a leak. The figures are sums the owners keep +// already, one driver query, VMA's budget query and one walk of the texture entries (vk_texture_census): well under a +// millisecond once a minute, at the present that logs. +static void owner_log(uint64_t frame) { + VkCtx *c = vk_ctx(); + if (!c || !c->vma) return; + VkPhysicalDeviceMemoryBudgetPropertiesEXT bp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + if (c->has_memory_budget) { + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bp}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + } + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + uint64_t usage = 0, blocks = 0, allocs = 0, hblocks = 0, hallocs = 0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) { + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) + usage += c->has_memory_budget ? bp.heapUsage[h] : vb[h].statistics.blockBytes, + blocks += vb[h].statistics.blockBytes, allocs += vb[h].statistics.allocationBytes; + else hblocks += vb[h].statistics.blockBytes, hallocs += vb[h].statistics.allocationBytes; + } + uint64_t phys = 0; + plat_mutex_lock_shared(&pool_lock); + for (int i = 0; i < nphys; i++) + if (phys_list[i]->alloc) phys += phys_list[i]->size; + plat_mutex_unlock_shared(&pool_lock); + VkTexCensus t; + vk_texture_census(&t); + uint64_t own[VK_OWN_N], hown[VK_OWN_N]; + for (int o = 0; o < VK_OWN_N; o++) own[o] = vk_vma_owner_bytes(o, 1), hown[o] = vk_vma_owner_bytes(o, 0); + uint64_t stg_blocks = 0, kept; + if (stg_pool) { + VmaStatistics st; + vmaGetPoolStatistics(c->vma, stg_pool, &st); + stg_blocks = st.blockBytes; + } + // Read without their locks: an upload holds cmd_lock while it waits for a batch (stg_create), tens of milliseconds + // the present would wait too; a figure a log reads a moment late does no harm. + kept = __atomic_load_n(&stg_cache_bytes, __ATOMIC_RELAXED) + + (uint64_t)__atomic_load_n(&narena_free, __ATOMIC_RELAXED) * ARENA_CHUNK; + uint64_t snaps = __atomic_load_n(&snap_pool_bytes, __ATOMIC_RELAXED); + int64_t dlss = vk_dlaa_vram(); + char ds[48] = "nothing"; + if (dlss >= 0) snprintf(ds, sizeof ds, "%.0f MiB", dlss / 1048576.0); + const double M = 1048576.0; + uint64_t mir = (uint64_t)__atomic_load_n(&m_live_bytes, __ATOMIC_RELAXED); + rt_log("mem: by owner frame %llu: device-local in use %.0f MiB (the driver's figure) = VMA's allocations %.0f + free " + "inside VMA's blocks %.0f + outside VMA %.0f (compiled pipelines, descriptor memory, the swapchain, DLSS and " + "frame generation features, the driver; DLSS reports %s). VMA's allocations: textures %.0f (live %.0f, of them " + "render targets and other GPU-drawn %.0f and idle %.0f; released %.0f, of them too lately to drop %.0f), pools " + "%.0f (VRAM mirrors %.0f, physical pools %.0f), upscalers %.0f, replay buffers %.0f, uniform snapshots %.0f, " + "device address table %.0f, ASTC decoder %.0f, present and post-processing %.0f, headroom spares %.0f, other " + "%.0f", (unsigned long long)frame, usage / M, allocs / M, blocks > allocs ? (blocks - allocs) / M : 0.0, + usage > blocks ? (usage - blocks) / M : 0.0, ds, own[VK_OWN_TEXTURES] / M, t.bytes_live / M, + t.bytes_live_gpu / M, t.bytes_live_idle / M, t.bytes_released / M, t.bytes_released_young / M, + own[VK_OWN_POOLS] / M, mir / M, phys / M, own[VK_OWN_UPSCALER] / M, own[VK_OWN_REPLAY] / M, + own[VK_OWN_USNAP] / M, own[VK_OWN_GPA] / M, own[VK_OWN_ASTC] / M, own[VK_OWN_PRESENT] / M, + own[VK_OWN_HEADROOM] / M, own[VK_OWN_OTHER] / M); + double proc, sys_used, sys_limit; + float ram_used, ram_total, ws; + plat_commit_info(&proc, &sys_used, &sys_limit); + plat_mem_info(&ram_used, &ram_total, &ws); + uint64_t heap, heap_c, maps; + mem_guest_usage(&heap, &heap_c, &maps); + rt_log("mem: host frame %llu: VMA's host-visible blocks %.0f MiB hold %.0f MiB: staging pool blocks %.0f (kept staging " + "buffers and arena chunks %.0f), write-back snapshots %.0f; by owner: pools and staging %.0f, textures %.0f, " + "upscalers %.0f, replay buffers %.0f, uniform snapshots %.0f, present and post-processing (the flight " + "recorder's thumbnails among them) %.0f, other %.0f. Process commit %.0f MB, working set %.0f MB; system commit " + "%.0f of %.0f MB; the game's heap %.0f MiB (%.0f committed), its physical maps %.0f MiB", + (unsigned long long)frame, hblocks / M, hallocs / M, stg_blocks / M, kept / M, snaps / M, hown[VK_OWN_POOLS] / M, + hown[VK_OWN_TEXTURES] / M, hown[VK_OWN_UPSCALER] / M, hown[VK_OWN_REPLAY] / M, hown[VK_OWN_USNAP] / M, + hown[VK_OWN_PRESENT] / M, (hown[VK_OWN_OTHER] + hown[VK_OWN_GPA] + hown[VK_OWN_ASTC] + hown[VK_OWN_HEADROOM]) / M, + proc, (double)ws, sys_used, sys_limit, heap / M, heap_c / M, maps / M); + vk_texture_reuse_log(frame); +} + // NX2NAVI_STAGING_FAIL_AT=n (test): the n-th staging buffer the driver makes is taken as refused for memory once, so // the give-back runs (vk_mem_give_back). static int staging_fail_now(void) { diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index edf46f74..fc4104bc 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -725,6 +725,9 @@ static int slot_names(const Tex *e) { enum { DROP_NONE, DROP_MAX, DROP_CAP, DROP_COMMIT, DROP_VRAM, DROP_IDLE_VRAM, DROP_IDLE_COMMIT, DROP_DISUSE, DROP_OVERWRITTEN, DROP_REPLACED, DROP_REUSED, DROP_RANGE, DROP_EXACT, DROP_SHUTDOWN, DROP_COMPACT, DROP_N }; static void drop_tex(Tex *e, int why); +static void reuse_dropped(const Tex *e, int why); // "reuse of released textures", after vk_texture_census +static void reuse_made(const Tex *e); +static void reuse_taken_back(const Tex *e, uint64_t gap); static int compact_on(void); // NX2NAVI_TEX_COMPACT ("compaction", near the end of the file) static void compact_step(void); static void compact_forget(Tex *e); @@ -2015,6 +2018,7 @@ static void drop_tex(Tex *e, int why) { else if (wb_wanted(e)) write_back_n(&e, 1), n_drop_wb++; } n_drop_by[why]++; + reuse_dropped(e, why); if (rt_trace & RT_TRACE_PERF) drops[ndrops++ % DROPS_N] = (DropRec){key_hash(&e->key), e->key.addr, frames_now(), (uint32_t)e->t.d.format, (uint32_t)e->t.d.flags, e->t.d.width, e->t.d.height, (uint8_t)why, @@ -4246,6 +4250,7 @@ static Tex *entry_of(const tex_t *t) { } if (!(e = create_tex(t, &k, tex_alias))) return create_failed(&k), NULL; origin_made(e); + reuse_made(e); perf_count(PERF_CNT_TEX_NEW); start_at(e, &k, 0); flight_ev(e, FLIGHT_TEX_NEW, e->og_start, 0); @@ -4263,6 +4268,7 @@ static Tex *entry_of(const tex_t *t) { e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one uint64_t gap = e->samp_frame - e->rel_frame; if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; + reuse_taken_back(e, gap); e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); if (e->pidx) pent[e->pidx - 1].rel = 0; // the pool's check looks at it again e->overwritten = 0; @@ -6324,25 +6330,99 @@ void vk_texture_stats(uint64_t *images, uint64_t *uploads, uint64_t *reuploads) unlock(); } +// A walk over the entries under the texture lock, with the sizes the entries keep (Tex.bytes): about 0.1 ms with ten +// thousand of them, at most once a HOLES_PERIOD (vk_memory.c owner_log) without the perf trace. void vk_texture_census(VkTexCensus *s) { - VkCtx *c = vk_ctx(); memset(s, 0, sizeof *s); lock(); + uint64_t now = frames_now(); for (uint32_t i = 0; i < by_key.cap; i++) { const Tex *e = by_key.vals[i]; if (!e) continue; - uint64_t bytes = 0; - if (e->alloc && c) { - VmaAllocationInfo ai; - vmaGetAllocationInfo(c->vma, e->alloc, &ai); - bytes = ai.size; + uint64_t bytes = e->alloc ? e->bytes : 0; + if (e->released) { + s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes; + if (now - e->rel_frame < stale_after(e)) s->bytes_released_young += bytes; + } else { + s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes; + if (!comes_back_intact(e)) s->bytes_live_gpu += bytes; + if (idle(e, now)) s->bytes_live_idle += bytes; } - if (e->released) s->released++, s->own_released += e->alloc != NULL, s->bytes_released += bytes; - else s->live++, s->own_live += e->alloc != NULL, s->bytes_live += bytes; } unlock(); } +// ---- reuse of released textures (vk_texture_reuse_log) +// How often a released entry is taken back and how long it had been released then, and how often an entry is made +// again after a drop that a bound or a pressure made (the image made and its content uploaded or decoded once more). +// The two tell what the bounds on the released entries cost: an entry taken back after minutes is one a smaller bound +// may have dropped, and one made again soon after its drop is one the bound did drop. Counted since the last line; the +// dropped keys are remembered in a table of DROPPED_N slots indexed by the key's hash, a newer drop taking the slot. +enum { AGE_YOUNG, AGE_10S, AGE_1M, AGE_5M, AGE_OLD, AGE_N }; +static const char *const age_names[AGE_N] = {"under 2 s", "2-10 s", "10-60 s", "1-5 min", "longer"}; +static uint64_t rb_n[AGE_N], rb_bytes[AGE_N]; // take-backs by the time released (tex_lock) +static uint64_t ra_n[AGE_N], ra_bytes, ra_why[DROP_N]; // made again by the time since the drop, and by its reason +#define DROPPED_N 8192 +static struct { uint64_t kh, frame; uint8_t why; } dropped[DROPPED_N]; // tex_lock + +static int age_of(uint64_t gap) { // presents, at the presents per second of now (PRESENTS) + return gap < STALE_FRAMES ? AGE_YOUNG : gap < PRESENTS(600) ? AGE_10S : gap < PRESENTS(3600) ? AGE_1M + : gap < PRESENTS(18000) ? AGE_5M : AGE_OLD; +} + +static void reuse_taken_back(const Tex *e, uint64_t gap) { // tex_lock held; e was released for gap presents + int a = age_of(gap); + rb_n[a]++, rb_bytes[a] += e->alloc ? e->bytes : 0; +} + +// Drops of an entry the game may still ask for: past a bound, or under a pressure, or unsampled. +static int drop_costs(int why) { + return why == DROP_MAX || why == DROP_CAP || why == DROP_COMMIT || why == DROP_VRAM || why == DROP_IDLE_VRAM || + why == DROP_IDLE_COMMIT || why == DROP_DISUSE; +} + +static void reuse_dropped(const Tex *e, int why) { // tex_lock held + if (!drop_costs(why)) return; + uint64_t kh = key_hash(&e->key); + dropped[kh % DROPPED_N].kh = kh, dropped[kh % DROPPED_N].frame = frames_now(), dropped[kh % DROPPED_N].why = (uint8_t)why; +} + +static void reuse_made(const Tex *e) { // tex_lock held; e was just made + uint64_t kh = key_hash(&e->key); + if (dropped[kh % DROPPED_N].kh != kh || !dropped[kh % DROPPED_N].why) return; + int a = age_of(frames_now() - dropped[kh % DROPPED_N].frame); + ra_n[a]++, ra_bytes += e->alloc ? e->bytes : 0, ra_why[dropped[kh % DROPPED_N].why]++; + dropped[kh % DROPPED_N].why = 0; +} + +void vk_texture_reuse_log(uint64_t frame) { + char line[1024]; + const double M = 1048576.0; + lock(); + int o = snprintf(line, sizeof line, "vk: released textures frame %llu, since the last line: taken back by the time " + "released:", (unsigned long long)frame); + for (int a = 0; a < AGE_N; a++) + o += snprintf(line + o, sizeof line - (size_t)o, " %s %llu (%.0f MiB);", age_names[a], (unsigned long long)rb_n[a], + rb_bytes[a] / M); + uint64_t again = 0; + for (int a = 0; a < AGE_N; a++) again += ra_n[a]; + o += snprintf(line + o, sizeof line - (size_t)o, " made again after a drop %llu (%.0f MiB; by the time since the drop:", + (unsigned long long)again, ra_bytes / M); + for (int a = 0; a < AGE_N; a++) + o += snprintf(line + o, sizeof line - (size_t)o, "%s %s %llu", a ? "," : "", age_names[a], (unsigned long long)ra_n[a]); + snprintf(line + o, sizeof line - (size_t)o, "; by the drop: released past the size bound %llu, past the count %llu, " + "VRAM pressure %llu released and %llu idle, commit %llu released and %llu idle, unsampled %llu); now %d " + "released, %.0f MiB of a bound of %.0f MiB", (unsigned long long)ra_why[DROP_CAP], + (unsigned long long)ra_why[DROP_MAX], (unsigned long long)ra_why[DROP_VRAM], + (unsigned long long)ra_why[DROP_IDLE_VRAM], (unsigned long long)ra_why[DROP_COMMIT], + (unsigned long long)ra_why[DROP_IDLE_COMMIT], (unsigned long long)ra_why[DROP_DISUSE], nreleased, + rel_bytes / M, released_cap_bytes() == UINT64_MAX ? 0.0 : released_cap_bytes() / M); + memset(rb_n, 0, sizeof rb_n), memset(rb_bytes, 0, sizeof rb_bytes), memset(ra_n, 0, sizeof ra_n); + memset(ra_why, 0, sizeof ra_why), ra_bytes = 0; + unlock(); + rt_log("%s", line); +} + // With NX2NAVI_TRACE=perf every 600 frames, after vk_memory.c's vram line: what the live entries' own memory holds. By // kind: decoded ASTC (as BC3 or RGBA8) and other textures whose content the CPU provides, all of which a re-upload // brings back (comes_back_intact), and the GPU-written ones (render targets, textures the GPU drew or copied into), diff --git a/runtime/vk_vma_budget.c b/runtime/vk_vma_budget.c index b70a849d..3a7a887d 100644 --- a/runtime/vk_vma_budget.c +++ b/runtime/vk_vma_budget.c @@ -1,5 +1,5 @@ // Video memory by the heaps' live budgets, for every allocation through VMA (vk.h maps vmaCreateImage, vmaCreateBuffer -// and vmaAllocateMemory here). +// and vmaAllocateMemory here, and the frees that go with them), and the allocations' sums by owner ("owners", below). // // An RX 580 (8 GiB, AMD 23.10) has a 6 GiB device-local heap and a 2 GiB device-local, host-visible one. The heap limits // of vk_device.c (vram_heap_limits) are fixed when the device is made: each heap's budget then, less a tenth for the @@ -19,6 +19,7 @@ // headroom's anchors, vk_headroom.c) are made as asked. #define VK_VMA_BUDGET_IMPL #include +#include #include "vk.h" #include "cfg.h" #include "host.h" @@ -83,42 +84,121 @@ static uint32_t all_bits(VmaAllocator vma, uint32_t bits) { return mp->memoryTypeCount >= 32 ? ~0u : (1u << mp->memoryTypeCount) - 1; } +// ---- owners +// Every allocation made through the wrappers below is counted under the owner of the file that made it, the device-local +// heaps apart from the others, and leaves the count when vk.h's vmaDestroyImage, vmaDestroyBuffer or +// vmaFreeMemory frees it. The allocation's user data (unused otherwise) holds 1 + owner * 2 + local, so the free knows +// where it was counted; one made before its owner was known, or by a file that does not include vk.h, has none and is +// never counted. The memory report (vk_memory.c owner_log) reads the sums: a category that only climbs over a session +// is where a leak shows. A count costs one VMA info query and two atomic adds per allocation and per free. +static volatile int64_t own_bytes[VK_OWN_N][2]; + +static int owner_of(const char *file) { + static const struct { const char *name; int owner; } map[] = { + {"vk_texture.c", VK_OWN_TEXTURES}, {"vk_memory.c", VK_OWN_POOLS}, // mirrors, physical pools, staging + {"vk_dlaa.c", VK_OWN_UPSCALER}, {"vk_fsr.c", VK_OWN_UPSCALER}, {"vk_dlssg.c", VK_OWN_UPSCALER}, + {"vk_exec_draw.c", VK_OWN_REPLAY}, {"vk_exec.c", VK_OWN_REPLAY}, {"vk_exec_xfer.c", VK_OWN_REPLAY}, + {"vk_mesh_fb.c", VK_OWN_REPLAY}, {"vk_usnap.c", VK_OWN_USNAP}, {"vk_gpa.c", VK_OWN_GPA}, + {"vk_astc.c", VK_OWN_ASTC}, {"gfx_present.c", VK_OWN_PRESENT}, {"vk_hdr.c", VK_OWN_PRESENT}, + {"vk_tonemap.c", VK_OWN_PRESENT}, {"vk_aa.c", VK_OWN_PRESENT}, {"vk_color.c", VK_OWN_PRESENT}, + {"vk_headroom.c", VK_OWN_HEADROOM}, + }; + if (!file) return VK_OWN_OTHER; + const char *b = file; + for (const char *p = file; *p; p++) + if (*p == '/' || *p == '\\') b = p + 1; + for (unsigned i = 0; i < sizeof map / sizeof *map; i++) + if (!strcmp(b, map[i].name)) return map[i].owner; + return VK_OWN_OTHER; +} + +static void own_add(VmaAllocator vma, VmaAllocation a, const char *file) { + if (!a) return; + VmaAllocationInfo i; + vmaGetAllocationInfo(vma, a, &i); + const VkPhysicalDeviceMemoryProperties *mp; + vmaGetMemoryProperties(vma, &mp); + int local = i.memoryType < mp->memoryTypeCount && + (mp->memoryHeaps[mp->memoryTypes[i.memoryType].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT); + int owner = owner_of(file); + vmaSetAllocationUserData(vma, a, (void *)(uintptr_t)(1 + owner * 2 + local)); + __atomic_add_fetch(&own_bytes[owner][local], (int64_t)i.size, __ATOMIC_RELAXED); +} + +static void own_sub(VmaAllocator vma, VmaAllocation a) { + if (!a) return; + VmaAllocationInfo i; + vmaGetAllocationInfo(vma, a, &i); + uintptr_t tag = (uintptr_t)i.pUserData; + if (!tag || tag > VK_OWN_N * 2) return; + __atomic_sub_fetch(&own_bytes[(tag - 1) / 2][(tag - 1) & 1], (int64_t)i.size, __ATOMIC_RELAXED); +} + +uint64_t vk_vma_owner_bytes(int owner, int local) { + if ((unsigned)owner >= VK_OWN_N) return 0; + int64_t v = __atomic_load_n(&own_bytes[owner][local != 0], __ATOMIC_RELAXED); + return v > 0 ? (uint64_t)v : 0; +} + +void vk_vma_destroy_image(VmaAllocator vma, VkImage img, VmaAllocation a) { + own_sub(vma, a); + vmaDestroyImage(vma, img, a); +} + +void vk_vma_destroy_buffer(VmaAllocator vma, VkBuffer buf, VmaAllocation a) { + own_sub(vma, a); + vmaDestroyBuffer(vma, buf, a); +} + +void vk_vma_free_memory(VmaAllocator vma, VmaAllocation a) { + own_sub(vma, a); + vmaFreeMemory(vma, a); +} + +static VkResult made(VkResult r, VmaAllocator vma, VmaAllocation *a, const char *file) { + if (r == VK_SUCCESS) own_add(vma, *a, file); + return r; +} + VkResult vk_vma_create_image(VmaAllocator vma, const VkImageCreateInfo *ici, const VmaAllocationCreateInfo *ai, VkImage *img, - VmaAllocation *a, VmaAllocationInfo *info) { + VmaAllocation *a, VmaAllocationInfo *info, const char *file) { if (!ai->pool) { uint32_t bits = all_bits(vma, ai->memoryTypeBits), nb = budget_bits(vma, bits); if (nb != bits) { VmaAllocationCreateInfo b = *ai; b.memoryTypeBits = nb; - if (vmaCreateImage(vma, ici, &b, img, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + if (vmaCreateImage(vma, ici, &b, img, a, info) == VK_SUCCESS) + return note_moved(), made(VK_SUCCESS, vma, a, file); } } - return vmaCreateImage(vma, ici, ai, img, a, info); + return made(vmaCreateImage(vma, ici, ai, img, a, info), vma, a, file); } VkResult vk_vma_create_buffer(VmaAllocator vma, const VkBufferCreateInfo *bci, const VmaAllocationCreateInfo *ai, - VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info) { + VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info, const char *file) { if (!ai->pool) { uint32_t bits = all_bits(vma, ai->memoryTypeBits), nb = budget_bits(vma, bits); if (nb != bits) { VmaAllocationCreateInfo b = *ai; b.memoryTypeBits = nb; - if (vmaCreateBuffer(vma, bci, &b, buf, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + if (vmaCreateBuffer(vma, bci, &b, buf, a, info) == VK_SUCCESS) + return note_moved(), made(VK_SUCCESS, vma, a, file); } } - return vmaCreateBuffer(vma, bci, ai, buf, a, info); + return made(vmaCreateBuffer(vma, bci, ai, buf, a, info), vma, a, file); } VkResult vk_vma_allocate_memory(VmaAllocator vma, const VkMemoryRequirements *req, const VmaAllocationCreateInfo *ai, - VmaAllocation *a, VmaAllocationInfo *info) { + VmaAllocation *a, VmaAllocationInfo *info, const char *file) { uint32_t bits = req->memoryTypeBits & all_bits(vma, ai->memoryTypeBits); if (!ai->pool && (bits & (bits - 1))) { // more than one type: one alone is asked for on purpose uint32_t nb = budget_bits(vma, bits); if (nb != bits) { VkMemoryRequirements r = *req; r.memoryTypeBits = nb; - if (vmaAllocateMemory(vma, &r, ai, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + if (vmaAllocateMemory(vma, &r, ai, a, info) == VK_SUCCESS) + return note_moved(), made(VK_SUCCESS, vma, a, file); } } - return vmaAllocateMemory(vma, req, ai, a, info); + return made(vmaAllocateMemory(vma, req, ai, a, info), vma, a, file); } diff --git a/tests/gpa_check.c b/tests/gpa_check.c index 47a1c24d..7c58c2dd 100644 --- a/tests/gpa_check.c +++ b/tests/gpa_check.c @@ -38,9 +38,10 @@ void rt_log(const char *fmt, ...) { static const uint32_t *table; // the first buffer vk_gpa.c creates static uintptr_t next_handle = 0x1000; -VkResult vmaCreateBuffer(VmaAllocator a, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *buf, - VmaAllocation *alloc, VmaAllocationInfo *info) { - (void)a, (void)ai; +// vk.h sends vmaCreateBuffer and vmaDestroyBuffer to these wrappers (vk_vma_budget.c) +VkResult vk_vma_create_buffer(VmaAllocator a, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *buf, + VmaAllocation *alloc, VmaAllocationInfo *info, const char *file) { + (void)a, (void)ai, (void)file; void *mem = calloc(1, (size_t)bi->size); if (!table) table = mem; *buf = (VkBuffer)(next_handle += 0x1000);